envio 3.3.0-alpha.8 → 3.3.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/licenses/CLA.md +35 -0
- package/licenses/EULA.md +67 -0
- package/licenses/LICENSE.md +45 -0
- package/licenses/README.md +35 -0
- package/package.json +9 -8
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +0 -7
- package/src/BatchProcessing.res.mjs +1 -8
- package/src/ChainFetching.res +30 -18
- package/src/ChainFetching.res.mjs +23 -13
- package/src/ChainState.res +205 -59
- package/src/ChainState.res.mjs +130 -27
- package/src/ChainState.resi +16 -1
- package/src/Config.res +9 -5
- package/src/Config.res.mjs +2 -2
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +12 -0
- package/src/CrossChainState.res +147 -39
- package/src/CrossChainState.res.mjs +42 -10
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +30 -56
- package/src/EventConfigBuilder.res.mjs +24 -30
- package/src/EventProcessing.res +8 -5
- package/src/EventProcessing.res.mjs +2 -1
- package/src/FetchState.res +842 -474
- package/src/FetchState.res.mjs +545 -356
- package/src/HandlerLoader.res +1 -1
- package/src/HandlerLoader.res.mjs +1 -1
- package/src/HandlerRegister.res +478 -305
- package/src/HandlerRegister.res.mjs +277 -209
- package/src/HandlerRegister.resi +11 -6
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +15 -39
- package/src/IndexerState.res.mjs +18 -29
- package/src/IndexerState.resi +2 -10
- package/src/Internal.res +121 -18
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/LogSelection.res +92 -217
- package/src/LogSelection.res.mjs +95 -184
- package/src/Main.res +29 -143
- package/src/Main.res.mjs +25 -88
- package/src/Metrics.res +1 -1
- package/src/Metrics.res.mjs +1 -1
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/PruneStaleHistory.res +146 -35
- package/src/PruneStaleHistory.res.mjs +114 -20
- package/src/RawEvent.res +2 -2
- package/src/Rollback.res +32 -13
- package/src/Rollback.res.mjs +24 -11
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateDeadInputTracker.res +1 -1
- package/src/SimulateItems.res +39 -7
- package/src/SimulateItems.res.mjs +28 -9
- package/src/TestIndexer.res +2 -2
- package/src/TestIndexer.res.mjs +2 -2
- package/src/TopicFilter.res +1 -25
- package/src/TopicFilter.res.mjs +0 -74
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +60 -21
- package/src/Writing.res.mjs +27 -9
- package/src/bindings/NodeJs.res +5 -0
- package/src/bindings/Viem.res +0 -5
- package/src/sources/EventRouter.res +0 -21
- package/src/sources/EventRouter.res.mjs +0 -12
- package/src/sources/Evm.res +4 -1
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/EvmChain.res +2 -27
- package/src/sources/EvmChain.res.mjs +2 -23
- package/src/sources/EvmRpcClient.res +58 -23
- package/src/sources/EvmRpcClient.res.mjs +11 -5
- package/src/sources/Fuel.res +3 -1
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/HyperSync.res +9 -38
- package/src/sources/HyperSync.res.mjs +16 -28
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +88 -11
- package/src/sources/HyperSyncClient.res.mjs +39 -6
- package/src/sources/HyperSyncSource.res +18 -210
- package/src/sources/HyperSyncSource.res.mjs +9 -137
- package/src/sources/Rpc.res +0 -32
- package/src/sources/Rpc.res.mjs +1 -46
- package/src/sources/RpcSource.res +129 -533
- package/src/sources/RpcSource.res.mjs +161 -379
- package/src/sources/SimulateSource.res +37 -19
- package/src/sources/SimulateSource.res.mjs +27 -10
- package/src/sources/SourceManager.res +4 -14
- package/src/sources/SourceManager.res.mjs +3 -9
- package/src/sources/SourceManager.resi +0 -2
- package/src/sources/SvmHyperSyncSource.res +13 -3
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
package/src/UserContext.res
CHANGED
|
@@ -28,10 +28,23 @@ Utils.Object.defineProperty(
|
|
|
28
28
|
},
|
|
29
29
|
)
|
|
30
30
|
%%raw(`
|
|
31
|
-
|
|
31
|
+
// Top-level so the getter is created once, not per context. \`this\` is the
|
|
32
|
+
// EffectContext instance; only cross-chain contexts install it.
|
|
33
|
+
function throwCrossChainChainAccess() {
|
|
34
|
+
throw new Error('context.chain is not available on the cross-chain effect "' + this._effectName + '". Set \`crossChain: false\` in its options to scope the effect to a single chain, then read context.chain.id.');
|
|
35
|
+
}
|
|
36
|
+
var crossChainChainDescriptor = { get: throwCrossChainChainAccess, enumerable: true };
|
|
37
|
+
var EffectContext = function(params, chainId, effectName, defaultShouldCache, callEffect) {
|
|
32
38
|
paramsByThis.set(this, params);
|
|
33
39
|
this.effect = callEffect;
|
|
34
40
|
this.cache = defaultShouldCache;
|
|
41
|
+
if (chainId === undefined) {
|
|
42
|
+
// Cross-chain: reading context.chain throws, via the shared getter.
|
|
43
|
+
Object.defineProperty(this, "_effectName", { value: effectName });
|
|
44
|
+
Object.defineProperty(this, "chain", crossChainChainDescriptor);
|
|
45
|
+
} else {
|
|
46
|
+
this.chain = { id: chainId };
|
|
47
|
+
}
|
|
35
48
|
};
|
|
36
49
|
EffectContext.prototype = effectContextPrototype;
|
|
37
50
|
`)
|
|
@@ -39,35 +52,61 @@ EffectContext.prototype = effectContextPrototype;
|
|
|
39
52
|
@new
|
|
40
53
|
external makeEffectContext: (
|
|
41
54
|
contextParams,
|
|
55
|
+
~chainId: option<int>,
|
|
56
|
+
~effectName: string,
|
|
42
57
|
~defaultShouldCache: bool,
|
|
43
58
|
~callEffect: (Internal.effect, Internal.effectInput) => promise<Internal.effectOutput>,
|
|
44
59
|
) => Internal.effectContext = "EffectContext"
|
|
45
60
|
|
|
46
61
|
let initEffect = (params: contextParams) => {
|
|
47
|
-
let
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
62
|
+
let handlerChainId = params.item->Internal.getItemChainId
|
|
63
|
+
// A chain-scoped effect always resolves against the chain of the handler that
|
|
64
|
+
// triggered the call, even several effects deep, so the chain id is captured
|
|
65
|
+
// once from the item and reused for the whole nested-call tree.
|
|
66
|
+
let rec makeCaller = (~caller: option<Internal.effect>) => {
|
|
67
|
+
(effect: Internal.effect, input: Internal.effectInput) => {
|
|
68
|
+
let scope = effect.crossChain ? Internal.CrossChain : Internal.Chain(handlerChainId)
|
|
69
|
+
|
|
70
|
+
switch caller {
|
|
71
|
+
| Some(callerEffect) if callerEffect.crossChain && !effect.crossChain =>
|
|
72
|
+
// A cross-chain effect isn't tied to a single chain, so it has no chain
|
|
73
|
+
// to resolve a chain-scoped child against. Reject before any cache work.
|
|
74
|
+
JsError.throwWithMessage(
|
|
75
|
+
`The cross-chain effect "${callerEffect.name}" cannot call the chain-scoped effect "${effect.name}", because a cross-chain effect isn't tied to a single chain. Make "${effect.name}" cross-chain (\`crossChain: true\`), or make "${callerEffect.name}" chain-scoped (\`crossChain: false\`).`,
|
|
76
|
+
)
|
|
77
|
+
| _ => ()
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
let effectContext = makeEffectContext(
|
|
81
|
+
params,
|
|
82
|
+
~chainId=switch scope {
|
|
83
|
+
| Internal.Chain(chainId) => Some(chainId)
|
|
84
|
+
| Internal.CrossChain => None
|
|
85
|
+
},
|
|
86
|
+
~effectName=effect.name,
|
|
87
|
+
~defaultShouldCache=effect.defaultShouldCache,
|
|
88
|
+
~callEffect=makeCaller(~caller=Some(effect)),
|
|
89
|
+
)
|
|
90
|
+
let effectArgs: Internal.effectArgs = {
|
|
91
|
+
input,
|
|
92
|
+
context: effectContext,
|
|
93
|
+
cacheKey: input->S.reverseConvertOrThrow(effect.input)->Utils.Hash.makeOrThrow,
|
|
94
|
+
checkpointId: params.checkpointId,
|
|
95
|
+
}
|
|
96
|
+
LoadLayer.loadEffect(
|
|
97
|
+
~loadManager=params.loadManager,
|
|
98
|
+
~persistence=params.persistence,
|
|
99
|
+
~effect,
|
|
100
|
+
~effectArgs,
|
|
101
|
+
~scope,
|
|
102
|
+
~indexerState=params.indexerState,
|
|
103
|
+
~shouldGroup=params.isPreload,
|
|
104
|
+
~item=params.item,
|
|
105
|
+
~ecosystem=params.config.ecosystem,
|
|
106
|
+
)
|
|
58
107
|
}
|
|
59
|
-
LoadLayer.loadEffect(
|
|
60
|
-
~loadManager=params.loadManager,
|
|
61
|
-
~persistence=params.persistence,
|
|
62
|
-
~effect,
|
|
63
|
-
~effectArgs,
|
|
64
|
-
~indexerState=params.indexerState,
|
|
65
|
-
~shouldGroup=params.isPreload,
|
|
66
|
-
~item=params.item,
|
|
67
|
-
~ecosystem=params.config.ecosystem,
|
|
68
|
-
)
|
|
69
108
|
}
|
|
70
|
-
|
|
109
|
+
makeCaller(~caller=None)
|
|
71
110
|
}
|
|
72
111
|
|
|
73
112
|
type entityContextParams = {
|
package/src/UserContext.res.mjs
CHANGED
|
@@ -25,17 +25,37 @@ Object.defineProperty(effectContextPrototype, "log", {
|
|
|
25
25
|
}
|
|
26
26
|
});
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
// Top-level so the getter is created once, not per context. \`this\` is the
|
|
29
|
+
// EffectContext instance; only cross-chain contexts install it.
|
|
30
|
+
function throwCrossChainChainAccess() {
|
|
31
|
+
throw new Error('context.chain is not available on the cross-chain effect "' + this._effectName + '". Set \`crossChain: false\` in its options to scope the effect to a single chain, then read context.chain.id.');
|
|
32
|
+
}
|
|
33
|
+
var crossChainChainDescriptor = { get: throwCrossChainChainAccess, enumerable: true };
|
|
34
|
+
var EffectContext = function(params, chainId, effectName, defaultShouldCache, callEffect) {
|
|
29
35
|
paramsByThis.set(this, params);
|
|
30
36
|
this.effect = callEffect;
|
|
31
37
|
this.cache = defaultShouldCache;
|
|
38
|
+
if (chainId === undefined) {
|
|
39
|
+
// Cross-chain: reading context.chain throws, via the shared getter.
|
|
40
|
+
Object.defineProperty(this, "_effectName", { value: effectName });
|
|
41
|
+
Object.defineProperty(this, "chain", crossChainChainDescriptor);
|
|
42
|
+
} else {
|
|
43
|
+
this.chain = { id: chainId };
|
|
44
|
+
}
|
|
32
45
|
};
|
|
33
46
|
EffectContext.prototype = effectContextPrototype;
|
|
34
47
|
;
|
|
35
48
|
|
|
36
49
|
function initEffect(params) {
|
|
37
|
-
let
|
|
38
|
-
|
|
50
|
+
let handlerChainId = Internal.getItemChainId(params.item);
|
|
51
|
+
let makeCaller = caller => ((effect, input) => {
|
|
52
|
+
let scope = effect.crossChain ? "crossChain" : handlerChainId;
|
|
53
|
+
if (caller !== undefined && caller.crossChain && !effect.crossChain) {
|
|
54
|
+
Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
|
|
55
|
+
}
|
|
56
|
+
let tmp;
|
|
57
|
+
tmp = scope === "crossChain" ? undefined : scope;
|
|
58
|
+
let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, makeCaller(effect));
|
|
39
59
|
let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
|
|
40
60
|
let effectArgs_checkpointId = params.checkpointId;
|
|
41
61
|
let effectArgs = {
|
|
@@ -44,9 +64,9 @@ function initEffect(params) {
|
|
|
44
64
|
cacheKey: effectArgs_cacheKey,
|
|
45
65
|
checkpointId: effectArgs_checkpointId
|
|
46
66
|
};
|
|
47
|
-
return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
|
|
48
|
-
};
|
|
49
|
-
return
|
|
67
|
+
return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
|
|
68
|
+
});
|
|
69
|
+
return makeCaller(undefined);
|
|
50
70
|
}
|
|
51
71
|
|
|
52
72
|
function getWhereHandler(params, filter) {
|
package/src/Writing.res
CHANGED
|
@@ -16,8 +16,8 @@ let getChangesCount = (state: IndexerState.t) => {
|
|
|
16
16
|
total := total.contents +. (state->InMemoryStore.getInMemTable(~entityConfig)).changesCount
|
|
17
17
|
})
|
|
18
18
|
state
|
|
19
|
-
->IndexerState.
|
|
20
|
-
->
|
|
19
|
+
->IndexerState.effectState
|
|
20
|
+
->EffectState.forEach(inMemTable => {
|
|
21
21
|
total := total.contents +. inMemTable.changesCount
|
|
22
22
|
})
|
|
23
23
|
state
|
|
@@ -38,9 +38,9 @@ let waitForCommit = (state: IndexerState.t): promise<unit> =>
|
|
|
38
38
|
let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updatedEffectCache> => {
|
|
39
39
|
let acc = []
|
|
40
40
|
state
|
|
41
|
-
->IndexerState.
|
|
42
|
-
->
|
|
43
|
-
let {idsToStore, dict, effect, invalidationsCount} = inMemTable
|
|
41
|
+
->IndexerState.effectState
|
|
42
|
+
->EffectState.forEach(inMemTable => {
|
|
43
|
+
let {idsToStore, dict, effect, invalidationsCount, scope, table} = inMemTable
|
|
44
44
|
switch idsToStore {
|
|
45
45
|
| [] => ()
|
|
46
46
|
| ids =>
|
|
@@ -51,17 +51,33 @@ let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updated
|
|
|
51
51
|
}
|
|
52
52
|
)
|
|
53
53
|
let effectName = effect.name
|
|
54
|
-
let
|
|
54
|
+
let tableName = table.tableName
|
|
55
|
+
let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(tableName) {
|
|
55
56
|
| Some(c) => c
|
|
56
57
|
| None =>
|
|
57
|
-
let c: Persistence.effectCacheRecord = {effectName, count: 0}
|
|
58
|
-
cache->Dict.set(
|
|
58
|
+
let c: Persistence.effectCacheRecord = {effectName, scope, tableName, count: 0}
|
|
59
|
+
cache->Dict.set(tableName, c)
|
|
59
60
|
c
|
|
60
61
|
}
|
|
61
62
|
let shouldInitialize = effectCacheRecord.count === 0
|
|
62
63
|
effectCacheRecord.count = effectCacheRecord.count + items->Array.length - invalidationsCount
|
|
63
|
-
Prometheus.EffectCacheCount.set(
|
|
64
|
-
|
|
64
|
+
Prometheus.EffectCacheCount.set(
|
|
65
|
+
~count=effectCacheRecord.count,
|
|
66
|
+
~effectName,
|
|
67
|
+
~scope=scope->Internal.EffectCache.scopeToString,
|
|
68
|
+
)
|
|
69
|
+
acc
|
|
70
|
+
->Array.push(
|
|
71
|
+
(
|
|
72
|
+
{
|
|
73
|
+
table,
|
|
74
|
+
itemSchema: effect.storageMeta.itemSchema,
|
|
75
|
+
items,
|
|
76
|
+
shouldInitialize,
|
|
77
|
+
}: Persistence.updatedEffectCache
|
|
78
|
+
),
|
|
79
|
+
)
|
|
80
|
+
->ignore
|
|
65
81
|
}
|
|
66
82
|
inMemTable.idsToStore = []
|
|
67
83
|
inMemTable.invalidationsCount = 0
|
|
@@ -113,17 +129,36 @@ let runOneWrite = async (state: IndexerState.t) => {
|
|
|
113
129
|
})
|
|
114
130
|
let updatedEffectsCache = snapshotEffects(state, ~cache)
|
|
115
131
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
~
|
|
123
|
-
~
|
|
124
|
-
~chainMetaData,
|
|
132
|
+
let writtenEntityNames = Utils.Set.make()
|
|
133
|
+
updatedEntities->Array.forEach(({entityConfig}) =>
|
|
134
|
+
writtenEntityNames->Utils.Set.add(entityConfig.name)->ignore
|
|
135
|
+
)
|
|
136
|
+
let pruneTargets = PruneStaleHistory.select(
|
|
137
|
+
state,
|
|
138
|
+
~writtenEntityNames,
|
|
139
|
+
~isRollback=rollback->Option.isSome,
|
|
125
140
|
)
|
|
126
141
|
|
|
142
|
+
// The prune runs concurrently with the batch write, but only for entities
|
|
143
|
+
// absent from it, so they never touch the same history table. Both must be
|
|
144
|
+
// awaited before the next write starts, otherwise a still-running prune
|
|
145
|
+
// could overlap the next batch's history writes and lose an anchor (which
|
|
146
|
+
// breaks a later rollback). Rollback writes touch every history table, so
|
|
147
|
+
// they get no concurrent prune at all.
|
|
148
|
+
let _ = await Promise.all2((
|
|
149
|
+
persistence.storage.writeBatch(
|
|
150
|
+
~batch,
|
|
151
|
+
~rollback,
|
|
152
|
+
~isInReorgThreshold=batch.isInReorgThreshold,
|
|
153
|
+
~config,
|
|
154
|
+
~allEntities=persistence.allEntities,
|
|
155
|
+
~updatedEntities,
|
|
156
|
+
~updatedEffectsCache,
|
|
157
|
+
~chainMetaData,
|
|
158
|
+
),
|
|
159
|
+
PruneStaleHistory.runConcurrent(state, ~targets=pruneTargets),
|
|
160
|
+
))
|
|
161
|
+
|
|
127
162
|
state->IndexerState.markCommitted(~upToCheckpointId)
|
|
128
163
|
|
|
129
164
|
switch rollback {
|
|
@@ -131,6 +166,10 @@ let runOneWrite = async (state: IndexerState.t) => {
|
|
|
131
166
|
await RollbackCommit.fire(~progressBlockNumberByChainId)
|
|
132
167
|
| _ => ()
|
|
133
168
|
}
|
|
169
|
+
|
|
170
|
+
// Entities starved of the concurrent prune (eg written in every batch) are
|
|
171
|
+
// pruned here with no other pg writer running.
|
|
172
|
+
await PruneStaleHistory.runForced(state, ~targets=pruneTargets)
|
|
134
173
|
}
|
|
135
174
|
}
|
|
136
175
|
|
|
@@ -192,8 +231,8 @@ let dropCommitted = (state: IndexerState.t, ~keepLoadedFromDb) => {
|
|
|
192
231
|
->InMemoryTable.Entity.dropCommittedChanges(~committedCheckpointId, ~keepLoadedFromDb)
|
|
193
232
|
)
|
|
194
233
|
state
|
|
195
|
-
->IndexerState.
|
|
196
|
-
->
|
|
234
|
+
->IndexerState.effectState
|
|
235
|
+
->EffectState.forEach(inMemTable =>
|
|
197
236
|
inMemTable->InMemoryStore.dropCommittedEffects(~committedCheckpointId, ~keepLoadedFromDb)
|
|
198
237
|
)
|
|
199
238
|
}
|
package/src/Writing.res.mjs
CHANGED
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Env from "./Env.res.mjs";
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
|
+
import * as Internal from "./Internal.res.mjs";
|
|
5
6
|
import * as Throttler from "./Throttler.res.mjs";
|
|
6
7
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
8
|
+
import * as EffectState from "./EffectState.res.mjs";
|
|
7
9
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
8
10
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
9
11
|
import * as InMemoryStore from "./InMemoryStore.res.mjs";
|
|
@@ -11,6 +13,7 @@ import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
|
11
13
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
12
14
|
import * as RollbackCommit from "./RollbackCommit.res.mjs";
|
|
13
15
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
16
|
+
import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
|
|
14
17
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
15
18
|
|
|
16
19
|
function getChangesCount(state) {
|
|
@@ -20,7 +23,7 @@ function getChangesCount(state) {
|
|
|
20
23
|
IndexerState.allEntities(state).forEach(entityConfig => {
|
|
21
24
|
total.contents = total.contents + InMemoryStore.getInMemTable(state, entityConfig).changesCount;
|
|
22
25
|
});
|
|
23
|
-
|
|
26
|
+
EffectState.forEach(IndexerState.effectState(state), inMemTable => {
|
|
24
27
|
total.contents = total.contents + inMemTable.changesCount;
|
|
25
28
|
});
|
|
26
29
|
IndexerState.processedBatches(state).forEach(batch => {
|
|
@@ -35,10 +38,12 @@ function waitForCommit(state) {
|
|
|
35
38
|
|
|
36
39
|
function snapshotEffects(state, cache) {
|
|
37
40
|
let acc = [];
|
|
38
|
-
|
|
41
|
+
EffectState.forEach(IndexerState.effectState(state), inMemTable => {
|
|
39
42
|
let idsToStore = inMemTable.idsToStore;
|
|
40
43
|
let invalidationsCount = inMemTable.invalidationsCount;
|
|
41
44
|
let dict = inMemTable.dict;
|
|
45
|
+
let table = inMemTable.table;
|
|
46
|
+
let scope = inMemTable.scope;
|
|
42
47
|
let effect = inMemTable.effect;
|
|
43
48
|
if (idsToStore.length !== 0) {
|
|
44
49
|
let items = Stdlib_Array.filterMap(idsToStore, id => {
|
|
@@ -51,23 +56,27 @@ function snapshotEffects(state, cache) {
|
|
|
51
56
|
}
|
|
52
57
|
});
|
|
53
58
|
let effectName = effect.name;
|
|
54
|
-
let
|
|
59
|
+
let tableName = table.tableName;
|
|
60
|
+
let c = cache[tableName];
|
|
55
61
|
let effectCacheRecord;
|
|
56
62
|
if (c !== undefined) {
|
|
57
63
|
effectCacheRecord = c;
|
|
58
64
|
} else {
|
|
59
65
|
let c$1 = {
|
|
60
66
|
effectName: effectName,
|
|
67
|
+
scope: scope,
|
|
68
|
+
tableName: tableName,
|
|
61
69
|
count: 0
|
|
62
70
|
};
|
|
63
|
-
cache[
|
|
71
|
+
cache[tableName] = c$1;
|
|
64
72
|
effectCacheRecord = c$1;
|
|
65
73
|
}
|
|
66
74
|
let shouldInitialize = effectCacheRecord.count === 0;
|
|
67
75
|
effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
|
|
68
|
-
Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
|
|
76
|
+
Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName, Internal.EffectCache.scopeToString(scope));
|
|
69
77
|
acc.push({
|
|
70
|
-
|
|
78
|
+
table: table,
|
|
79
|
+
itemSchema: effect.storageMeta.itemSchema,
|
|
71
80
|
items: items,
|
|
72
81
|
shouldInitialize: shouldInitialize
|
|
73
82
|
});
|
|
@@ -111,11 +120,20 @@ async function runOneWrite(state) {
|
|
|
111
120
|
}
|
|
112
121
|
});
|
|
113
122
|
let updatedEffectsCache = snapshotEffects(state, cache);
|
|
114
|
-
|
|
123
|
+
let writtenEntityNames = new Set();
|
|
124
|
+
updatedEntities.forEach(param => {
|
|
125
|
+
writtenEntityNames.add(param.entityConfig.name);
|
|
126
|
+
});
|
|
127
|
+
let pruneTargets = PruneStaleHistory.select(state, writtenEntityNames, Stdlib_Option.isSome(rollback));
|
|
128
|
+
await Promise.all([
|
|
129
|
+
persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData),
|
|
130
|
+
PruneStaleHistory.runConcurrent(state, pruneTargets)
|
|
131
|
+
]);
|
|
115
132
|
IndexerState.markCommitted(state, upToCheckpointId);
|
|
116
133
|
if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
|
|
117
|
-
|
|
134
|
+
await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
|
|
118
135
|
}
|
|
136
|
+
return await PruneStaleHistory.runForced(state, pruneTargets);
|
|
119
137
|
}
|
|
120
138
|
|
|
121
139
|
function hasPendingWrite(state) {
|
|
@@ -164,7 +182,7 @@ function commitBatch(state, batch) {
|
|
|
164
182
|
function dropCommitted(state, keepLoadedFromDb) {
|
|
165
183
|
let committedCheckpointId = IndexerState.committedCheckpointId(state);
|
|
166
184
|
IndexerState.allEntities(state).forEach(entityConfig => InMemoryTable.Entity.dropCommittedChanges(InMemoryStore.getInMemTable(state, entityConfig), committedCheckpointId, keepLoadedFromDb));
|
|
167
|
-
|
|
185
|
+
EffectState.forEach(IndexerState.effectState(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
|
|
168
186
|
}
|
|
169
187
|
|
|
170
188
|
async function awaitCapacity(state) {
|
package/src/bindings/NodeJs.res
CHANGED
|
@@ -173,5 +173,10 @@ module Fs = {
|
|
|
173
173
|
|
|
174
174
|
@module("fs") @scope("promises")
|
|
175
175
|
external readdir: Path.t => promise<array<string>> = "readdir"
|
|
176
|
+
|
|
177
|
+
type stats
|
|
178
|
+
@module("fs") @scope("promises")
|
|
179
|
+
external stat: Path.t => promise<stats> = "stat"
|
|
180
|
+
@send external statsIsDirectory: stats => bool = "isDirectory"
|
|
176
181
|
}
|
|
177
182
|
}
|
package/src/bindings/Viem.res
CHANGED
|
@@ -1,11 +1,6 @@
|
|
|
1
1
|
type hex = EvmTypes.Hex.t
|
|
2
2
|
@module("viem") external toHex: 'a => hex = "toHex"
|
|
3
|
-
@module("viem") external keccak256: hex => hex = "keccak256"
|
|
4
3
|
@module("viem") external pad: hex => hex = "pad"
|
|
5
4
|
|
|
6
5
|
type sizeOptions = {size: int}
|
|
7
6
|
@module("viem") external bigintToHex: (bigint, ~options: sizeOptions=?) => hex = "numberToHex"
|
|
8
|
-
@module("viem") external stringToHex: (string, ~options: sizeOptions=?) => hex = "stringToHex"
|
|
9
|
-
@module("viem") external boolToHex: (bool, ~options: sizeOptions=?) => hex = "boolToHex"
|
|
10
|
-
@module("viem") external bytesToHex: (Uint8Array.t, ~options: sizeOptions=?) => hex = "bytesToHex"
|
|
11
|
-
@module("viem") external concat: array<hex> => hex = "concat"
|
|
@@ -92,27 +92,6 @@ let get = (router: t<'a>, ~tag, ~contractAddress, ~contractNameByAddress) => {
|
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
let getEvmEventId = (~sighash, ~topicCount) => {
|
|
96
|
-
sighash ++ "_" ++ topicCount->Int.toString
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
let fromEvmEventModsOrThrow = (events: array<Internal.evmOnEventRegistration>, ~chain): t<
|
|
100
|
-
Internal.evmOnEventRegistration,
|
|
101
|
-
> => {
|
|
102
|
-
let router = empty()
|
|
103
|
-
events->Array.forEach(config => {
|
|
104
|
-
router->addOrThrow(
|
|
105
|
-
config.eventConfig.id,
|
|
106
|
-
config,
|
|
107
|
-
~contractName=config.eventConfig.contractName,
|
|
108
|
-
~eventName=config.eventConfig.name,
|
|
109
|
-
~chain,
|
|
110
|
-
~isWildcard=config.isWildcard,
|
|
111
|
-
)
|
|
112
|
-
})
|
|
113
|
-
router
|
|
114
|
-
}
|
|
115
|
-
|
|
116
95
|
/** Dispatch key for SVM instructions. `None` matches any instruction in the
|
|
117
96
|
program (lowest priority). */
|
|
118
97
|
let getSvmEventId = (~programId: SvmTypes.Pubkey.t, ~discriminator: option<string>) =>
|
|
@@ -97,16 +97,6 @@ function get$1(router, tag, contractAddress, contractNameByAddress) {
|
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
function getEvmEventId(sighash, topicCount) {
|
|
101
|
-
return sighash + "_" + topicCount.toString();
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
function fromEvmEventModsOrThrow(events, chain) {
|
|
105
|
-
let router = {};
|
|
106
|
-
events.forEach(config => addOrThrow$1(router, config.eventConfig.id, config, config.eventConfig.contractName, config.isWildcard, config.eventConfig.name, chain));
|
|
107
|
-
return router;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
100
|
function getSvmEventId(programId, discriminator) {
|
|
111
101
|
if (discriminator !== undefined) {
|
|
112
102
|
return programId + "_" + discriminator;
|
|
@@ -157,8 +147,6 @@ export {
|
|
|
157
147
|
empty$1 as empty,
|
|
158
148
|
addOrThrow$1 as addOrThrow,
|
|
159
149
|
get$1 as get,
|
|
160
|
-
getEvmEventId,
|
|
161
|
-
fromEvmEventModsOrThrow,
|
|
162
150
|
getSvmEventId,
|
|
163
151
|
fromSvmEventConfigsOrThrow,
|
|
164
152
|
}
|
package/src/sources/Evm.res
CHANGED
|
@@ -90,8 +90,11 @@ let make = (~logger: Pino.t): Ecosystem.t => {
|
|
|
90
90
|
// range chunk is validated by `eventBlockRangeSchema` in
|
|
91
91
|
// `LogSelection.res` which rejects `_lte`/`_every` (use `onBlock` for
|
|
92
92
|
// stride- and endBlock-based block handlers).
|
|
93
|
+
// `S.strict` on the inner object rejects unknown `block` fields (e.g. a
|
|
94
|
+
// `numbre` typo or `block: {timestamp: ...}`) instead of silently
|
|
95
|
+
// ignoring them.
|
|
93
96
|
onEventBlockFilterSchema: S.object(s =>
|
|
94
|
-
s.field("block", S.option(S.object(s2 => s2.field("number", S.unknown))))
|
|
97
|
+
s.field("block", S.option(S.object(s2 => s2.field("number", S.unknown))->S.strict))
|
|
95
98
|
),
|
|
96
99
|
logger,
|
|
97
100
|
// The payload carries `transaction` by batch prep (HyperSync) or inline
|
package/src/sources/Evm.res.mjs
CHANGED
|
@@ -58,7 +58,7 @@ function make(logger) {
|
|
|
58
58
|
cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
|
|
59
59
|
onBlockMethodName: "onBlock",
|
|
60
60
|
onBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown))))),
|
|
61
|
-
onEventBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown))))),
|
|
61
|
+
onEventBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.strict(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown)))))),
|
|
62
62
|
logger: logger,
|
|
63
63
|
toEvent: eventItem => eventItem.payload,
|
|
64
64
|
toEventLogger: eventItem => Logging.createChildFrom(logger, {
|
package/src/sources/EvmChain.res
CHANGED
|
@@ -39,25 +39,6 @@ let getSyncConfig = (
|
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
let collectEventParams = (onEventRegistrations: array<Internal.evmOnEventRegistration>): array<
|
|
43
|
-
HyperSyncClient.Decoder.eventParamsInput,
|
|
44
|
-
> => {
|
|
45
|
-
let result = []
|
|
46
|
-
onEventRegistrations->Array.forEach(reg => {
|
|
47
|
-
let event = reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.evmEventConfig)
|
|
48
|
-
result
|
|
49
|
-
->Array.push({
|
|
50
|
-
HyperSyncClient.Decoder.sighash: event.sighash,
|
|
51
|
-
topicCount: event.topicCount,
|
|
52
|
-
eventName: event.name,
|
|
53
|
-
contractName: event.contractName,
|
|
54
|
-
params: event.paramsMetadata,
|
|
55
|
-
})
|
|
56
|
-
->ignore
|
|
57
|
-
})
|
|
58
|
-
result
|
|
59
|
-
}
|
|
60
|
-
|
|
61
42
|
let makeSources = (
|
|
62
43
|
~chain,
|
|
63
44
|
~onEventRegistrations: array<Internal.evmOnEventRegistration>,
|
|
@@ -65,17 +46,12 @@ let makeSources = (
|
|
|
65
46
|
~rpcs: array<rpc>,
|
|
66
47
|
~lowercaseAddresses,
|
|
67
48
|
) => {
|
|
68
|
-
let eventRouter = onEventRegistrations->EventRouter.fromEvmEventModsOrThrow(~chain)
|
|
69
|
-
|
|
70
|
-
let allEventParams = collectEventParams(onEventRegistrations)
|
|
71
|
-
|
|
72
49
|
let sources = switch hyperSync {
|
|
73
50
|
| Some(endpointUrl) => [
|
|
74
51
|
HyperSyncSource.make({
|
|
75
52
|
chain,
|
|
76
53
|
endpointUrl,
|
|
77
|
-
|
|
78
|
-
eventRouter,
|
|
54
|
+
onEventRegistrations,
|
|
79
55
|
apiToken: Env.envioApiToken,
|
|
80
56
|
clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
|
|
81
57
|
lowercaseAddresses,
|
|
@@ -92,8 +68,7 @@ let makeSources = (
|
|
|
92
68
|
sourceFor,
|
|
93
69
|
syncConfig: getSyncConfig(syncConfig->Option.getOr({})),
|
|
94
70
|
url,
|
|
95
|
-
|
|
96
|
-
allEventParams,
|
|
71
|
+
onEventRegistrations,
|
|
97
72
|
lowercaseAddresses,
|
|
98
73
|
?ws,
|
|
99
74
|
?headers,
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Env from "../Env.res.mjs";
|
|
4
4
|
import * as RpcSource from "./RpcSource.res.mjs";
|
|
5
|
-
import * as EventRouter from "./EventRouter.res.mjs";
|
|
6
5
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
7
6
|
import * as HyperSyncSource from "./HyperSyncSource.res.mjs";
|
|
8
7
|
|
|
@@ -20,29 +19,11 @@ function getSyncConfig(param) {
|
|
|
20
19
|
};
|
|
21
20
|
}
|
|
22
21
|
|
|
23
|
-
function collectEventParams(onEventRegistrations) {
|
|
24
|
-
let result = [];
|
|
25
|
-
onEventRegistrations.forEach(reg => {
|
|
26
|
-
let event = reg.eventConfig;
|
|
27
|
-
result.push({
|
|
28
|
-
sighash: event.sighash,
|
|
29
|
-
topicCount: event.topicCount,
|
|
30
|
-
eventName: event.name,
|
|
31
|
-
contractName: event.contractName,
|
|
32
|
-
params: event.paramsMetadata
|
|
33
|
-
});
|
|
34
|
-
});
|
|
35
|
-
return result;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
22
|
function makeSources(chain, onEventRegistrations, hyperSync, rpcs, lowercaseAddresses) {
|
|
39
|
-
let eventRouter = EventRouter.fromEvmEventModsOrThrow(onEventRegistrations, chain);
|
|
40
|
-
let allEventParams = collectEventParams(onEventRegistrations);
|
|
41
23
|
let sources = hyperSync !== undefined ? [HyperSyncSource.make({
|
|
42
24
|
chain: chain,
|
|
43
25
|
endpointUrl: hyperSync,
|
|
44
|
-
|
|
45
|
-
eventRouter: eventRouter,
|
|
26
|
+
onEventRegistrations: onEventRegistrations,
|
|
46
27
|
apiToken: Env.envioApiToken,
|
|
47
28
|
clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
|
|
48
29
|
lowercaseAddresses: lowercaseAddresses,
|
|
@@ -56,8 +37,7 @@ function makeSources(chain, onEventRegistrations, hyperSync, rpcs, lowercaseAddr
|
|
|
56
37
|
syncConfig: getSyncConfig(Stdlib_Option.getOr(param.syncConfig, {})),
|
|
57
38
|
url: param.url,
|
|
58
39
|
chain: chain,
|
|
59
|
-
|
|
60
|
-
allEventParams: allEventParams,
|
|
40
|
+
onEventRegistrations: onEventRegistrations,
|
|
61
41
|
lowercaseAddresses: lowercaseAddresses,
|
|
62
42
|
ws: param.ws,
|
|
63
43
|
headers: param.headers
|
|
@@ -69,7 +49,6 @@ function makeSources(chain, onEventRegistrations, hyperSync, rpcs, lowercaseAddr
|
|
|
69
49
|
|
|
70
50
|
export {
|
|
71
51
|
getSyncConfig,
|
|
72
|
-
collectEventParams,
|
|
73
52
|
makeSources,
|
|
74
53
|
}
|
|
75
54
|
/* Env Not a pure module */
|