envio 3.3.0-alpha.9 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/licenses/CLA.md +35 -0
- package/licenses/EULA.md +67 -0
- package/licenses/LICENSE.md +45 -0
- package/licenses/README.md +35 -0
- package/package.json +9 -8
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +0 -7
- package/src/BatchProcessing.res.mjs +1 -8
- package/src/ChainFetching.res +30 -18
- package/src/ChainFetching.res.mjs +23 -13
- package/src/ChainState.res +134 -55
- package/src/ChainState.res.mjs +78 -36
- package/src/ChainState.resi +14 -2
- package/src/Config.res +9 -5
- package/src/Config.res.mjs +2 -2
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +12 -0
- package/src/CrossChainState.res +115 -34
- package/src/CrossChainState.res.mjs +37 -14
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/EventConfigBuilder.res +21 -46
- package/src/EventConfigBuilder.res.mjs +18 -25
- package/src/EventProcessing.res +8 -5
- package/src/EventProcessing.res.mjs +2 -1
- package/src/FetchState.res +814 -467
- package/src/FetchState.res.mjs +541 -369
- package/src/HandlerRegister.res +3 -1
- package/src/HandlerRegister.res.mjs +3 -2
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +15 -39
- package/src/IndexerState.res.mjs +18 -29
- package/src/IndexerState.resi +2 -10
- package/src/Internal.res +98 -14
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/LogSelection.res +0 -62
- package/src/LogSelection.res.mjs +0 -74
- package/src/Metrics.res +1 -1
- package/src/Metrics.res.mjs +1 -1
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/PruneStaleHistory.res +146 -35
- package/src/PruneStaleHistory.res.mjs +114 -20
- package/src/RawEvent.res +2 -2
- package/src/Rollback.res +32 -13
- package/src/Rollback.res.mjs +24 -11
- package/src/SimulateDeadInputTracker.res +1 -1
- package/src/SimulateItems.res +38 -6
- package/src/SimulateItems.res.mjs +28 -9
- package/src/TestIndexer.res +2 -2
- package/src/TestIndexer.res.mjs +2 -2
- package/src/TopicFilter.res +1 -25
- package/src/TopicFilter.res.mjs +0 -74
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +60 -21
- package/src/Writing.res.mjs +27 -9
- package/src/bindings/NodeJs.res +5 -0
- package/src/bindings/Viem.res +0 -5
- package/src/sources/EventRouter.res +0 -21
- package/src/sources/EventRouter.res.mjs +0 -12
- package/src/sources/EvmChain.res +2 -27
- package/src/sources/EvmChain.res.mjs +2 -23
- package/src/sources/EvmRpcClient.res +58 -23
- package/src/sources/EvmRpcClient.res.mjs +11 -5
- package/src/sources/HyperSync.res +9 -38
- package/src/sources/HyperSync.res.mjs +16 -28
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +88 -11
- package/src/sources/HyperSyncClient.res.mjs +39 -6
- package/src/sources/HyperSyncSource.res +18 -199
- package/src/sources/HyperSyncSource.res.mjs +9 -128
- package/src/sources/Rpc.res +0 -32
- package/src/sources/Rpc.res.mjs +1 -46
- package/src/sources/RpcSource.res +129 -522
- package/src/sources/RpcSource.res.mjs +161 -366
- package/src/sources/SimulateSource.res +37 -19
- package/src/sources/SimulateSource.res.mjs +27 -10
- package/src/sources/SourceManager.res +3 -7
- package/src/sources/SourceManager.res.mjs +2 -7
- package/src/sources/SourceManager.resi +0 -2
- package/src/sources/SvmHyperSyncSource.res +13 -3
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
package/src/Internal.res.mjs
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
import * as Table from "./db/Table.res.mjs";
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Address from "./Address.res.mjs";
|
|
6
|
+
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
7
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
6
8
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
7
9
|
|
|
8
10
|
let allEvmBlockFields = [
|
|
@@ -166,10 +168,101 @@ let fuelTransferParamsSchema = S$RescriptSchema.schema(s => ({
|
|
|
166
168
|
|
|
167
169
|
let cacheTablePrefix = "envio_effect_";
|
|
168
170
|
|
|
171
|
+
function toTableName(effectName, scope) {
|
|
172
|
+
if (scope === "crossChain") {
|
|
173
|
+
return cacheTablePrefix + effectName;
|
|
174
|
+
} else {
|
|
175
|
+
return `envio_` + scope.toString() + `_effect_` + effectName;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function scopeToString(scope) {
|
|
180
|
+
if (scope === "crossChain") {
|
|
181
|
+
return "crossChain";
|
|
182
|
+
} else {
|
|
183
|
+
return scope.toString();
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function parseChainId(str) {
|
|
188
|
+
let chainId = Stdlib_Int.fromString(str, undefined);
|
|
189
|
+
if (chainId !== undefined && chainId >= 0 && chainId.toString() === str) {
|
|
190
|
+
return chainId;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let chainScopedRe = /^envio_([0-9]+)_effect_(.+)$/;
|
|
195
|
+
|
|
196
|
+
let crossChainRe = /^envio_effect_(.+)$/;
|
|
197
|
+
|
|
198
|
+
function fromTableName(tableName) {
|
|
199
|
+
let result = chainScopedRe.exec(tableName);
|
|
200
|
+
if (result == null) {
|
|
201
|
+
let result$1 = crossChainRe.exec(tableName);
|
|
202
|
+
if (result$1 == null) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
let match = result$1.slice(1)[0];
|
|
206
|
+
if (match === undefined) {
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
let effectName = Primitive_option.valFromOption(match);
|
|
210
|
+
if (effectName !== undefined) {
|
|
211
|
+
return [
|
|
212
|
+
effectName,
|
|
213
|
+
"crossChain"
|
|
214
|
+
];
|
|
215
|
+
} else {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
let match$1 = result.slice(1)[0];
|
|
220
|
+
let match$2 = result.slice(1)[1];
|
|
221
|
+
if (match$1 === undefined) {
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
let chainIdStr = Primitive_option.valFromOption(match$1);
|
|
225
|
+
if (chainIdStr === undefined) {
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
if (match$2 === undefined) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
let effectName$1 = Primitive_option.valFromOption(match$2);
|
|
232
|
+
if (effectName$1 === undefined) {
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
let chainId = parseChainId(chainIdStr);
|
|
236
|
+
if (chainId !== undefined) {
|
|
237
|
+
return [
|
|
238
|
+
effectName$1,
|
|
239
|
+
chainId
|
|
240
|
+
];
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function toCachePath(effectName, scope) {
|
|
245
|
+
if (scope === "crossChain") {
|
|
246
|
+
return effectName + ".tsv";
|
|
247
|
+
} else {
|
|
248
|
+
return scope.toString() + `/` + effectName + `.tsv`;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
let EffectCache = {
|
|
253
|
+
toTableName: toTableName,
|
|
254
|
+
scopeToString: scopeToString,
|
|
255
|
+
parseChainId: parseChainId,
|
|
256
|
+
chainScopedRe: chainScopedRe,
|
|
257
|
+
crossChainRe: crossChainRe,
|
|
258
|
+
fromTableName: fromTableName,
|
|
259
|
+
toCachePath: toCachePath
|
|
260
|
+
};
|
|
261
|
+
|
|
169
262
|
let cacheOutputSchema = S$RescriptSchema.json(false);
|
|
170
263
|
|
|
171
|
-
function makeCacheTable(effectName) {
|
|
172
|
-
return Table.mkTable(
|
|
264
|
+
function makeCacheTable(effectName, scope) {
|
|
265
|
+
return Table.mkTable(toTableName(effectName, scope), undefined, [
|
|
173
266
|
Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
|
|
174
267
|
Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined)
|
|
175
268
|
], undefined);
|
|
@@ -195,6 +288,7 @@ export {
|
|
|
195
288
|
fuelSupplyParamsSchema,
|
|
196
289
|
fuelTransferParamsSchema,
|
|
197
290
|
cacheTablePrefix,
|
|
291
|
+
EffectCache,
|
|
198
292
|
cacheOutputSchema,
|
|
199
293
|
makeCacheTable,
|
|
200
294
|
loadedFromDbCheckpointId,
|
package/src/LoadLayer.res
CHANGED
|
@@ -72,28 +72,29 @@ let loadById = (
|
|
|
72
72
|
let callEffect = (
|
|
73
73
|
~effect: Internal.effect,
|
|
74
74
|
~arg: Internal.effectArgs,
|
|
75
|
-
~inMemTable:
|
|
75
|
+
~inMemTable: EffectState.effectCacheInMemTable,
|
|
76
76
|
~timerRef,
|
|
77
77
|
~onError,
|
|
78
78
|
) => {
|
|
79
79
|
let effectName = effect.name
|
|
80
|
-
let
|
|
81
|
-
|
|
80
|
+
let scopeLabel = inMemTable.scope->Internal.EffectCache.scopeToString
|
|
81
|
+
let hadActiveCalls = inMemTable.activeCallsCount > 0
|
|
82
|
+
inMemTable.activeCallsCount = inMemTable.activeCallsCount + 1
|
|
82
83
|
Prometheus.EffectCalls.activeCallsCount->Prometheus.SafeGauge.handleInt(
|
|
83
|
-
~labels=effectName,
|
|
84
|
-
~value=
|
|
84
|
+
~labels={"effect": effectName, "scope": scopeLabel},
|
|
85
|
+
~value=inMemTable.activeCallsCount,
|
|
85
86
|
)
|
|
86
87
|
|
|
87
88
|
if hadActiveCalls {
|
|
88
|
-
let elapsed = Performance.secondsBetween(~from=
|
|
89
|
+
let elapsed = Performance.secondsBetween(~from=inMemTable.prevCallStartTimerRef, ~to=timerRef)
|
|
89
90
|
if elapsed > 0. {
|
|
90
91
|
Prometheus.EffectCalls.timeCounter->Prometheus.SafeCounter.handleFloat(
|
|
91
|
-
~labels=effectName,
|
|
92
|
+
~labels={"effect": effectName, "scope": scopeLabel},
|
|
92
93
|
~value=elapsed,
|
|
93
94
|
)
|
|
94
95
|
}
|
|
95
96
|
}
|
|
96
|
-
|
|
97
|
+
inMemTable.prevCallStartTimerRef = timerRef
|
|
97
98
|
|
|
98
99
|
effect.handler(arg)
|
|
99
100
|
->Promise.thenResolve(output => {
|
|
@@ -108,21 +109,23 @@ let callEffect = (
|
|
|
108
109
|
onError(~inputKey=arg.cacheKey, ~exn)
|
|
109
110
|
})
|
|
110
111
|
->Promise.finally(() => {
|
|
111
|
-
|
|
112
|
+
inMemTable.activeCallsCount = inMemTable.activeCallsCount - 1
|
|
112
113
|
Prometheus.EffectCalls.activeCallsCount->Prometheus.SafeGauge.handleInt(
|
|
113
|
-
~labels=effectName,
|
|
114
|
-
~value=
|
|
114
|
+
~labels={"effect": effectName, "scope": scopeLabel},
|
|
115
|
+
~value=inMemTable.activeCallsCount,
|
|
115
116
|
)
|
|
116
117
|
let newTimer = Performance.now()
|
|
117
118
|
Prometheus.EffectCalls.timeCounter->Prometheus.SafeCounter.handleFloat(
|
|
118
|
-
~labels=effectName,
|
|
119
|
-
~value=Performance.secondsBetween(~from=
|
|
119
|
+
~labels={"effect": effectName, "scope": scopeLabel},
|
|
120
|
+
~value=Performance.secondsBetween(~from=inMemTable.prevCallStartTimerRef, ~to=newTimer),
|
|
120
121
|
)
|
|
121
|
-
|
|
122
|
+
inMemTable.prevCallStartTimerRef = newTimer
|
|
122
123
|
|
|
123
|
-
Prometheus.EffectCalls.totalCallsCount->Prometheus.SafeCounter.increment(
|
|
124
|
+
Prometheus.EffectCalls.totalCallsCount->Prometheus.SafeCounter.increment(
|
|
125
|
+
~labels={"effect": effectName, "scope": scopeLabel},
|
|
126
|
+
)
|
|
124
127
|
Prometheus.EffectCalls.sumTimeCounter->Prometheus.SafeCounter.handleFloat(
|
|
125
|
-
~labels=effectName,
|
|
128
|
+
~labels={"effect": effectName, "scope": scopeLabel},
|
|
126
129
|
~value=timerRef->Performance.secondsSince,
|
|
127
130
|
)
|
|
128
131
|
})
|
|
@@ -131,7 +134,7 @@ let callEffect = (
|
|
|
131
134
|
let rec executeWithRateLimit = (
|
|
132
135
|
~effect: Internal.effect,
|
|
133
136
|
~effectArgs: array<Internal.effectArgs>,
|
|
134
|
-
~inMemTable,
|
|
137
|
+
~inMemTable: EffectState.effectCacheInMemTable,
|
|
135
138
|
~onError,
|
|
136
139
|
~isFromQueue: bool,
|
|
137
140
|
) => {
|
|
@@ -140,7 +143,7 @@ let rec executeWithRateLimit = (
|
|
|
140
143
|
let timerRef = Performance.now()
|
|
141
144
|
let promises = []
|
|
142
145
|
|
|
143
|
-
switch
|
|
146
|
+
switch inMemTable.rateLimitState {
|
|
144
147
|
| None =>
|
|
145
148
|
// No rate limiting - execute all immediately
|
|
146
149
|
for idx in 0 to effectArgs->Array.length - 1 {
|
|
@@ -193,7 +196,11 @@ let rec executeWithRateLimit = (
|
|
|
193
196
|
if immediateCount > 0 && isFromQueue {
|
|
194
197
|
// Update queue count metric
|
|
195
198
|
state.queueCount = state.queueCount - immediateCount
|
|
196
|
-
Prometheus.EffectQueueCount.set(
|
|
199
|
+
Prometheus.EffectQueueCount.set(
|
|
200
|
+
~count=state.queueCount,
|
|
201
|
+
~effectName,
|
|
202
|
+
~scope=inMemTable.scope->Internal.EffectCache.scopeToString,
|
|
203
|
+
)
|
|
197
204
|
}
|
|
198
205
|
|
|
199
206
|
// Handle queued items
|
|
@@ -201,7 +208,11 @@ let rec executeWithRateLimit = (
|
|
|
201
208
|
if !isFromQueue {
|
|
202
209
|
// Update queue count metric
|
|
203
210
|
state.queueCount = state.queueCount + queuedArgs->Array.length
|
|
204
|
-
Prometheus.EffectQueueCount.set(
|
|
211
|
+
Prometheus.EffectQueueCount.set(
|
|
212
|
+
~count=state.queueCount,
|
|
213
|
+
~effectName,
|
|
214
|
+
~scope=inMemTable.scope->Internal.EffectCache.scopeToString,
|
|
215
|
+
)
|
|
205
216
|
}
|
|
206
217
|
|
|
207
218
|
let millisUntilReset = ref(0)
|
|
@@ -249,14 +260,23 @@ let loadEffect = (
|
|
|
249
260
|
~persistence: Persistence.t,
|
|
250
261
|
~effect: Internal.effect,
|
|
251
262
|
~effectArgs,
|
|
263
|
+
~scope: Internal.chainScope,
|
|
252
264
|
~indexerState,
|
|
253
265
|
~shouldGroup,
|
|
254
266
|
~item,
|
|
255
267
|
~ecosystem,
|
|
256
268
|
) => {
|
|
257
269
|
let effectName = effect.name
|
|
258
|
-
let
|
|
259
|
-
let
|
|
270
|
+
let inMemTable = indexerState->InMemoryStore.getEffectInMemTable(~effect, ~scope)
|
|
271
|
+
let table = inMemTable.table
|
|
272
|
+
let tableName = table.tableName
|
|
273
|
+
// The operation key must differ per scope so chain-scoped calls with the same
|
|
274
|
+
// input never dedup across chains. Cross-chain keeps the bare effect name so
|
|
275
|
+
// the storage-load metric stays stable.
|
|
276
|
+
let key = switch scope {
|
|
277
|
+
| CrossChain => `${effectName}.effect`
|
|
278
|
+
| Chain(chainId) => `${effectName}.effect.${chainId->Int.toString}`
|
|
279
|
+
}
|
|
260
280
|
|
|
261
281
|
let load = async (args, ~onError) => {
|
|
262
282
|
let idsToLoad = args->Array.map((arg: Internal.effectArgs) => arg.cacheKey)
|
|
@@ -264,13 +284,13 @@ let loadEffect = (
|
|
|
264
284
|
|
|
265
285
|
if (
|
|
266
286
|
switch persistence.storageStatus {
|
|
267
|
-
| Ready({cache}) => cache->Dict.has(
|
|
287
|
+
| Ready({cache}) => cache->Dict.has(tableName)
|
|
268
288
|
| _ => false
|
|
269
289
|
}
|
|
270
290
|
) {
|
|
271
291
|
let storage = persistence->Persistence.getInitializedStorageOrThrow
|
|
272
292
|
let timerRef = Prometheus.StorageLoad.startOperation(~storage=storage.name, ~operation=key)
|
|
273
|
-
let {
|
|
293
|
+
let {outputSchema} = effect.storageMeta
|
|
274
294
|
|
|
275
295
|
let dbEntities = try {
|
|
276
296
|
(
|
package/src/LoadLayer.res.mjs
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import * as Table from "./db/Table.res.mjs";
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Logging from "./Logging.res.mjs";
|
|
6
|
+
import * as Internal from "./Internal.res.mjs";
|
|
6
7
|
import * as Ecosystem from "./Ecosystem.res.mjs";
|
|
7
8
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
8
9
|
import * as LoadManager from "./LoadManager.res.mjs";
|
|
@@ -51,24 +52,43 @@ function loadById(loadManager, persistence, entityConfig, indexerState, shouldGr
|
|
|
51
52
|
|
|
52
53
|
function callEffect(effect, arg, inMemTable, timerRef, onError) {
|
|
53
54
|
let effectName = effect.name;
|
|
54
|
-
let
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
let scopeLabel = Internal.EffectCache.scopeToString(inMemTable.scope);
|
|
56
|
+
let hadActiveCalls = inMemTable.activeCallsCount > 0;
|
|
57
|
+
inMemTable.activeCallsCount = inMemTable.activeCallsCount + 1 | 0;
|
|
58
|
+
Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, {
|
|
59
|
+
effect: effectName,
|
|
60
|
+
scope: scopeLabel
|
|
61
|
+
}, inMemTable.activeCallsCount);
|
|
57
62
|
if (hadActiveCalls) {
|
|
58
|
-
let elapsed = Performance.secondsBetween(
|
|
63
|
+
let elapsed = Performance.secondsBetween(inMemTable.prevCallStartTimerRef, timerRef);
|
|
59
64
|
if (elapsed > 0) {
|
|
60
|
-
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter,
|
|
65
|
+
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, {
|
|
66
|
+
effect: effectName,
|
|
67
|
+
scope: scopeLabel
|
|
68
|
+
}, elapsed);
|
|
61
69
|
}
|
|
62
70
|
}
|
|
63
|
-
|
|
71
|
+
inMemTable.prevCallStartTimerRef = timerRef;
|
|
64
72
|
return effect.handler(arg).then(output => InMemoryStore.setEffectOutput(inMemTable, arg.checkpointId, arg.cacheKey, output, arg.context.cache)).catch(exn => onError(arg.cacheKey, exn)).finally(() => {
|
|
65
|
-
|
|
66
|
-
Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount,
|
|
73
|
+
inMemTable.activeCallsCount = inMemTable.activeCallsCount - 1 | 0;
|
|
74
|
+
Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, {
|
|
75
|
+
effect: effectName,
|
|
76
|
+
scope: scopeLabel
|
|
77
|
+
}, inMemTable.activeCallsCount);
|
|
67
78
|
let newTimer = Performance.now();
|
|
68
|
-
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter,
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
79
|
+
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, {
|
|
80
|
+
effect: effectName,
|
|
81
|
+
scope: scopeLabel
|
|
82
|
+
}, Performance.secondsBetween(inMemTable.prevCallStartTimerRef, newTimer));
|
|
83
|
+
inMemTable.prevCallStartTimerRef = newTimer;
|
|
84
|
+
Prometheus.SafeCounter.increment(Prometheus.EffectCalls.totalCallsCount, {
|
|
85
|
+
effect: effectName,
|
|
86
|
+
scope: scopeLabel
|
|
87
|
+
});
|
|
88
|
+
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.sumTimeCounter, {
|
|
89
|
+
effect: effectName,
|
|
90
|
+
scope: scopeLabel
|
|
91
|
+
}, Performance.secondsSince(timerRef));
|
|
72
92
|
});
|
|
73
93
|
}
|
|
74
94
|
|
|
@@ -76,7 +96,7 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
|
|
|
76
96
|
let effectName = effect.name;
|
|
77
97
|
let timerRef = Performance.now();
|
|
78
98
|
let promises = [];
|
|
79
|
-
let state =
|
|
99
|
+
let state = inMemTable.rateLimitState;
|
|
80
100
|
if (state !== undefined) {
|
|
81
101
|
let now = Date.now();
|
|
82
102
|
if (now >= state.windowStartTime + state.durationMs) {
|
|
@@ -93,12 +113,12 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
|
|
|
93
113
|
}
|
|
94
114
|
if (immediateCount > 0 && isFromQueue) {
|
|
95
115
|
state.queueCount = state.queueCount - immediateCount | 0;
|
|
96
|
-
Prometheus.EffectQueueCount.set(state.queueCount, effectName);
|
|
116
|
+
Prometheus.EffectQueueCount.set(state.queueCount, effectName, Internal.EffectCache.scopeToString(inMemTable.scope));
|
|
97
117
|
}
|
|
98
118
|
if (Utils.$$Array.notEmpty(queuedArgs)) {
|
|
99
119
|
if (!isFromQueue) {
|
|
100
120
|
state.queueCount = state.queueCount + queuedArgs.length | 0;
|
|
101
|
-
Prometheus.EffectQueueCount.set(state.queueCount, effectName);
|
|
121
|
+
Prometheus.EffectQueueCount.set(state.queueCount, effectName, Internal.EffectCache.scopeToString(inMemTable.scope));
|
|
102
122
|
}
|
|
103
123
|
let millisUntilReset = {
|
|
104
124
|
contents: 0
|
|
@@ -128,16 +148,19 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
|
|
|
128
148
|
return Promise.all(promises);
|
|
129
149
|
}
|
|
130
150
|
|
|
131
|
-
function loadEffect(loadManager, persistence, effect, effectArgs, indexerState, shouldGroup, item, ecosystem) {
|
|
151
|
+
function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexerState, shouldGroup, item, ecosystem) {
|
|
132
152
|
let effectName = effect.name;
|
|
133
|
-
let
|
|
134
|
-
let
|
|
153
|
+
let inMemTable = InMemoryStore.getEffectInMemTable(indexerState, effect, scope);
|
|
154
|
+
let table = inMemTable.table;
|
|
155
|
+
let tableName = table.tableName;
|
|
156
|
+
let key;
|
|
157
|
+
key = scope === "crossChain" ? effectName + `.effect` : effectName + `.effect.` + scope.toString();
|
|
135
158
|
let load = async (args, onError) => {
|
|
136
159
|
let idsToLoad = args.map(arg => arg.cacheKey);
|
|
137
160
|
let idsFromCache = new Set();
|
|
138
161
|
let match = persistence.storageStatus;
|
|
139
162
|
let tmp;
|
|
140
|
-
tmp = typeof match !== "object" || match.TAG === "Initializing" ? false :
|
|
163
|
+
tmp = typeof match !== "object" || match.TAG === "Initializing" ? false : tableName in match._0.cache;
|
|
141
164
|
if (tmp) {
|
|
142
165
|
let storage = Persistence.getInitializedStorageOrThrow(persistence);
|
|
143
166
|
let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
|
|
@@ -149,7 +172,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, indexerState,
|
|
|
149
172
|
operator: "in",
|
|
150
173
|
fieldName: Table.idFieldName,
|
|
151
174
|
fieldValue: idsToLoad
|
|
152
|
-
},
|
|
175
|
+
}, table);
|
|
153
176
|
} catch (raw_exn) {
|
|
154
177
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
155
178
|
Logging.childWarn(Ecosystem.getItemLogger(item, ecosystem), {
|
package/src/LoadLayer.resi
CHANGED
package/src/LogSelection.res
CHANGED
|
@@ -1,65 +1,3 @@
|
|
|
1
|
-
exception MissingRequiredTopic0
|
|
2
|
-
let makeTopicSelection = (~topic0, ~topic1=[], ~topic2=[], ~topic3=[]): result<
|
|
3
|
-
Internal.topicSelection,
|
|
4
|
-
exn,
|
|
5
|
-
> =>
|
|
6
|
-
if topic0->Utils.Array.isEmpty {
|
|
7
|
-
Error(MissingRequiredTopic0)
|
|
8
|
-
} else {
|
|
9
|
-
{
|
|
10
|
-
Internal.topic0,
|
|
11
|
-
topic1,
|
|
12
|
-
topic2,
|
|
13
|
-
topic3,
|
|
14
|
-
}->Ok
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
let hasFilters = ({topic1, topic2, topic3}: Internal.topicSelection) => {
|
|
18
|
-
[topic1, topic2, topic3]->Array.find(topic => !Utils.Array.isEmpty(topic))->Option.isSome
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
For a group of topic selections, if multiple only use topic0, then they can be compressed into one
|
|
23
|
-
selection combining the topic0s
|
|
24
|
-
*/
|
|
25
|
-
let compressTopicSelections = (topicSelections: array<Internal.topicSelection>) => {
|
|
26
|
-
let topic0sOfSelectionsWithoutFilters = []
|
|
27
|
-
|
|
28
|
-
let selectionsWithFilters = []
|
|
29
|
-
|
|
30
|
-
topicSelections->Array.forEach(selection => {
|
|
31
|
-
if selection->hasFilters {
|
|
32
|
-
selectionsWithFilters->Array.push(selection)->ignore
|
|
33
|
-
} else {
|
|
34
|
-
selection.topic0->Array.forEach(topic0 => {
|
|
35
|
-
topic0sOfSelectionsWithoutFilters->Array.push(topic0)->ignore
|
|
36
|
-
})
|
|
37
|
-
}
|
|
38
|
-
})
|
|
39
|
-
|
|
40
|
-
switch topic0sOfSelectionsWithoutFilters {
|
|
41
|
-
| [] => selectionsWithFilters
|
|
42
|
-
| topic0 =>
|
|
43
|
-
let selectionWithoutFilters: Internal.topicSelection = {
|
|
44
|
-
topic0,
|
|
45
|
-
topic1: [],
|
|
46
|
-
topic2: [],
|
|
47
|
-
topic3: [],
|
|
48
|
-
}
|
|
49
|
-
Array.concat([selectionWithoutFilters], selectionsWithFilters)
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
type t = {
|
|
54
|
-
addresses: array<Address.t>,
|
|
55
|
-
topicSelections: array<Internal.topicSelection>,
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
let make = (~addresses, ~topicSelections) => {
|
|
59
|
-
let topicSelections = compressTopicSelections(topicSelections)
|
|
60
|
-
{addresses, topicSelections}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
1
|
// Expand a resolved topic selection into concrete topic values for a query:
|
|
64
2
|
// `ContractAddresses` markers become the given partition addresses encoded as
|
|
65
3
|
// topics; `Values` pass through.
|
package/src/LogSelection.res.mjs
CHANGED
|
@@ -8,75 +8,6 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
|
|
|
8
8
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
9
9
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
10
10
|
|
|
11
|
-
let MissingRequiredTopic0 = /* @__PURE__ */Primitive_exceptions.create("LogSelection.MissingRequiredTopic0");
|
|
12
|
-
|
|
13
|
-
function makeTopicSelection(topic0, topic1Opt, topic2Opt, topic3Opt) {
|
|
14
|
-
let topic1 = topic1Opt !== undefined ? topic1Opt : [];
|
|
15
|
-
let topic2 = topic2Opt !== undefined ? topic2Opt : [];
|
|
16
|
-
let topic3 = topic3Opt !== undefined ? topic3Opt : [];
|
|
17
|
-
if (Utils.$$Array.isEmpty(topic0)) {
|
|
18
|
-
return {
|
|
19
|
-
TAG: "Error",
|
|
20
|
-
_0: {
|
|
21
|
-
RE_EXN_ID: MissingRequiredTopic0
|
|
22
|
-
}
|
|
23
|
-
};
|
|
24
|
-
} else {
|
|
25
|
-
return {
|
|
26
|
-
TAG: "Ok",
|
|
27
|
-
_0: {
|
|
28
|
-
topic0: topic0,
|
|
29
|
-
topic1: topic1,
|
|
30
|
-
topic2: topic2,
|
|
31
|
-
topic3: topic3
|
|
32
|
-
}
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
function hasFilters(param) {
|
|
38
|
-
return Stdlib_Option.isSome([
|
|
39
|
-
param.topic1,
|
|
40
|
-
param.topic2,
|
|
41
|
-
param.topic3
|
|
42
|
-
].find(topic => !Utils.$$Array.isEmpty(topic)));
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
function compressTopicSelections(topicSelections) {
|
|
46
|
-
let topic0sOfSelectionsWithoutFilters = [];
|
|
47
|
-
let selectionsWithFilters = [];
|
|
48
|
-
topicSelections.forEach(selection => {
|
|
49
|
-
if (hasFilters(selection)) {
|
|
50
|
-
selectionsWithFilters.push(selection);
|
|
51
|
-
} else {
|
|
52
|
-
selection.topic0.forEach(topic0 => {
|
|
53
|
-
topic0sOfSelectionsWithoutFilters.push(topic0);
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
});
|
|
57
|
-
if (topic0sOfSelectionsWithoutFilters.length === 0) {
|
|
58
|
-
return selectionsWithFilters;
|
|
59
|
-
}
|
|
60
|
-
let selectionWithoutFilters_topic1 = [];
|
|
61
|
-
let selectionWithoutFilters_topic2 = [];
|
|
62
|
-
let selectionWithoutFilters_topic3 = [];
|
|
63
|
-
let selectionWithoutFilters = {
|
|
64
|
-
topic0: topic0sOfSelectionsWithoutFilters,
|
|
65
|
-
topic1: selectionWithoutFilters_topic1,
|
|
66
|
-
topic2: selectionWithoutFilters_topic2,
|
|
67
|
-
topic3: selectionWithoutFilters_topic3
|
|
68
|
-
};
|
|
69
|
-
return [selectionWithoutFilters].concat(selectionsWithFilters);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function make(addresses, topicSelections) {
|
|
73
|
-
let topicSelections$1 = compressTopicSelections(topicSelections);
|
|
74
|
-
return {
|
|
75
|
-
addresses: addresses,
|
|
76
|
-
topicSelections: topicSelections$1
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
|
|
80
11
|
function materializeTopicFilter(filter, addresses) {
|
|
81
12
|
if (filter.TAG === "Values") {
|
|
82
13
|
return filter._0;
|
|
@@ -299,11 +230,6 @@ function parseWhereOrThrow(where, sighash, params, contractName, chainId, onEven
|
|
|
299
230
|
}
|
|
300
231
|
|
|
301
232
|
export {
|
|
302
|
-
MissingRequiredTopic0,
|
|
303
|
-
makeTopicSelection,
|
|
304
|
-
hasFilters,
|
|
305
|
-
compressTopicSelections,
|
|
306
|
-
make,
|
|
307
233
|
materializeTopicFilter,
|
|
308
234
|
materializeTopicSelections,
|
|
309
235
|
eventBlockRangeSchema,
|
package/src/Metrics.res
CHANGED
|
@@ -47,7 +47,7 @@ let renderSourceRequests = (~samples: array<SourceManager.requestStatSample>) =>
|
|
|
47
47
|
if sample.seconds !== 0. {
|
|
48
48
|
secondsOut :=
|
|
49
49
|
secondsOut.contents ++
|
|
50
|
-
`\n${sourceRequestSecondsTotalName}${labels} ${sample.seconds->Float.
|
|
50
|
+
`\n${sourceRequestSecondsTotalName}${labels} ${sample.seconds->Float.toFixed(~digits=3)}`
|
|
51
51
|
}
|
|
52
52
|
})
|
|
53
53
|
countOut.contents ++ secondsOut.contents
|
package/src/Metrics.res.mjs
CHANGED
|
@@ -44,7 +44,7 @@ function renderSourceRequests(samples) {
|
|
|
44
44
|
let labels = `{source="` + sample.sourceName + `",chainId="` + sample.chainId.toString() + `",method="` + sample.method + `"}`;
|
|
45
45
|
countOut.contents = countOut.contents + (`\n` + sourceRequestTotalName + labels + ` ` + sample.count.toString());
|
|
46
46
|
if (sample.seconds !== 0) {
|
|
47
|
-
secondsOut.contents = secondsOut.contents + (`\n` + sourceRequestSecondsTotalName + labels + ` ` + sample.seconds.
|
|
47
|
+
secondsOut.contents = secondsOut.contents + (`\n` + sourceRequestSecondsTotalName + labels + ` ` + sample.seconds.toFixed(3));
|
|
48
48
|
return;
|
|
49
49
|
}
|
|
50
50
|
});
|
package/src/Persistence.res
CHANGED
|
@@ -6,9 +6,14 @@
|
|
|
6
6
|
// DbFunctions, Db, Migrations, InMemoryStore modules which use codegen code directly.
|
|
7
7
|
|
|
8
8
|
// The type reflects an cache table in the db
|
|
9
|
-
// It might be present even if the effect is not used in the application
|
|
9
|
+
// It might be present even if the effect is not used in the application.
|
|
10
|
+
// `initialState.cache` is keyed by `tableName` (the full cache address), so a
|
|
11
|
+
// cross-chain and a chain-scoped cache for the same effect are tracked
|
|
12
|
+
// independently.
|
|
10
13
|
type effectCacheRecord = {
|
|
11
14
|
effectName: string,
|
|
15
|
+
scope: Internal.chainScope,
|
|
16
|
+
tableName: string,
|
|
12
17
|
// Number of rows in the table
|
|
13
18
|
mutable count: int,
|
|
14
19
|
}
|
|
@@ -40,8 +45,12 @@ type initialState = {
|
|
|
40
45
|
envioInfo: option<JSON.t>,
|
|
41
46
|
}
|
|
42
47
|
|
|
48
|
+
// Carries the already-resolved cache address (`table`) rather than an effect +
|
|
49
|
+
// scope: the scope is contextual (resolved per call from the handler's chain),
|
|
50
|
+
// so the write layer only needs the concrete table it targets.
|
|
43
51
|
type updatedEffectCache = {
|
|
44
|
-
|
|
52
|
+
table: Table.table,
|
|
53
|
+
itemSchema: S.t<Internal.effectCacheItem>,
|
|
45
54
|
items: array<Internal.effectCacheItem>,
|
|
46
55
|
shouldInitialize: bool,
|
|
47
56
|
}
|
|
@@ -112,7 +121,7 @@ type storage = {
|
|
|
112
121
|
getRollbackData: (
|
|
113
122
|
~entityConfig: Internal.entityConfig,
|
|
114
123
|
~rollbackTargetCheckpointId: Internal.checkpointId,
|
|
115
|
-
) => promise<(array<
|
|
124
|
+
) => promise<(array<string>, array<unknown>)>,
|
|
116
125
|
// Write batch to storage
|
|
117
126
|
writeBatch: (
|
|
118
127
|
~batch: Batch.t,
|