envio 3.0.0-alpha.21 → 3.0.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +2 -48
- package/evm.schema.json +67 -0
- package/fuel.schema.json +67 -0
- package/index.d.ts +822 -38
- package/index.js +5 -3
- package/package.json +10 -8
- package/rescript.json +5 -9
- package/src/Address.res +4 -5
- package/src/Address.res.mjs +9 -12
- package/src/Api.res +15 -0
- package/src/Api.res.mjs +20 -0
- package/src/Batch.res +32 -34
- package/src/Batch.res.mjs +172 -187
- package/src/Bin.res +89 -0
- package/src/Bin.res.mjs +97 -0
- package/src/ChainFetcher.res +33 -57
- package/src/ChainFetcher.res.mjs +197 -227
- package/src/ChainManager.res +6 -14
- package/src/ChainManager.res.mjs +74 -85
- package/src/ChainMap.res +14 -16
- package/src/ChainMap.res.mjs +38 -38
- package/src/Config.res +193 -135
- package/src/Config.res.mjs +566 -592
- package/src/Core.res +182 -0
- package/src/Core.res.mjs +207 -0
- package/src/Ecosystem.res +25 -4
- package/src/Ecosystem.res.mjs +12 -13
- package/src/Env.res +20 -13
- package/src/Env.res.mjs +124 -113
- package/src/EnvSafe.res +269 -0
- package/src/EnvSafe.res.mjs +296 -0
- package/src/EnvSafe.resi +18 -0
- package/src/Envio.res +37 -26
- package/src/Envio.res.mjs +59 -60
- package/src/ErrorHandling.res +2 -2
- package/src/ErrorHandling.res.mjs +15 -15
- package/src/EventConfigBuilder.res +219 -81
- package/src/EventConfigBuilder.res.mjs +259 -202
- package/src/EventProcessing.res +27 -38
- package/src/EventProcessing.res.mjs +165 -183
- package/src/EventUtils.res +11 -11
- package/src/EventUtils.res.mjs +21 -22
- package/src/EvmTypes.res +0 -1
- package/src/EvmTypes.res.mjs +5 -5
- package/src/FetchState.res +360 -256
- package/src/FetchState.res.mjs +958 -914
- package/src/GlobalState.res +365 -351
- package/src/GlobalState.res.mjs +958 -992
- package/src/GlobalStateManager.res +1 -2
- package/src/GlobalStateManager.res.mjs +36 -44
- package/src/HandlerLoader.res +107 -23
- package/src/HandlerLoader.res.mjs +128 -38
- package/src/HandlerRegister.res +127 -103
- package/src/HandlerRegister.res.mjs +164 -164
- package/src/HandlerRegister.resi +12 -4
- package/src/Hasura.res +35 -22
- package/src/Hasura.res.mjs +158 -167
- package/src/InMemoryStore.res +20 -27
- package/src/InMemoryStore.res.mjs +64 -80
- package/src/InMemoryTable.res +34 -39
- package/src/InMemoryTable.res.mjs +165 -170
- package/src/Internal.res +52 -33
- package/src/Internal.res.mjs +84 -81
- package/src/LazyLoader.res.mjs +55 -61
- package/src/LoadLayer.res +77 -78
- package/src/LoadLayer.res.mjs +160 -189
- package/src/LoadManager.res +16 -21
- package/src/LoadManager.res.mjs +79 -84
- package/src/LogSelection.res +236 -68
- package/src/LogSelection.res.mjs +211 -141
- package/src/Logging.res +13 -9
- package/src/Logging.res.mjs +130 -143
- package/src/Main.res +428 -51
- package/src/Main.res.mjs +528 -271
- package/src/Persistence.res +77 -84
- package/src/Persistence.res.mjs +131 -132
- package/src/PgStorage.res +291 -167
- package/src/PgStorage.res.mjs +797 -817
- package/src/Prometheus.res +50 -58
- package/src/Prometheus.res.mjs +345 -373
- package/src/ReorgDetection.res +22 -24
- package/src/ReorgDetection.res.mjs +100 -106
- package/src/SafeCheckpointTracking.res +7 -7
- package/src/SafeCheckpointTracking.res.mjs +40 -43
- package/src/SimulateItems.res +41 -49
- package/src/SimulateItems.res.mjs +257 -272
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +22 -26
- package/src/TableIndices.res +1 -2
- package/src/TableIndices.res.mjs +42 -48
- package/src/TestIndexer.res +196 -189
- package/src/TestIndexer.res.mjs +536 -536
- package/src/TestIndexerProxyStorage.res +15 -16
- package/src/TestIndexerProxyStorage.res.mjs +98 -122
- package/src/TestIndexerWorker.res +4 -0
- package/src/TestIndexerWorker.res.mjs +7 -0
- package/src/Throttler.res +3 -3
- package/src/Throttler.res.mjs +23 -24
- package/src/Time.res +1 -1
- package/src/Time.res.mjs +18 -21
- package/src/TopicFilter.res +3 -3
- package/src/TopicFilter.res.mjs +29 -30
- package/src/UserContext.res +93 -54
- package/src/UserContext.res.mjs +197 -182
- package/src/Utils.res +141 -86
- package/src/Utils.res.mjs +334 -295
- package/src/bindings/BigDecimal.res +0 -2
- package/src/bindings/BigDecimal.res.mjs +19 -23
- package/src/bindings/ClickHouse.res +28 -27
- package/src/bindings/ClickHouse.res.mjs +243 -240
- package/src/bindings/DateFns.res +11 -11
- package/src/bindings/DateFns.res.mjs +7 -7
- package/src/bindings/EventSource.res.mjs +2 -2
- package/src/bindings/Express.res +2 -5
- package/src/bindings/Hrtime.res +2 -2
- package/src/bindings/Hrtime.res.mjs +30 -32
- package/src/bindings/Lodash.res.mjs +1 -1
- package/src/bindings/NodeJs.res +14 -9
- package/src/bindings/NodeJs.res.mjs +20 -20
- package/src/bindings/Pino.res +8 -10
- package/src/bindings/Pino.res.mjs +40 -43
- package/src/bindings/Postgres.res +2 -5
- package/src/bindings/Postgres.res.mjs +9 -9
- package/src/bindings/PromClient.res +17 -2
- package/src/bindings/PromClient.res.mjs +30 -7
- package/src/bindings/SDSL.res.mjs +2 -2
- package/src/bindings/Viem.res +4 -4
- package/src/bindings/Viem.res.mjs +20 -22
- package/src/bindings/Vitest.res +1 -1
- package/src/bindings/Vitest.res.mjs +2 -2
- package/src/bindings/WebSocket.res +1 -1
- package/src/db/EntityHistory.res +9 -3
- package/src/db/EntityHistory.res.mjs +84 -59
- package/src/db/InternalTable.res +62 -60
- package/src/db/InternalTable.res.mjs +271 -203
- package/src/db/Schema.res +1 -2
- package/src/db/Schema.res.mjs +28 -32
- package/src/db/Table.res +28 -27
- package/src/db/Table.res.mjs +276 -292
- package/src/sources/EventRouter.res +21 -16
- package/src/sources/EventRouter.res.mjs +55 -57
- package/src/sources/Evm.res +17 -1
- package/src/sources/Evm.res.mjs +16 -8
- package/src/sources/EvmChain.res +15 -17
- package/src/sources/EvmChain.res.mjs +40 -42
- package/src/sources/Fuel.res +14 -1
- package/src/sources/Fuel.res.mjs +16 -8
- package/src/sources/FuelSDK.res +1 -1
- package/src/sources/FuelSDK.res.mjs +6 -8
- package/src/sources/HyperFuel.res +8 -10
- package/src/sources/HyperFuel.res.mjs +113 -123
- package/src/sources/HyperFuelClient.res.mjs +6 -7
- package/src/sources/HyperFuelSource.res +19 -20
- package/src/sources/HyperFuelSource.res.mjs +339 -356
- package/src/sources/HyperSync.res +11 -13
- package/src/sources/HyperSync.res.mjs +206 -220
- package/src/sources/HyperSyncClient.res +5 -7
- package/src/sources/HyperSyncClient.res.mjs +70 -75
- package/src/sources/HyperSyncHeightStream.res +8 -9
- package/src/sources/HyperSyncHeightStream.res.mjs +78 -86
- package/src/sources/HyperSyncJsonApi.res +18 -15
- package/src/sources/HyperSyncJsonApi.res.mjs +201 -231
- package/src/sources/HyperSyncSource.res +17 -21
- package/src/sources/HyperSyncSource.res.mjs +268 -290
- package/src/sources/Rpc.res +5 -5
- package/src/sources/Rpc.res.mjs +168 -192
- package/src/sources/RpcSource.res +166 -167
- package/src/sources/RpcSource.res.mjs +972 -1046
- package/src/sources/RpcWebSocketHeightStream.res +10 -11
- package/src/sources/RpcWebSocketHeightStream.res.mjs +131 -145
- package/src/sources/SimulateSource.res +1 -1
- package/src/sources/SimulateSource.res.mjs +35 -38
- package/src/sources/Source.res +1 -1
- package/src/sources/Source.res.mjs +3 -3
- package/src/sources/SourceManager.res +39 -20
- package/src/sources/SourceManager.res.mjs +340 -371
- package/src/sources/SourceManager.resi +2 -1
- package/src/sources/Svm.res +12 -5
- package/src/sources/Svm.res.mjs +44 -41
- package/src/tui/Tui.res +23 -12
- package/src/tui/Tui.res.mjs +292 -290
- package/src/tui/bindings/Ink.res +2 -4
- package/src/tui/bindings/Ink.res.mjs +35 -41
- package/src/tui/components/BufferedProgressBar.res +7 -7
- package/src/tui/components/BufferedProgressBar.res.mjs +46 -46
- package/src/tui/components/CustomHooks.res +1 -2
- package/src/tui/components/CustomHooks.res.mjs +102 -122
- package/src/tui/components/Messages.res +1 -2
- package/src/tui/components/Messages.res.mjs +38 -42
- package/src/tui/components/SyncETA.res +10 -11
- package/src/tui/components/SyncETA.res.mjs +178 -196
- package/src/tui/components/TuiData.res +1 -1
- package/src/tui/components/TuiData.res.mjs +7 -6
- package/src/vendored/Rest.res +52 -66
- package/src/vendored/Rest.res.mjs +324 -364
- package/svm.schema.json +67 -0
- package/src/Address.gen.ts +0 -8
- package/src/Config.gen.ts +0 -19
- package/src/Envio.gen.ts +0 -55
- package/src/EvmTypes.gen.ts +0 -6
- package/src/InMemoryStore.gen.ts +0 -6
- package/src/Internal.gen.ts +0 -64
- package/src/PgStorage.gen.ts +0 -10
- package/src/PgStorage.res.d.mts +0 -5
- package/src/Types.ts +0 -56
- package/src/bindings/BigDecimal.gen.ts +0 -14
- package/src/bindings/BigDecimal.res.d.mts +0 -5
- package/src/bindings/BigInt.gen.ts +0 -10
- package/src/bindings/BigInt.res +0 -70
- package/src/bindings/BigInt.res.d.mts +0 -5
- package/src/bindings/BigInt.res.mjs +0 -154
- package/src/bindings/Ethers.res.d.mts +0 -5
- package/src/bindings/Pino.gen.ts +0 -17
- package/src/bindings/Postgres.gen.ts +0 -8
- package/src/bindings/Postgres.res.d.mts +0 -5
- package/src/bindings/Promise.res +0 -67
- package/src/bindings/Promise.res.mjs +0 -26
- package/src/db/InternalTable.gen.ts +0 -36
- package/src/sources/HyperSyncClient.gen.ts +0 -19
package/src/LoadLayer.res.mjs
CHANGED
|
@@ -4,9 +4,8 @@ import * as Utils from "./Utils.res.mjs";
|
|
|
4
4
|
import * as Hrtime from "./bindings/Hrtime.res.mjs";
|
|
5
5
|
import * as Logging from "./Logging.res.mjs";
|
|
6
6
|
import * as Internal from "./Internal.res.mjs";
|
|
7
|
-
import * as Belt_Array from "rescript/lib/es6/
|
|
7
|
+
import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
|
|
8
8
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
9
|
-
import * as Caml_option from "rescript/lib/es6/caml_option.js";
|
|
10
9
|
import * as LoadManager from "./LoadManager.res.mjs";
|
|
11
10
|
import * as Persistence from "./Persistence.res.mjs";
|
|
12
11
|
import * as TableIndices from "./TableIndices.res.mjs";
|
|
@@ -14,90 +13,82 @@ import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
|
14
13
|
import * as InMemoryStore from "./InMemoryStore.res.mjs";
|
|
15
14
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
16
15
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
17
|
-
import * as
|
|
16
|
+
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
18
17
|
|
|
19
18
|
function loadById(loadManager, persistence, entityConfig, inMemoryStore, shouldGroup, item, entityId) {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
let key = entityConfig.name + `.get`;
|
|
20
|
+
let inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
|
|
21
|
+
let load = async (idsToLoad, param) => {
|
|
22
|
+
let storage = Persistence.getInitializedStorageOrThrow(persistence);
|
|
23
|
+
let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
|
|
24
|
+
let dbEntities;
|
|
25
25
|
try {
|
|
26
|
-
dbEntities = await
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
26
|
+
dbEntities = await storage.loadByIdsOrThrow(idsToLoad, entityConfig.table, entityConfig.rowsSchema);
|
|
27
|
+
} catch (raw_exn) {
|
|
28
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
30
29
|
if (exn.RE_EXN_ID === Persistence.StorageError) {
|
|
31
30
|
dbEntities = ErrorHandling.mkLogAndRaise(Logging.getItemLogger(item), exn.message, exn.reason);
|
|
32
31
|
} else {
|
|
33
32
|
throw exn;
|
|
34
33
|
}
|
|
35
34
|
}
|
|
36
|
-
|
|
37
|
-
for(
|
|
38
|
-
|
|
35
|
+
let entitiesMap = {};
|
|
36
|
+
for (let idx = 0, idx_finish = dbEntities.length; idx < idx_finish; ++idx) {
|
|
37
|
+
let entity = dbEntities[idx];
|
|
39
38
|
entitiesMap[entity.id] = entity;
|
|
40
39
|
}
|
|
41
|
-
idsToLoad.forEach(
|
|
42
|
-
|
|
43
|
-
});
|
|
44
|
-
return Prometheus.StorageLoad.endOperation(timerRef, key, idsToLoad.length, dbEntities.length);
|
|
40
|
+
idsToLoad.forEach(entityId => InMemoryTable.Entity.initValue(inMemTable, entityId, entitiesMap[entityId], false));
|
|
41
|
+
return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
|
|
45
42
|
};
|
|
46
|
-
return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, (
|
|
47
|
-
return InMemoryTable.hasByHash(inMemTable.table, hash);
|
|
48
|
-
}), InMemoryTable.Entity.getUnsafe(inMemTable));
|
|
43
|
+
return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, hash => InMemoryTable.hasByHash(inMemTable.table, hash), InMemoryTable.Entity.getUnsafe(inMemTable));
|
|
49
44
|
}
|
|
50
45
|
|
|
51
46
|
function callEffect(effect, arg, inMemTable, timerRef, onError) {
|
|
52
|
-
|
|
53
|
-
|
|
47
|
+
let effectName = effect.name;
|
|
48
|
+
let hadActiveCalls = effect.activeCallsCount > 0;
|
|
54
49
|
effect.activeCallsCount = effect.activeCallsCount + 1 | 0;
|
|
55
50
|
Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
|
|
56
51
|
if (hadActiveCalls) {
|
|
57
|
-
|
|
52
|
+
let elapsed = Hrtime.secondsBetween(effect.prevCallStartTimerRef, timerRef);
|
|
58
53
|
if (elapsed > 0) {
|
|
59
54
|
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, effectName, elapsed);
|
|
60
55
|
}
|
|
61
|
-
|
|
62
56
|
}
|
|
63
57
|
effect.prevCallStartTimerRef = timerRef;
|
|
64
|
-
return effect.handler(arg).then(
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
Prometheus.SafeCounter.increment(Prometheus.EffectCalls.totalCallsCount, effectName);
|
|
80
|
-
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.sumTimeCounter, effectName, Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef)));
|
|
81
|
-
});
|
|
58
|
+
return effect.handler(arg).then(output => {
|
|
59
|
+
inMemTable.dict[arg.cacheKey] = output;
|
|
60
|
+
if (arg.context.cache) {
|
|
61
|
+
inMemTable.idsToStore.push(arg.cacheKey);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
}).catch(exn => onError(arg.cacheKey, exn)).finally(() => {
|
|
65
|
+
effect.activeCallsCount = effect.activeCallsCount - 1 | 0;
|
|
66
|
+
Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
|
|
67
|
+
let newTimer = Hrtime.makeTimer();
|
|
68
|
+
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, effectName, Hrtime.secondsBetween(effect.prevCallStartTimerRef, newTimer));
|
|
69
|
+
effect.prevCallStartTimerRef = newTimer;
|
|
70
|
+
Prometheus.SafeCounter.increment(Prometheus.EffectCalls.totalCallsCount, effectName);
|
|
71
|
+
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.sumTimeCounter, effectName, Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef)));
|
|
72
|
+
});
|
|
82
73
|
}
|
|
83
74
|
|
|
84
75
|
function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQueue) {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
76
|
+
let effectName = effect.name;
|
|
77
|
+
let timerRef = Hrtime.makeTimer();
|
|
78
|
+
let promises = [];
|
|
79
|
+
let state = effect.rateLimit;
|
|
89
80
|
if (state !== undefined) {
|
|
90
|
-
|
|
81
|
+
let now = Date.now();
|
|
91
82
|
if (now >= state.windowStartTime + state.durationMs) {
|
|
92
83
|
state.availableCalls = state.callsPerDuration;
|
|
93
84
|
state.windowStartTime = now;
|
|
94
85
|
state.nextWindowPromise = undefined;
|
|
95
86
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
87
|
+
let immediateCount = Math.min(state.availableCalls, effectArgs.length);
|
|
88
|
+
let immediateArgs = Belt_Array.slice(effectArgs, 0, immediateCount);
|
|
89
|
+
let queuedArgs = Belt_Array.sliceToEnd(effectArgs, immediateCount);
|
|
99
90
|
state.availableCalls = state.availableCalls - immediateCount | 0;
|
|
100
|
-
for(
|
|
91
|
+
for (let idx = 0, idx_finish = immediateArgs.length; idx < idx_finish; ++idx) {
|
|
101
92
|
promises.push(callEffect(effect, immediateArgs[idx], inMemTable, timerRef, onError));
|
|
102
93
|
}
|
|
103
94
|
if (immediateCount > 0 && isFromQueue) {
|
|
@@ -109,29 +100,28 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
|
|
|
109
100
|
state.queueCount = state.queueCount + queuedArgs.length | 0;
|
|
110
101
|
Prometheus.EffectQueueCount.set(state.queueCount, effectName);
|
|
111
102
|
}
|
|
112
|
-
|
|
103
|
+
let millisUntilReset = {
|
|
113
104
|
contents: 0
|
|
114
105
|
};
|
|
115
|
-
|
|
116
|
-
|
|
106
|
+
let p = state.nextWindowPromise;
|
|
107
|
+
let nextWindowPromise;
|
|
117
108
|
if (p !== undefined) {
|
|
118
|
-
nextWindowPromise =
|
|
109
|
+
nextWindowPromise = p;
|
|
119
110
|
} else {
|
|
120
111
|
millisUntilReset.contents = state.windowStartTime + state.durationMs - now | 0;
|
|
121
|
-
|
|
122
|
-
state.nextWindowPromise =
|
|
112
|
+
let p$1 = Utils.delay(millisUntilReset.contents);
|
|
113
|
+
state.nextWindowPromise = p$1;
|
|
123
114
|
nextWindowPromise = p$1;
|
|
124
115
|
}
|
|
125
|
-
promises.push(nextWindowPromise.then(
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
116
|
+
promises.push(nextWindowPromise.then(() => {
|
|
117
|
+
if (millisUntilReset.contents > 0) {
|
|
118
|
+
Prometheus.SafeCounter.handleFloat(Prometheus.EffectQueueCount.timeCounter, effectName, millisUntilReset.contents / 1000);
|
|
119
|
+
}
|
|
120
|
+
return executeWithRateLimit(effect, queuedArgs, inMemTable, onError, true);
|
|
121
|
+
}));
|
|
131
122
|
}
|
|
132
|
-
|
|
133
123
|
} else {
|
|
134
|
-
for(
|
|
124
|
+
for (let idx$1 = 0, idx_finish$1 = effectArgs.length; idx$1 < idx_finish$1; ++idx$1) {
|
|
135
125
|
promises.push(callEffect(effect, effectArgs[idx$1], inMemTable, timerRef, onError));
|
|
136
126
|
}
|
|
137
127
|
}
|
|
@@ -139,158 +129,139 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
|
|
|
139
129
|
}
|
|
140
130
|
|
|
141
131
|
function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore, shouldGroup, item) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
var match = persistence.storageStatus;
|
|
151
|
-
var tmp;
|
|
132
|
+
let effectName = effect.name;
|
|
133
|
+
let key = effectName + `.effect`;
|
|
134
|
+
let inMemTable = InMemoryStore.getEffectInMemTable(inMemoryStore, effect);
|
|
135
|
+
let load = async (args, onError) => {
|
|
136
|
+
let idsToLoad = args.map(arg => arg.cacheKey);
|
|
137
|
+
let idsFromCache = new Set();
|
|
138
|
+
let match = persistence.storageStatus;
|
|
139
|
+
let tmp;
|
|
152
140
|
tmp = typeof match !== "object" || match.TAG === "Initializing" ? false : Utils.Dict.has(match._0.cache, effectName);
|
|
153
141
|
if (tmp) {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
142
|
+
let storage = Persistence.getInitializedStorageOrThrow(persistence);
|
|
143
|
+
let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
|
|
144
|
+
let match$1 = effect.storageMeta;
|
|
145
|
+
let outputSchema = match$1.outputSchema;
|
|
146
|
+
let dbEntities;
|
|
158
147
|
try {
|
|
159
|
-
dbEntities = await
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
148
|
+
dbEntities = await storage.loadByIdsOrThrow(idsToLoad, match$1.table, Internal.effectCacheItemRowsSchema);
|
|
149
|
+
} catch (raw_exn) {
|
|
150
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
163
151
|
Logging.childWarn(Logging.getItemLogger(item), {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
152
|
+
msg: `Failed to load cache effect cache. The indexer will continue working, but the effect will not be able to use the cache.`,
|
|
153
|
+
err: Utils.prettifyExn(exn),
|
|
154
|
+
effect: effectName
|
|
155
|
+
});
|
|
168
156
|
dbEntities = [];
|
|
169
157
|
}
|
|
170
|
-
dbEntities.forEach(
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
Prometheus.StorageLoad.endOperation(timerRef, key, idsToLoad.length, dbEntities.length);
|
|
158
|
+
dbEntities.forEach(dbEntity => {
|
|
159
|
+
try {
|
|
160
|
+
let output = S$RescriptSchema.parseOrThrow(dbEntity.output, outputSchema);
|
|
161
|
+
idsFromCache.add(dbEntity.id);
|
|
162
|
+
inMemTable.dict[dbEntity.id] = output;
|
|
163
|
+
return;
|
|
164
|
+
} catch (raw_error) {
|
|
165
|
+
let error = Primitive_exceptions.internalToException(raw_error);
|
|
166
|
+
if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
|
|
167
|
+
inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1 | 0;
|
|
168
|
+
Prometheus.EffectCacheInvalidationsCount.increment(effectName);
|
|
169
|
+
return Logging.childTrace(Logging.getItemLogger(item), {
|
|
170
|
+
msg: "Invalidated effect cache",
|
|
171
|
+
input: dbEntity.id,
|
|
172
|
+
effect: effectName,
|
|
173
|
+
err: S$RescriptSchema.$$Error.message(error._1)
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
throw error;
|
|
177
|
+
}
|
|
178
|
+
});
|
|
179
|
+
Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
|
|
193
180
|
}
|
|
194
|
-
|
|
181
|
+
let remainingCallsCount = idsToLoad.length - idsFromCache.size | 0;
|
|
195
182
|
if (remainingCallsCount <= 0) {
|
|
196
|
-
return
|
|
183
|
+
return;
|
|
197
184
|
}
|
|
198
|
-
|
|
199
|
-
for(
|
|
200
|
-
|
|
185
|
+
let argsToCall = [];
|
|
186
|
+
for (let idx = 0, idx_finish = args.length; idx < idx_finish; ++idx) {
|
|
187
|
+
let arg = args[idx];
|
|
201
188
|
if (!idsFromCache.has(arg.cacheKey)) {
|
|
202
189
|
argsToCall.push(arg);
|
|
203
190
|
}
|
|
204
|
-
|
|
205
191
|
}
|
|
206
192
|
if (Utils.$$Array.notEmpty(argsToCall)) {
|
|
207
193
|
return await executeWithRateLimit(effect, argsToCall, inMemTable, onError, false);
|
|
208
194
|
}
|
|
209
|
-
|
|
210
195
|
};
|
|
211
|
-
return LoadManager.call(loadManager, effectArgs, key, load, (
|
|
212
|
-
return args.cacheKey;
|
|
213
|
-
}), shouldGroup, (function (hash) {
|
|
214
|
-
return Utils.Dict.has(inMemTable.dict, hash);
|
|
215
|
-
}), (function (hash) {
|
|
216
|
-
return inMemTable.dict[hash];
|
|
217
|
-
}));
|
|
196
|
+
return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => Utils.Dict.has(inMemTable.dict, hash), hash => inMemTable.dict[hash]);
|
|
218
197
|
}
|
|
219
198
|
|
|
220
199
|
function loadByField(loadManager, persistence, operator, entityConfig, inMemoryStore, fieldName, fieldValueSchema, shouldGroup, item, fieldValue) {
|
|
221
|
-
|
|
200
|
+
let operatorCallName;
|
|
222
201
|
switch (operator) {
|
|
223
202
|
case "Eq" :
|
|
224
|
-
|
|
225
|
-
|
|
203
|
+
operatorCallName = "eq";
|
|
204
|
+
break;
|
|
226
205
|
case "Gt" :
|
|
227
|
-
|
|
228
|
-
|
|
206
|
+
operatorCallName = "gt";
|
|
207
|
+
break;
|
|
229
208
|
case "Lt" :
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
209
|
+
operatorCallName = "lt";
|
|
210
|
+
break;
|
|
233
211
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
212
|
+
let key = entityConfig.name + `.getWhere.` + fieldName + `.` + operatorCallName;
|
|
213
|
+
let inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
|
|
214
|
+
let load = async (fieldValues, param) => {
|
|
215
|
+
let storage = Persistence.getInitializedStorageOrThrow(persistence);
|
|
216
|
+
let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
|
|
217
|
+
let size = {
|
|
239
218
|
contents: 0
|
|
240
219
|
};
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
}), exn.message, exn.reason);
|
|
280
|
-
}
|
|
281
|
-
throw exn;
|
|
282
|
-
}
|
|
283
|
-
}));
|
|
284
|
-
return Prometheus.StorageLoad.endOperation(timerRef, key, fieldValues.length, size.contents);
|
|
220
|
+
let indiciesToLoad = fieldValues.map(fieldValue => ({
|
|
221
|
+
fieldName: fieldName,
|
|
222
|
+
fieldValue: fieldValue,
|
|
223
|
+
operator: operator
|
|
224
|
+
}));
|
|
225
|
+
await Promise.all(indiciesToLoad.map(async index => {
|
|
226
|
+
InMemoryTable.Entity.addEmptyIndex(inMemTable, index);
|
|
227
|
+
try {
|
|
228
|
+
let tmp;
|
|
229
|
+
switch (index.operator) {
|
|
230
|
+
case "Eq" :
|
|
231
|
+
tmp = "=";
|
|
232
|
+
break;
|
|
233
|
+
case "Gt" :
|
|
234
|
+
tmp = ">";
|
|
235
|
+
break;
|
|
236
|
+
case "Lt" :
|
|
237
|
+
tmp = "<";
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
let entities = await storage.loadByFieldOrThrow(TableIndices.Index.getFieldName(index), fieldValueSchema, index.fieldValue, tmp, entityConfig.table, entityConfig.rowsSchema);
|
|
241
|
+
entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, entity.id, entity, false));
|
|
242
|
+
size.contents = size.contents + entities.length | 0;
|
|
243
|
+
return;
|
|
244
|
+
} catch (raw_exn) {
|
|
245
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
246
|
+
if (exn.RE_EXN_ID === Persistence.StorageError) {
|
|
247
|
+
return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Logging.getItemLogger(item), {
|
|
248
|
+
operator: operatorCallName,
|
|
249
|
+
tableName: entityConfig.table.tableName,
|
|
250
|
+
fieldName: fieldName,
|
|
251
|
+
fieldValue: fieldValue
|
|
252
|
+
}), exn.message, exn.reason);
|
|
253
|
+
}
|
|
254
|
+
throw exn;
|
|
255
|
+
}
|
|
256
|
+
}));
|
|
257
|
+
return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, fieldValues.length, size.contents);
|
|
285
258
|
};
|
|
286
|
-
return LoadManager.call(loadManager, fieldValue, key, load,
|
|
287
|
-
return TableIndices.FieldValue.toString(fieldValue);
|
|
288
|
-
}), shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable, fieldName, operator), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable, fieldName, operator));
|
|
259
|
+
return LoadManager.call(loadManager, fieldValue, key, load, fieldValue => TableIndices.FieldValue.toString(fieldValue), shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable, fieldName, operator), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable, fieldName, operator));
|
|
289
260
|
}
|
|
290
261
|
|
|
291
262
|
export {
|
|
292
|
-
loadById
|
|
293
|
-
loadByField
|
|
294
|
-
loadEffect
|
|
263
|
+
loadById,
|
|
264
|
+
loadByField,
|
|
265
|
+
loadEffect,
|
|
295
266
|
}
|
|
296
267
|
/* Utils Not a pure module */
|
package/src/LoadManager.res
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
open Belt
|
|
2
|
-
|
|
3
1
|
module Call = {
|
|
4
2
|
type input
|
|
5
3
|
type output
|
|
@@ -31,7 +29,7 @@ type t = {
|
|
|
31
29
|
}
|
|
32
30
|
|
|
33
31
|
let make = () => {
|
|
34
|
-
groups:
|
|
32
|
+
groups: Dict.make(),
|
|
35
33
|
isCollecting: false,
|
|
36
34
|
}
|
|
37
35
|
|
|
@@ -47,22 +45,19 @@ let schedule = async loadManager => {
|
|
|
47
45
|
|
|
48
46
|
let groups = loadManager.groups
|
|
49
47
|
groups
|
|
50
|
-
->
|
|
48
|
+
->Dict.keysToArray
|
|
51
49
|
->Utils.Array.forEachAsync(async key => {
|
|
52
|
-
let group = groups->
|
|
50
|
+
let group = groups->Dict.getUnsafe(key)
|
|
53
51
|
let calls = group.calls
|
|
54
52
|
|
|
55
53
|
let inputsToLoad = []
|
|
56
54
|
let currentInputKeys = []
|
|
57
|
-
calls
|
|
58
|
-
->Js.Dict.keys
|
|
59
|
-
->Js.Array2.forEach(inputKey => {
|
|
60
|
-
let call = calls->Js.Dict.unsafeGet(inputKey)
|
|
55
|
+
calls->Utils.Dict.forEachWithKey((call, inputKey) => {
|
|
61
56
|
if !call.isLoading {
|
|
62
57
|
call.isLoading = true
|
|
63
|
-
currentInputKeys->
|
|
58
|
+
currentInputKeys->Array.push(inputKey)->ignore
|
|
64
59
|
if group.hasInMemory(inputKey)->not {
|
|
65
|
-
inputsToLoad->
|
|
60
|
+
inputsToLoad->Array.push(call.input)->ignore
|
|
66
61
|
}
|
|
67
62
|
}
|
|
68
63
|
})
|
|
@@ -70,14 +65,14 @@ let schedule = async loadManager => {
|
|
|
70
65
|
if inputsToLoad->Utils.Array.isEmpty->not {
|
|
71
66
|
try {
|
|
72
67
|
await group.load(inputsToLoad, ~onError=(~inputKey, ~exn) => {
|
|
73
|
-
let call = calls->
|
|
68
|
+
let call = calls->Dict.getUnsafe(inputKey)
|
|
74
69
|
call.exn = Some(exn)
|
|
75
70
|
})
|
|
76
71
|
} catch {
|
|
77
72
|
| exn => {
|
|
78
73
|
let exn = exn->Utils.prettifyExn
|
|
79
74
|
currentInputKeys->Array.forEach(inputKey => {
|
|
80
|
-
let call = calls->
|
|
75
|
+
let call = calls->Dict.getUnsafe(inputKey)
|
|
81
76
|
call.exn = Some(exn)
|
|
82
77
|
})
|
|
83
78
|
}
|
|
@@ -85,8 +80,8 @@ let schedule = async loadManager => {
|
|
|
85
80
|
}
|
|
86
81
|
|
|
87
82
|
if currentInputKeys->Utils.Array.isEmpty->not {
|
|
88
|
-
currentInputKeys->
|
|
89
|
-
let call = calls->
|
|
83
|
+
currentInputKeys->Array.forEach(inputKey => {
|
|
84
|
+
let call = calls->Dict.getUnsafe(inputKey)
|
|
90
85
|
calls->Utils.Dict.deleteInPlace(inputKey)
|
|
91
86
|
switch call.exn {
|
|
92
87
|
| Some(exn) => call.reject(exn->Utils.prettifyExn)
|
|
@@ -97,8 +92,8 @@ let schedule = async loadManager => {
|
|
|
97
92
|
// Clean up executed batch to reset
|
|
98
93
|
// provided load function which
|
|
99
94
|
// might have an outdated function context
|
|
100
|
-
let latestGroup = groups->
|
|
101
|
-
if latestGroup.calls->
|
|
95
|
+
let latestGroup = groups->Dict.getUnsafe(key)
|
|
96
|
+
if latestGroup.calls->Utils.Dict.isEmpty {
|
|
102
97
|
groups->Utils.Dict.deleteInPlace(key)
|
|
103
98
|
}
|
|
104
99
|
}
|
|
@@ -144,7 +139,7 @@ let call = (
|
|
|
144
139
|
| Some(group) => group
|
|
145
140
|
| None => {
|
|
146
141
|
let g: Group.t = {
|
|
147
|
-
calls:
|
|
142
|
+
calls: Dict.make(),
|
|
148
143
|
load: load->(
|
|
149
144
|
Utils.magic: (
|
|
150
145
|
(array<'input>, ~onError: (~inputKey: string, ~exn: exn) => unit) => promise<unit>
|
|
@@ -158,7 +153,7 @@ let call = (
|
|
|
158
153
|
),
|
|
159
154
|
hasInMemory: hasInMemory->(Utils.magic: (string => bool) => string => bool),
|
|
160
155
|
}
|
|
161
|
-
loadManager.groups->
|
|
156
|
+
loadManager.groups->Dict.set(key, g)
|
|
162
157
|
g
|
|
163
158
|
}
|
|
164
159
|
}
|
|
@@ -175,11 +170,11 @@ let call = (
|
|
|
175
170
|
exn: None,
|
|
176
171
|
isLoading: false,
|
|
177
172
|
}
|
|
178
|
-
group.calls->
|
|
173
|
+
group.calls->Dict.set(inputKey, call)
|
|
179
174
|
})
|
|
180
175
|
|
|
181
176
|
// Don't use ref since it'll allocate an object to store .contents
|
|
182
|
-
(group.calls->
|
|
177
|
+
(group.calls->Dict.getUnsafe(inputKey)).promise = promise
|
|
183
178
|
|
|
184
179
|
if !loadManager.isCollecting {
|
|
185
180
|
let _: promise<unit> = loadManager->schedule
|