envio 3.1.2 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +83 -11
- package/fuel.schema.json +83 -11
- package/index.d.ts +184 -3
- package/package.json +6 -6
- package/src/Batch.res +2 -2
- package/src/ChainFetcher.res +27 -3
- package/src/ChainFetcher.res.mjs +17 -3
- package/src/ChainManager.res +163 -0
- package/src/ChainManager.res.mjs +136 -0
- package/src/Config.res +213 -30
- package/src/Config.res.mjs +102 -41
- package/src/Core.res +16 -10
- package/src/Ecosystem.res +0 -3
- package/src/Env.res +2 -2
- package/src/Env.res.mjs +2 -2
- package/src/Envio.res +101 -2
- package/src/Envio.res.mjs +2 -3
- package/src/EventConfigBuilder.res +52 -0
- package/src/EventConfigBuilder.res.mjs +32 -0
- package/src/EventUtils.res +2 -2
- package/src/FetchState.res +23 -14
- package/src/FetchState.res.mjs +21 -15
- package/src/GlobalState.res +219 -363
- package/src/GlobalState.res.mjs +314 -491
- package/src/GlobalStateManager.res +49 -59
- package/src/GlobalStateManager.res.mjs +5 -4
- package/src/GlobalStateManager.resi +1 -1
- package/src/HandlerLoader.res +12 -1
- package/src/HandlerLoader.res.mjs +6 -1
- package/src/HandlerRegister.res +9 -9
- package/src/HandlerRegister.res.mjs +9 -9
- package/src/Hasura.res +102 -32
- package/src/Hasura.res.mjs +88 -34
- package/src/InMemoryStore.res +10 -1
- package/src/InMemoryStore.res.mjs +4 -1
- package/src/InMemoryTable.res +83 -136
- package/src/InMemoryTable.res.mjs +57 -86
- package/src/Internal.res +54 -5
- package/src/Internal.res.mjs +2 -8
- package/src/LazyLoader.res +2 -2
- package/src/LazyLoader.res.mjs +3 -3
- package/src/LoadLayer.res +47 -60
- package/src/LoadLayer.res.mjs +28 -50
- package/src/LoadLayer.resi +2 -5
- package/src/LogSelection.res +4 -4
- package/src/LogSelection.res.mjs +5 -7
- package/src/Logging.res +1 -1
- package/src/Main.res +61 -2
- package/src/Main.res.mjs +37 -1
- package/src/Persistence.res +3 -16
- package/src/PgStorage.res +125 -114
- package/src/PgStorage.res.mjs +112 -95
- package/src/Ports.res +5 -0
- package/src/Ports.res.mjs +9 -0
- package/src/Prometheus.res +3 -3
- package/src/Prometheus.res.mjs +4 -4
- package/src/ReorgDetection.res +4 -4
- package/src/ReorgDetection.res.mjs +4 -5
- package/src/SafeCheckpointTracking.res +16 -16
- package/src/SafeCheckpointTracking.res.mjs +2 -2
- package/src/SimulateItems.res +10 -14
- package/src/SimulateItems.res.mjs +5 -2
- package/src/Sink.res +1 -1
- package/src/Sink.res.mjs +1 -2
- package/src/SvmTypes.res +9 -0
- package/src/SvmTypes.res.mjs +14 -0
- package/src/TestIndexer.res +17 -57
- package/src/TestIndexer.res.mjs +14 -48
- package/src/TestIndexerProxyStorage.res +23 -23
- package/src/TestIndexerProxyStorage.res.mjs +12 -15
- package/src/Throttler.res +2 -2
- package/src/Time.res +2 -2
- package/src/Time.res.mjs +2 -2
- package/src/UserContext.res +19 -118
- package/src/UserContext.res.mjs +10 -66
- package/src/Utils.res +15 -15
- package/src/Utils.res.mjs +7 -8
- package/src/adapters/MarkBatchProcessedAdapter.res +5 -0
- package/src/adapters/MarkBatchProcessedAdapter.res.mjs +14 -0
- package/src/bindings/BigDecimal.res +1 -1
- package/src/bindings/BigDecimal.res.mjs +2 -2
- package/src/bindings/ClickHouse.res +8 -6
- package/src/bindings/ClickHouse.res.mjs +5 -5
- package/src/bindings/Hrtime.res +1 -1
- package/src/bindings/Pino.res +2 -2
- package/src/bindings/Pino.res.mjs +3 -4
- package/src/db/EntityFilter.res +410 -0
- package/src/db/EntityFilter.res.mjs +424 -0
- package/src/db/EntityHistory.res +1 -1
- package/src/db/EntityHistory.res.mjs +1 -1
- package/src/db/InternalTable.res +10 -10
- package/src/db/InternalTable.res.mjs +41 -45
- package/src/db/Schema.res +2 -2
- package/src/db/Schema.res.mjs +3 -3
- package/src/db/Table.res +106 -22
- package/src/db/Table.res.mjs +84 -35
- package/src/sources/EventRouter.res +67 -2
- package/src/sources/EventRouter.res.mjs +45 -3
- package/src/sources/Evm.res +0 -7
- package/src/sources/Evm.res.mjs +0 -15
- package/src/sources/EvmChain.res +1 -1
- package/src/sources/EvmChain.res.mjs +1 -2
- package/src/sources/EvmRpcClient.res +42 -0
- package/src/sources/EvmRpcClient.res.mjs +64 -0
- package/src/sources/Fuel.res +0 -7
- package/src/sources/Fuel.res.mjs +0 -15
- package/src/sources/HyperFuelSource.res +5 -4
- package/src/sources/HyperFuelSource.res.mjs +2 -2
- package/src/sources/HyperSyncClient.res +9 -5
- package/src/sources/HyperSyncClient.res.mjs +2 -2
- package/src/sources/HyperSyncHeightStream.res +2 -2
- package/src/sources/HyperSyncHeightStream.res.mjs +2 -2
- package/src/sources/HyperSyncSource.res +10 -9
- package/src/sources/HyperSyncSource.res.mjs +4 -4
- package/src/sources/Rpc.res +1 -5
- package/src/sources/Rpc.res.mjs +1 -9
- package/src/sources/RpcSource.res +57 -21
- package/src/sources/RpcSource.res.mjs +47 -20
- package/src/sources/RpcWebSocketHeightStream.res +1 -1
- package/src/sources/SourceManager.res +3 -2
- package/src/sources/SourceManager.res.mjs +1 -1
- package/src/sources/Svm.res +3 -10
- package/src/sources/Svm.res.mjs +4 -18
- package/src/sources/SvmHyperSyncClient.res +265 -0
- package/src/sources/SvmHyperSyncClient.res.mjs +28 -0
- package/src/sources/SvmHyperSyncSource.res +638 -0
- package/src/sources/SvmHyperSyncSource.res.mjs +557 -0
- package/src/tui/Tui.res +9 -2
- package/src/tui/Tui.res.mjs +18 -3
- package/src/tui/components/BufferedProgressBar.res +2 -2
- package/src/tui/components/TuiData.res +3 -0
- package/svm.schema.json +523 -14
- package/src/TableIndices.res +0 -115
- package/src/TableIndices.res.mjs +0 -144
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
4
|
import * as Internal from "./Internal.res.mjs";
|
|
5
|
+
import * as EntityFilter from "./db/EntityFilter.res.mjs";
|
|
5
6
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
6
|
-
import * as TableIndices from "./TableIndices.res.mjs";
|
|
7
7
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
8
8
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
9
9
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
@@ -23,33 +23,23 @@ function getEntityIdUnsafe(entity) {
|
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
function
|
|
27
|
-
let s = self.
|
|
26
|
+
function getOrCreateEntityFilters(self, entityId) {
|
|
27
|
+
let s = self.filtersByEntityId[entityId];
|
|
28
28
|
if (s !== undefined) {
|
|
29
29
|
return Primitive_option.valFromOption(s);
|
|
30
30
|
}
|
|
31
31
|
let s$1 = new Set();
|
|
32
|
-
self.
|
|
32
|
+
self.filtersByEntityId[entityId] = s$1;
|
|
33
33
|
return s$1;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
function makeIndicesSerializedToValue(index, relatedEntityIdsOpt) {
|
|
37
|
-
let relatedEntityIds = relatedEntityIdsOpt !== undefined ? Primitive_option.valFromOption(relatedEntityIdsOpt) : new Set();
|
|
38
|
-
let empty = {};
|
|
39
|
-
empty[TableIndices.Index.toString(index)] = [
|
|
40
|
-
index,
|
|
41
|
-
relatedEntityIds
|
|
42
|
-
];
|
|
43
|
-
return empty;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
36
|
function make() {
|
|
47
37
|
return {
|
|
48
38
|
latestEntityChangeById: {},
|
|
49
39
|
changesCount: 0,
|
|
50
40
|
prevEntityChanges: [],
|
|
51
|
-
|
|
52
|
-
|
|
41
|
+
filtersByEntityId: {},
|
|
42
|
+
filterIndices: {}
|
|
53
43
|
};
|
|
54
44
|
}
|
|
55
45
|
|
|
@@ -57,14 +47,20 @@ function snapshotChanges(self, committedCheckpointId, upToCheckpointId) {
|
|
|
57
47
|
let changes = [];
|
|
58
48
|
let keptPrev = [];
|
|
59
49
|
self.prevEntityChanges.forEach(change => {
|
|
60
|
-
|
|
50
|
+
let checkpointId = change.checkpointId;
|
|
51
|
+
if (checkpointId > upToCheckpointId) {
|
|
61
52
|
keptPrev.push(change);
|
|
62
|
-
|
|
53
|
+
return;
|
|
54
|
+
} else if (checkpointId > committedCheckpointId) {
|
|
63
55
|
changes.push(change);
|
|
56
|
+
return;
|
|
57
|
+
} else {
|
|
58
|
+
return;
|
|
64
59
|
}
|
|
65
60
|
});
|
|
61
|
+
let removedCount = self.prevEntityChanges.length - keptPrev.length | 0;
|
|
66
62
|
self.prevEntityChanges = keptPrev;
|
|
67
|
-
self.changesCount = self.changesCount -
|
|
63
|
+
self.changesCount = self.changesCount - removedCount;
|
|
68
64
|
Utils.Dict.forEach(self.latestEntityChangeById, change => {
|
|
69
65
|
let checkpointId = change.checkpointId;
|
|
70
66
|
if (checkpointId > committedCheckpointId && checkpointId <= upToCheckpointId) {
|
|
@@ -86,51 +82,46 @@ function dropCommittedChanges(self, committedCheckpointId, keepLoadedFromDb) {
|
|
|
86
82
|
});
|
|
87
83
|
keysToDelete.forEach(key => Utils.Dict.deleteInPlace(self.latestEntityChangeById, key));
|
|
88
84
|
self.changesCount = self.changesCount - keysToDelete.length;
|
|
89
|
-
self.
|
|
90
|
-
self.
|
|
85
|
+
self.filtersByEntityId = {};
|
|
86
|
+
self.filterIndices = {};
|
|
91
87
|
}
|
|
92
88
|
|
|
93
89
|
function updateIndices(self, entity) {
|
|
94
90
|
let entityId = getEntityIdUnsafe(entity);
|
|
95
|
-
let
|
|
96
|
-
if (
|
|
97
|
-
let
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (!TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
|
|
102
|
-
entityIndices$1.delete(index);
|
|
91
|
+
let entityFilters = self.filtersByEntityId[entityId];
|
|
92
|
+
if (entityFilters !== undefined) {
|
|
93
|
+
let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
|
|
94
|
+
entityFilters$1.forEach(filter => {
|
|
95
|
+
if (!EntityFilter.matches(filter, entity)) {
|
|
96
|
+
entityFilters$1.delete(filter);
|
|
103
97
|
return;
|
|
104
98
|
}
|
|
105
99
|
});
|
|
106
100
|
}
|
|
107
|
-
Utils.Dict.
|
|
108
|
-
let
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
relatedEntityIds.delete(entityId);
|
|
117
|
-
}
|
|
118
|
-
});
|
|
101
|
+
Utils.Dict.forEach(self.filterIndices, param => {
|
|
102
|
+
let relatedEntityIds = param[1];
|
|
103
|
+
let filter = param[0];
|
|
104
|
+
if (EntityFilter.matches(filter, entity)) {
|
|
105
|
+
relatedEntityIds.add(entityId);
|
|
106
|
+
getOrCreateEntityFilters(self, entityId).add(filter);
|
|
107
|
+
} else {
|
|
108
|
+
relatedEntityIds.delete(entityId);
|
|
109
|
+
}
|
|
119
110
|
});
|
|
120
111
|
}
|
|
121
112
|
|
|
122
113
|
function deleteEntityFromIndices(self, entityId) {
|
|
123
|
-
let
|
|
124
|
-
if (
|
|
114
|
+
let entityFilters = self.filtersByEntityId[entityId];
|
|
115
|
+
if (entityFilters === undefined) {
|
|
125
116
|
return;
|
|
126
117
|
}
|
|
127
|
-
let
|
|
128
|
-
|
|
129
|
-
let match =
|
|
118
|
+
let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
|
|
119
|
+
entityFilters$1.forEach(filter => {
|
|
120
|
+
let match = self.filterIndices[EntityFilter.toString(filter)];
|
|
130
121
|
if (match !== undefined) {
|
|
131
122
|
match[1].delete(entityId);
|
|
132
123
|
}
|
|
133
|
-
|
|
124
|
+
entityFilters$1.delete(filter);
|
|
134
125
|
});
|
|
135
126
|
}
|
|
136
127
|
|
|
@@ -181,26 +172,14 @@ function getUnsafe(inMemTable) {
|
|
|
181
172
|
return key => mapChangeToEntity(inMemTable.latestEntityChangeById[key]);
|
|
182
173
|
}
|
|
183
174
|
|
|
184
|
-
function hasIndex(inMemTable
|
|
185
|
-
return
|
|
186
|
-
let indicesSerializedToValue = inMemTable.fieldNameIndices[fieldName];
|
|
187
|
-
if (indicesSerializedToValue === undefined) {
|
|
188
|
-
return false;
|
|
189
|
-
}
|
|
190
|
-
let key = TableIndices.Index.toStringByParts(fieldName, operator, fieldValueHash);
|
|
191
|
-
return indicesSerializedToValue[key] !== undefined;
|
|
192
|
-
};
|
|
175
|
+
function hasIndex(inMemTable) {
|
|
176
|
+
return filterKey => inMemTable.filterIndices[filterKey] !== undefined;
|
|
193
177
|
}
|
|
194
178
|
|
|
195
|
-
function getUnsafeOnIndex(inMemTable
|
|
179
|
+
function getUnsafeOnIndex(inMemTable) {
|
|
196
180
|
let getEntity = getUnsafe(inMemTable);
|
|
197
|
-
return
|
|
198
|
-
let
|
|
199
|
-
if (indicesSerializedToValue === undefined) {
|
|
200
|
-
return Stdlib_JsError.throwWithMessage(`Unexpected error. Must have an index on field ` + fieldName);
|
|
201
|
-
}
|
|
202
|
-
let key = TableIndices.Index.toStringByParts(fieldName, operator, fieldValueHash);
|
|
203
|
-
let match = indicesSerializedToValue[key];
|
|
181
|
+
return filterKey => {
|
|
182
|
+
let match = inMemTable.filterIndices[filterKey];
|
|
204
183
|
if (match !== undefined) {
|
|
205
184
|
return Stdlib_Array.filterMap(Array.from(match[1]), entityId => {
|
|
206
185
|
if (entityId in inMemTable.latestEntityChangeById) {
|
|
@@ -208,48 +187,40 @@ function getUnsafeOnIndex(inMemTable, fieldName, operator) {
|
|
|
208
187
|
}
|
|
209
188
|
});
|
|
210
189
|
} else {
|
|
211
|
-
return Stdlib_JsError.throwWithMessage(`Unexpected error. Must have an index for the
|
|
190
|
+
return Stdlib_JsError.throwWithMessage(`Unexpected error. Must have an index for the filter ` + filterKey);
|
|
212
191
|
}
|
|
213
192
|
};
|
|
214
193
|
}
|
|
215
194
|
|
|
216
|
-
function addEmptyIndex(inMemTable,
|
|
217
|
-
let
|
|
195
|
+
function addEmptyIndex(inMemTable, filter) {
|
|
196
|
+
let filterKey = EntityFilter.toString(filter);
|
|
197
|
+
let match = inMemTable.filterIndices[filterKey];
|
|
198
|
+
if (match !== undefined) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
218
201
|
let relatedEntityIds = new Set();
|
|
219
202
|
Utils.Dict.forEach(inMemTable.latestEntityChangeById, change => {
|
|
220
203
|
let entity = mapChangeToEntity(change);
|
|
221
204
|
if (entity === undefined) {
|
|
222
205
|
return;
|
|
223
206
|
}
|
|
224
|
-
|
|
225
|
-
if (!TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
|
|
207
|
+
if (!EntityFilter.matches(filter, entity)) {
|
|
226
208
|
return;
|
|
227
209
|
}
|
|
228
210
|
let entityId = getEntityIdUnsafe(entity);
|
|
229
|
-
|
|
211
|
+
getOrCreateEntityFilters(inMemTable, entityId).add(filter);
|
|
230
212
|
relatedEntityIds.add(entityId);
|
|
231
213
|
});
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
return;
|
|
237
|
-
} else {
|
|
238
|
-
indicesSerializedToValue[TableIndices.Index.toString(index)] = [
|
|
239
|
-
index,
|
|
240
|
-
relatedEntityIds
|
|
241
|
-
];
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
inMemTable.fieldNameIndices[fieldName] = makeIndicesSerializedToValue(index, Primitive_option.some(relatedEntityIds));
|
|
214
|
+
inMemTable.filterIndices[filterKey] = [
|
|
215
|
+
filter,
|
|
216
|
+
relatedEntityIds
|
|
217
|
+
];
|
|
246
218
|
}
|
|
247
219
|
|
|
248
220
|
let Entity = {
|
|
249
221
|
UnexpectedIdNotDefinedOnEntity: UnexpectedIdNotDefinedOnEntity,
|
|
250
222
|
getEntityIdUnsafe: getEntityIdUnsafe,
|
|
251
|
-
|
|
252
|
-
makeIndicesSerializedToValue: makeIndicesSerializedToValue,
|
|
223
|
+
getOrCreateEntityFilters: getOrCreateEntityFilters,
|
|
253
224
|
make: make,
|
|
254
225
|
snapshotChanges: snapshotChanges,
|
|
255
226
|
dropCommittedChanges: dropCommittedChanges,
|
package/src/Internal.res
CHANGED
|
@@ -265,10 +265,17 @@ type genericEvent<'params, 'block, 'transaction> = {
|
|
|
265
265
|
block: 'block,
|
|
266
266
|
}
|
|
267
267
|
|
|
268
|
-
|
|
268
|
+
// Opaque internally — block-level values needed by the runtime
|
|
269
|
+
// (blockNumber, timestamp, blockHash) live on the item instead.
|
|
270
|
+
type event
|
|
269
271
|
|
|
270
272
|
external fromGenericEvent: genericEvent<'a, 'b, 'c> => event = "%identity"
|
|
271
273
|
|
|
274
|
+
// Escape hatch for serialization boundaries (raw events, logging)
|
|
275
|
+
// which genuinely need the runtime shape.
|
|
276
|
+
external toGenericEvent: event => genericEvent<eventParams, eventBlock, eventTransaction> =
|
|
277
|
+
"%identity"
|
|
278
|
+
|
|
272
279
|
type genericLoaderArgs<'event, 'context> = {
|
|
273
280
|
event: 'event,
|
|
274
281
|
context: 'context,
|
|
@@ -412,6 +419,50 @@ type evmContractConfig = {
|
|
|
412
419
|
events: array<evmEventConfig>,
|
|
413
420
|
}
|
|
414
421
|
|
|
422
|
+
type svmAccountFilter = {
|
|
423
|
+
position: int,
|
|
424
|
+
values: array<SvmTypes.Pubkey.t>,
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** AND-group: every entry must match the same instruction. */
|
|
428
|
+
type svmAccountFilterGroup = array<svmAccountFilter>
|
|
429
|
+
|
|
430
|
+
type svmInstructionEventConfig = {
|
|
431
|
+
...eventConfig,
|
|
432
|
+
/** Base58 Solana program id this instruction belongs to. */
|
|
433
|
+
programId: SvmTypes.Pubkey.t,
|
|
434
|
+
/** Hex-encoded discriminator. `None` matches every instruction in the program. */
|
|
435
|
+
discriminator: option<string>,
|
|
436
|
+
/** Length of the discriminator in bytes (0 / 1 / 2 / 4 / 8). Drives the
|
|
437
|
+
`dN` selector at query time and the dispatch-key precomputation in the
|
|
438
|
+
router. */
|
|
439
|
+
discriminatorByteLen: int,
|
|
440
|
+
includeTransaction: bool,
|
|
441
|
+
includeLogs: bool,
|
|
442
|
+
includeTokenBalances: bool,
|
|
443
|
+
/** Disjunctive normal form: outer array is OR of AND-groups, inner array is
|
|
444
|
+
AND across positions. Empty outer array means "no account filter". */
|
|
445
|
+
accountFilters: array<svmAccountFilterGroup>,
|
|
446
|
+
/** `None` matches both outer and inner (CPI-invoked) instructions. */
|
|
447
|
+
isInner: option<bool>,
|
|
448
|
+
/** Positional account names from the Borsh schema, in declared order.
|
|
449
|
+
`[]` means no schema is attached for this instruction. */
|
|
450
|
+
accounts: array<string>,
|
|
451
|
+
/** Borsh args layout as `Vec<ArgDef>` JSON (see `human_config::svm::ArgDef`
|
|
452
|
+
on the Rust side). `JSON.Null` means no schema is attached. */
|
|
453
|
+
args: JSON.t,
|
|
454
|
+
/** Program-level nominal-type registry (`BTreeMap<String, ArgType>` JSON).
|
|
455
|
+
Duplicated on every event of the same program — the runtime dedups by
|
|
456
|
+
`programId` when registering. `JSON.Null` when empty. */
|
|
457
|
+
definedTypes: JSON.t,
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
type svmProgramConfig = {
|
|
461
|
+
name: string,
|
|
462
|
+
programId: SvmTypes.Pubkey.t,
|
|
463
|
+
instructions: array<svmInstructionEventConfig>,
|
|
464
|
+
}
|
|
465
|
+
|
|
415
466
|
type indexingAddress = {
|
|
416
467
|
address: Address.t,
|
|
417
468
|
contractName: string,
|
|
@@ -430,6 +481,7 @@ type eventItem = private {
|
|
|
430
481
|
timestamp: int,
|
|
431
482
|
chain: ChainMap.Chain.t,
|
|
432
483
|
blockNumber: int,
|
|
484
|
+
blockHash: string,
|
|
433
485
|
logIndex: int,
|
|
434
486
|
event: event,
|
|
435
487
|
}
|
|
@@ -463,6 +515,7 @@ type item =
|
|
|
463
515
|
timestamp: int,
|
|
464
516
|
chain: ChainMap.Chain.t,
|
|
465
517
|
blockNumber: int,
|
|
518
|
+
blockHash: string,
|
|
466
519
|
logIndex: int,
|
|
467
520
|
event: event,
|
|
468
521
|
})
|
|
@@ -523,7 +576,6 @@ type genericEntityConfig<'entity> = {
|
|
|
523
576
|
name: string,
|
|
524
577
|
index: int,
|
|
525
578
|
schema: S.t<'entity>,
|
|
526
|
-
rowsSchema: S.t<array<'entity>>,
|
|
527
579
|
table: Table.table,
|
|
528
580
|
storage: entityStorage,
|
|
529
581
|
}
|
|
@@ -569,9 +621,6 @@ type effect = {
|
|
|
569
621
|
}
|
|
570
622
|
let cacheTablePrefix = "envio_effect_"
|
|
571
623
|
let cacheOutputSchema = S.json(~validate=false)->(Utils.magic: S.t<JSON.t> => S.t<effectOutput>)
|
|
572
|
-
let effectCacheItemRowsSchema = S.array(
|
|
573
|
-
S.schema(s => {id: s.matches(S.string), output: s.matches(cacheOutputSchema)}),
|
|
574
|
-
)
|
|
575
624
|
let makeCacheTable = (~effectName) => {
|
|
576
625
|
Table.mkTable(
|
|
577
626
|
cacheTablePrefix ++ effectName,
|
package/src/Internal.res.mjs
CHANGED
|
@@ -144,15 +144,10 @@ let cacheTablePrefix = "envio_effect_";
|
|
|
144
144
|
|
|
145
145
|
let cacheOutputSchema = S$RescriptSchema.json(false);
|
|
146
146
|
|
|
147
|
-
let effectCacheItemRowsSchema = S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
|
|
148
|
-
id: s.m(S$RescriptSchema.string),
|
|
149
|
-
output: s.m(cacheOutputSchema)
|
|
150
|
-
})));
|
|
151
|
-
|
|
152
147
|
function makeCacheTable(effectName) {
|
|
153
148
|
return Table.mkTable(cacheTablePrefix + effectName, undefined, [
|
|
154
|
-
Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined),
|
|
155
|
-
Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined)
|
|
149
|
+
Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
|
|
150
|
+
Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined)
|
|
156
151
|
], undefined);
|
|
157
152
|
}
|
|
158
153
|
|
|
@@ -173,7 +168,6 @@ export {
|
|
|
173
168
|
fuelTransferParamsSchema,
|
|
174
169
|
cacheTablePrefix,
|
|
175
170
|
cacheOutputSchema,
|
|
176
|
-
effectCacheItemRowsSchema,
|
|
177
171
|
makeCacheTable,
|
|
178
172
|
loadedFromDbCheckpointId,
|
|
179
173
|
initialCheckpointId,
|
package/src/LazyLoader.res
CHANGED
|
@@ -54,7 +54,7 @@ let deleteKey: (dict<'value>, string) => unit = (_obj, _k) => %raw(`delete _obj[
|
|
|
54
54
|
let timeoutAfter = timeoutMillis =>
|
|
55
55
|
Utils.delay(timeoutMillis)->Promise.then(() =>
|
|
56
56
|
Promise.reject(
|
|
57
|
-
LoaderTimeout(`Query took longer than ${
|
|
57
|
+
LoaderTimeout(`Query took longer than ${Int.toString(timeoutMillis / 1000)} seconds`),
|
|
58
58
|
)
|
|
59
59
|
)
|
|
60
60
|
|
|
@@ -67,7 +67,7 @@ let rec loadNext = async (am: asyncMap<'key, 'value>, k: 'key) => {
|
|
|
67
67
|
// Resolve the external promise
|
|
68
68
|
am.resolvers
|
|
69
69
|
->Utils.Map.get(k)
|
|
70
|
-
->
|
|
70
|
+
->Option.forEach(r => {
|
|
71
71
|
let _ = am.resolvers->Utils.Map.delete(k)
|
|
72
72
|
r(val)
|
|
73
73
|
})
|
package/src/LazyLoader.res.mjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
4
|
import * as JsSdsl from "js-sdsl";
|
|
5
|
-
import * as
|
|
5
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
6
6
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
7
7
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
8
8
|
|
|
@@ -35,7 +35,7 @@ function deleteKey(_obj, _k) {
|
|
|
35
35
|
function timeoutAfter(timeoutMillis) {
|
|
36
36
|
return Utils.delay(timeoutMillis).then(() => Promise.reject({
|
|
37
37
|
RE_EXN_ID: LoaderTimeout,
|
|
38
|
-
_1: `Query took longer than ` +
|
|
38
|
+
_1: `Query took longer than ` + (timeoutMillis / 1000 | 0).toString() + ` seconds`
|
|
39
39
|
}));
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -46,7 +46,7 @@ async function loadNext(am, k) {
|
|
|
46
46
|
am.loaderFn(k),
|
|
47
47
|
timeoutAfter(am._timeoutMillis)
|
|
48
48
|
]);
|
|
49
|
-
|
|
49
|
+
Stdlib_Option.forEach(am.resolvers.get(k), r => {
|
|
50
50
|
am.resolvers.delete(k);
|
|
51
51
|
r(val);
|
|
52
52
|
});
|
package/src/LoadLayer.res
CHANGED
|
@@ -17,11 +17,15 @@ let loadById = (
|
|
|
17
17
|
// Since LoadManager.call prevents registerign entities already existing in the inMemoryStore,
|
|
18
18
|
// we can be sure that we load only the new ones.
|
|
19
19
|
let dbEntities = try {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
(
|
|
21
|
+
await storage.loadOrThrow(
|
|
22
|
+
~table=entityConfig.table,
|
|
23
|
+
~filter=EntityFilter.In({
|
|
24
|
+
fieldName: Table.idFieldName,
|
|
25
|
+
fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
|
|
26
|
+
}),
|
|
27
|
+
)
|
|
28
|
+
)->(Utils.magic: array<unknown> => array<Internal.entity>)
|
|
25
29
|
} catch {
|
|
26
30
|
| Persistence.StorageError({message, reason}) =>
|
|
27
31
|
reason->ErrorHandling.mkLogAndRaise(~logger=item->Logging.getItemLogger, ~msg=message)
|
|
@@ -161,8 +165,8 @@ let rec executeWithRateLimit = (
|
|
|
161
165
|
|
|
162
166
|
// Split into immediate and queued
|
|
163
167
|
let immediateCount = Math.Int.min(state.availableCalls, effectArgs->Array.length)
|
|
164
|
-
let immediateArgs = effectArgs->
|
|
165
|
-
let queuedArgs = effectArgs->
|
|
168
|
+
let immediateArgs = effectArgs->Array.slice(~start=0, ~end=immediateCount)
|
|
169
|
+
let queuedArgs = effectArgs->Array.slice(~start=immediateCount)
|
|
166
170
|
|
|
167
171
|
// Update available calls
|
|
168
172
|
state.availableCalls = state.availableCalls - immediateCount
|
|
@@ -264,11 +268,15 @@ let loadEffect = (
|
|
|
264
268
|
let {table, outputSchema} = effect.storageMeta
|
|
265
269
|
|
|
266
270
|
let dbEntities = try {
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
271
|
+
(
|
|
272
|
+
await storage.loadOrThrow(
|
|
273
|
+
~table,
|
|
274
|
+
~filter=EntityFilter.In({
|
|
275
|
+
fieldName: Table.idFieldName,
|
|
276
|
+
fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
|
|
277
|
+
}),
|
|
278
|
+
)
|
|
279
|
+
)->(Utils.magic: array<unknown> => array<Internal.effectCacheItem>)
|
|
272
280
|
} catch {
|
|
273
281
|
| exn =>
|
|
274
282
|
item
|
|
@@ -342,60 +350,38 @@ let loadEffect = (
|
|
|
342
350
|
)
|
|
343
351
|
}
|
|
344
352
|
|
|
345
|
-
let
|
|
353
|
+
let loadByFilter = (
|
|
346
354
|
~loadManager,
|
|
347
355
|
~persistence: Persistence.t,
|
|
348
|
-
~operator: TableIndices.Operator.t,
|
|
349
356
|
~entityConfig: Internal.entityConfig,
|
|
350
357
|
~inMemoryStore,
|
|
351
|
-
~fieldName,
|
|
352
|
-
~fieldValueSchema,
|
|
353
358
|
~shouldGroup,
|
|
354
359
|
~item,
|
|
355
|
-
~
|
|
360
|
+
~filter: EntityFilter.t,
|
|
356
361
|
) => {
|
|
357
|
-
let
|
|
358
|
-
| Eq => "eq"
|
|
359
|
-
| Gt => "gt"
|
|
360
|
-
| Lt => "lt"
|
|
361
|
-
}
|
|
362
|
-
let key = `${entityConfig.name}.getWhere.${fieldName}.${operatorCallName}`
|
|
362
|
+
let key = filter->EntityFilter.toOperationKey(~entityName=entityConfig.name)
|
|
363
363
|
let inMemTable = inMemoryStore->InMemoryStore.getInMemTable(~entityConfig)
|
|
364
364
|
|
|
365
|
-
let load = async (
|
|
365
|
+
let load = async (filters: array<EntityFilter.t>, ~onError as _) => {
|
|
366
366
|
let storage = persistence->Persistence.getInitializedStorageOrThrow
|
|
367
367
|
let timerRef = Prometheus.StorageLoad.startOperation(~storage=storage.name, ~operation=key)
|
|
368
368
|
|
|
369
369
|
let size = ref(0)
|
|
370
370
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
})
|
|
371
|
+
filters->Array.forEach(filter => inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter))
|
|
372
|
+
|
|
373
|
+
// Loading a superset of rows via a merged query is safe: every loaded
|
|
374
|
+
// entity is matched against all registered indices, not only the
|
|
375
|
+
// query's own filter.
|
|
376
|
+
let queries = filters->EntityFilter.merge
|
|
378
377
|
|
|
379
|
-
let _ = await
|
|
380
|
-
->Array.map(async
|
|
381
|
-
inMemTable->InMemoryTable.Entity.addEmptyIndex(~index)
|
|
378
|
+
let _ = await queries
|
|
379
|
+
->Array.map(async filter => {
|
|
382
380
|
try {
|
|
383
|
-
let entities =
|
|
384
|
-
~
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
| Single({operator: Lt}) => #"<"
|
|
388
|
-
},
|
|
389
|
-
~table=entityConfig.table,
|
|
390
|
-
~rowsSchema=entityConfig.rowsSchema,
|
|
391
|
-
~fieldName=index->TableIndices.Index.getFieldName,
|
|
392
|
-
~fieldValue=switch index {
|
|
393
|
-
| Single({fieldValue}) => fieldValue
|
|
394
|
-
},
|
|
395
|
-
~fieldSchema=fieldValueSchema->(
|
|
396
|
-
Utils.magic: S.t<'fieldValue> => S.t<TableIndices.FieldValue.t>
|
|
397
|
-
),
|
|
398
|
-
)
|
|
381
|
+
let entities =
|
|
382
|
+
(await storage.loadOrThrow(~table=entityConfig.table, ~filter))->(
|
|
383
|
+
Utils.magic: array<unknown> => array<Internal.entity>
|
|
384
|
+
)
|
|
399
385
|
|
|
400
386
|
entities->Array.forEach(entity => {
|
|
401
387
|
inMemTable->InMemoryTable.Entity.initValue(
|
|
@@ -411,11 +397,13 @@ let loadByField = (
|
|
|
411
397
|
reason->ErrorHandling.mkLogAndRaise(
|
|
412
398
|
~logger=Logging.createChildFrom(
|
|
413
399
|
~logger=item->Logging.getItemLogger,
|
|
400
|
+
// The executed query might be merged from multiple getWhere
|
|
401
|
+
// calls, so report it as the operation users write with the
|
|
402
|
+
// values bound to its placeholders, instead of an internal
|
|
403
|
+
// filter representation they never constructed.
|
|
414
404
|
~params={
|
|
415
|
-
"
|
|
416
|
-
"
|
|
417
|
-
"fieldName": fieldName,
|
|
418
|
-
"fieldValue": fieldValue,
|
|
405
|
+
"operation": key,
|
|
406
|
+
"params": filter->EntityFilter.getParams,
|
|
419
407
|
},
|
|
420
408
|
),
|
|
421
409
|
~msg=message,
|
|
@@ -427,7 +415,7 @@ let loadByField = (
|
|
|
427
415
|
timerRef->Prometheus.StorageLoad.endOperation(
|
|
428
416
|
~storage=storage.name,
|
|
429
417
|
~operation=key,
|
|
430
|
-
~whereSize=
|
|
418
|
+
~whereSize=queries->Array.reduce(0, (acc, query) => acc + query->EntityFilter.valuesCount),
|
|
431
419
|
~size=size.contents,
|
|
432
420
|
)
|
|
433
421
|
}
|
|
@@ -435,11 +423,10 @@ let loadByField = (
|
|
|
435
423
|
loadManager->LoadManager.call(
|
|
436
424
|
~key,
|
|
437
425
|
~load,
|
|
438
|
-
~input=
|
|
426
|
+
~input=filter,
|
|
439
427
|
~shouldGroup,
|
|
440
|
-
~hasher=
|
|
441
|
-
|
|
442
|
-
~
|
|
443
|
-
~hasInMemory=inMemTable->InMemoryTable.Entity.hasIndex(~fieldName, ~operator),
|
|
428
|
+
~hasher=EntityFilter.toString,
|
|
429
|
+
~getUnsafeInMemory=inMemTable->InMemoryTable.Entity.getUnsafeOnIndex,
|
|
430
|
+
~hasInMemory=inMemTable->InMemoryTable.Entity.hasIndex,
|
|
444
431
|
)
|
|
445
432
|
}
|