envio 3.1.2 → 3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/evm.schema.json +83 -11
  2. package/fuel.schema.json +83 -11
  3. package/index.d.ts +184 -3
  4. package/package.json +6 -6
  5. package/src/Batch.res +2 -2
  6. package/src/ChainFetcher.res +27 -3
  7. package/src/ChainFetcher.res.mjs +17 -3
  8. package/src/ChainManager.res +163 -0
  9. package/src/ChainManager.res.mjs +136 -0
  10. package/src/Config.res +213 -30
  11. package/src/Config.res.mjs +102 -41
  12. package/src/Core.res +16 -10
  13. package/src/Ecosystem.res +0 -3
  14. package/src/Env.res +2 -2
  15. package/src/Env.res.mjs +2 -2
  16. package/src/Envio.res +101 -2
  17. package/src/Envio.res.mjs +2 -3
  18. package/src/EventConfigBuilder.res +87 -0
  19. package/src/EventConfigBuilder.res.mjs +53 -0
  20. package/src/EventUtils.res +2 -2
  21. package/src/FetchState.res +63 -67
  22. package/src/FetchState.res.mjs +44 -42
  23. package/src/GlobalState.res +219 -363
  24. package/src/GlobalState.res.mjs +314 -491
  25. package/src/GlobalStateManager.res +49 -59
  26. package/src/GlobalStateManager.res.mjs +5 -4
  27. package/src/GlobalStateManager.resi +1 -1
  28. package/src/HandlerLoader.res +18 -2
  29. package/src/HandlerLoader.res.mjs +16 -34
  30. package/src/HandlerRegister.res +9 -9
  31. package/src/HandlerRegister.res.mjs +9 -9
  32. package/src/Hasura.res +102 -32
  33. package/src/Hasura.res.mjs +88 -34
  34. package/src/InMemoryStore.res +10 -1
  35. package/src/InMemoryStore.res.mjs +4 -1
  36. package/src/InMemoryTable.res +83 -136
  37. package/src/InMemoryTable.res.mjs +57 -86
  38. package/src/Internal.res +70 -5
  39. package/src/Internal.res.mjs +2 -8
  40. package/src/LazyLoader.res +2 -2
  41. package/src/LazyLoader.res.mjs +3 -3
  42. package/src/LoadLayer.res +47 -60
  43. package/src/LoadLayer.res.mjs +28 -50
  44. package/src/LoadLayer.resi +2 -5
  45. package/src/LogSelection.res +90 -21
  46. package/src/LogSelection.res.mjs +72 -21
  47. package/src/Logging.res +1 -1
  48. package/src/Main.res +61 -2
  49. package/src/Main.res.mjs +37 -1
  50. package/src/Persistence.res +3 -16
  51. package/src/PgStorage.res +125 -114
  52. package/src/PgStorage.res.mjs +112 -95
  53. package/src/Ports.res +5 -0
  54. package/src/Ports.res.mjs +9 -0
  55. package/src/Prometheus.res +3 -3
  56. package/src/Prometheus.res.mjs +4 -4
  57. package/src/ReorgDetection.res +4 -4
  58. package/src/ReorgDetection.res.mjs +4 -5
  59. package/src/SafeCheckpointTracking.res +16 -16
  60. package/src/SafeCheckpointTracking.res.mjs +2 -2
  61. package/src/SimulateItems.res +10 -14
  62. package/src/SimulateItems.res.mjs +5 -2
  63. package/src/Sink.res +1 -1
  64. package/src/Sink.res.mjs +1 -2
  65. package/src/SvmTypes.res +9 -0
  66. package/src/SvmTypes.res.mjs +14 -0
  67. package/src/TestIndexer.res +35 -68
  68. package/src/TestIndexer.res.mjs +17 -48
  69. package/src/TestIndexerProxyStorage.res +23 -23
  70. package/src/TestIndexerProxyStorage.res.mjs +12 -15
  71. package/src/Throttler.res +2 -2
  72. package/src/Time.res +2 -2
  73. package/src/Time.res.mjs +2 -2
  74. package/src/UserContext.res +19 -118
  75. package/src/UserContext.res.mjs +10 -66
  76. package/src/Utils.res +15 -15
  77. package/src/Utils.res.mjs +7 -8
  78. package/src/adapters/MarkBatchProcessedAdapter.res +5 -0
  79. package/src/adapters/MarkBatchProcessedAdapter.res.mjs +14 -0
  80. package/src/bindings/BigDecimal.res +1 -1
  81. package/src/bindings/BigDecimal.res.mjs +2 -2
  82. package/src/bindings/ClickHouse.res +8 -6
  83. package/src/bindings/ClickHouse.res.mjs +5 -5
  84. package/src/bindings/Hrtime.res +1 -1
  85. package/src/bindings/Pino.res +2 -2
  86. package/src/bindings/Pino.res.mjs +3 -4
  87. package/src/db/EntityFilter.res +410 -0
  88. package/src/db/EntityFilter.res.mjs +424 -0
  89. package/src/db/EntityHistory.res +1 -1
  90. package/src/db/EntityHistory.res.mjs +1 -1
  91. package/src/db/InternalTable.res +10 -10
  92. package/src/db/InternalTable.res.mjs +41 -45
  93. package/src/db/Schema.res +2 -2
  94. package/src/db/Schema.res.mjs +3 -3
  95. package/src/db/Table.res +106 -22
  96. package/src/db/Table.res.mjs +84 -35
  97. package/src/sources/EventRouter.res +67 -2
  98. package/src/sources/EventRouter.res.mjs +45 -3
  99. package/src/sources/Evm.res +0 -7
  100. package/src/sources/Evm.res.mjs +0 -15
  101. package/src/sources/EvmChain.res +1 -1
  102. package/src/sources/EvmChain.res.mjs +1 -2
  103. package/src/sources/EvmRpcClient.res +42 -0
  104. package/src/sources/EvmRpcClient.res.mjs +64 -0
  105. package/src/sources/Fuel.res +0 -7
  106. package/src/sources/Fuel.res.mjs +0 -15
  107. package/src/sources/HyperFuelSource.res +5 -4
  108. package/src/sources/HyperFuelSource.res.mjs +2 -2
  109. package/src/sources/HyperSyncClient.res +9 -5
  110. package/src/sources/HyperSyncClient.res.mjs +2 -2
  111. package/src/sources/HyperSyncHeightStream.res +2 -2
  112. package/src/sources/HyperSyncHeightStream.res.mjs +2 -2
  113. package/src/sources/HyperSyncSource.res +12 -11
  114. package/src/sources/HyperSyncSource.res.mjs +6 -6
  115. package/src/sources/Rpc.res +1 -5
  116. package/src/sources/Rpc.res.mjs +1 -9
  117. package/src/sources/RpcSource.res +57 -21
  118. package/src/sources/RpcSource.res.mjs +47 -20
  119. package/src/sources/RpcWebSocketHeightStream.res +1 -1
  120. package/src/sources/SourceManager.res +3 -2
  121. package/src/sources/SourceManager.res.mjs +1 -1
  122. package/src/sources/Svm.res +3 -10
  123. package/src/sources/Svm.res.mjs +4 -18
  124. package/src/sources/SvmHyperSyncClient.res +265 -0
  125. package/src/sources/SvmHyperSyncClient.res.mjs +28 -0
  126. package/src/sources/SvmHyperSyncSource.res +638 -0
  127. package/src/sources/SvmHyperSyncSource.res.mjs +557 -0
  128. package/src/tui/Tui.res +9 -2
  129. package/src/tui/Tui.res.mjs +18 -3
  130. package/src/tui/components/BufferedProgressBar.res +2 -2
  131. package/src/tui/components/TuiData.res +3 -0
  132. package/svm.schema.json +523 -14
  133. package/src/TableIndices.res +0 -115
  134. package/src/TableIndices.res.mjs +0 -144
@@ -2,8 +2,8 @@
2
2
 
3
3
  import * as Utils from "./Utils.res.mjs";
4
4
  import * as Internal from "./Internal.res.mjs";
5
+ import * as EntityFilter from "./db/EntityFilter.res.mjs";
5
6
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
6
- import * as TableIndices from "./TableIndices.res.mjs";
7
7
  import * as ErrorHandling from "./ErrorHandling.res.mjs";
8
8
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
9
9
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
@@ -23,33 +23,23 @@ function getEntityIdUnsafe(entity) {
23
23
  }
24
24
  }
25
25
 
26
- function getOrCreateEntityIndices(self, entityId) {
27
- let s = self.indicesByEntityId[entityId];
26
+ function getOrCreateEntityFilters(self, entityId) {
27
+ let s = self.filtersByEntityId[entityId];
28
28
  if (s !== undefined) {
29
29
  return Primitive_option.valFromOption(s);
30
30
  }
31
31
  let s$1 = new Set();
32
- self.indicesByEntityId[entityId] = s$1;
32
+ self.filtersByEntityId[entityId] = s$1;
33
33
  return s$1;
34
34
  }
35
35
 
36
- function makeIndicesSerializedToValue(index, relatedEntityIdsOpt) {
37
- let relatedEntityIds = relatedEntityIdsOpt !== undefined ? Primitive_option.valFromOption(relatedEntityIdsOpt) : new Set();
38
- let empty = {};
39
- empty[TableIndices.Index.toString(index)] = [
40
- index,
41
- relatedEntityIds
42
- ];
43
- return empty;
44
- }
45
-
46
36
  function make() {
47
37
  return {
48
38
  latestEntityChangeById: {},
49
39
  changesCount: 0,
50
40
  prevEntityChanges: [],
51
- indicesByEntityId: {},
52
- fieldNameIndices: {}
41
+ filtersByEntityId: {},
42
+ filterIndices: {}
53
43
  };
54
44
  }
55
45
 
@@ -57,14 +47,20 @@ function snapshotChanges(self, committedCheckpointId, upToCheckpointId) {
57
47
  let changes = [];
58
48
  let keptPrev = [];
59
49
  self.prevEntityChanges.forEach(change => {
60
- if (change.checkpointId > upToCheckpointId) {
50
+ let checkpointId = change.checkpointId;
51
+ if (checkpointId > upToCheckpointId) {
61
52
  keptPrev.push(change);
62
- } else {
53
+ return;
54
+ } else if (checkpointId > committedCheckpointId) {
63
55
  changes.push(change);
56
+ return;
57
+ } else {
58
+ return;
64
59
  }
65
60
  });
61
+ let removedCount = self.prevEntityChanges.length - keptPrev.length | 0;
66
62
  self.prevEntityChanges = keptPrev;
67
- self.changesCount = self.changesCount - changes.length;
63
+ self.changesCount = self.changesCount - removedCount;
68
64
  Utils.Dict.forEach(self.latestEntityChangeById, change => {
69
65
  let checkpointId = change.checkpointId;
70
66
  if (checkpointId > committedCheckpointId && checkpointId <= upToCheckpointId) {
@@ -86,51 +82,46 @@ function dropCommittedChanges(self, committedCheckpointId, keepLoadedFromDb) {
86
82
  });
87
83
  keysToDelete.forEach(key => Utils.Dict.deleteInPlace(self.latestEntityChangeById, key));
88
84
  self.changesCount = self.changesCount - keysToDelete.length;
89
- self.indicesByEntityId = {};
90
- self.fieldNameIndices = {};
85
+ self.filtersByEntityId = {};
86
+ self.filterIndices = {};
91
87
  }
92
88
 
93
89
  function updateIndices(self, entity) {
94
90
  let entityId = getEntityIdUnsafe(entity);
95
- let entityIndices = self.indicesByEntityId[entityId];
96
- if (entityIndices !== undefined) {
97
- let entityIndices$1 = Primitive_option.valFromOption(entityIndices);
98
- entityIndices$1.forEach(index => {
99
- let fieldName = TableIndices.Index.getFieldName(index);
100
- let fieldValue = entity[fieldName];
101
- if (!TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
102
- entityIndices$1.delete(index);
91
+ let entityFilters = self.filtersByEntityId[entityId];
92
+ if (entityFilters !== undefined) {
93
+ let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
94
+ entityFilters$1.forEach(filter => {
95
+ if (!EntityFilter.matches(filter, entity)) {
96
+ entityFilters$1.delete(filter);
103
97
  return;
104
98
  }
105
99
  });
106
100
  }
107
- Utils.Dict.forEachWithKey(self.fieldNameIndices, (indices, fieldName) => {
108
- let fieldValue = entity[fieldName];
109
- Utils.Dict.forEach(indices, param => {
110
- let relatedEntityIds = param[1];
111
- let index = param[0];
112
- if (TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
113
- relatedEntityIds.add(entityId);
114
- getOrCreateEntityIndices(self, entityId).add(index);
115
- } else {
116
- relatedEntityIds.delete(entityId);
117
- }
118
- });
101
+ Utils.Dict.forEach(self.filterIndices, param => {
102
+ let relatedEntityIds = param[1];
103
+ let filter = param[0];
104
+ if (EntityFilter.matches(filter, entity)) {
105
+ relatedEntityIds.add(entityId);
106
+ getOrCreateEntityFilters(self, entityId).add(filter);
107
+ } else {
108
+ relatedEntityIds.delete(entityId);
109
+ }
119
110
  });
120
111
  }
121
112
 
122
113
  function deleteEntityFromIndices(self, entityId) {
123
- let entityIndices = self.indicesByEntityId[entityId];
124
- if (entityIndices === undefined) {
114
+ let entityFilters = self.filtersByEntityId[entityId];
115
+ if (entityFilters === undefined) {
125
116
  return;
126
117
  }
127
- let entityIndices$1 = Primitive_option.valFromOption(entityIndices);
128
- entityIndices$1.forEach(index => {
129
- let match = Stdlib_Option.flatMap(self.fieldNameIndices[TableIndices.Index.getFieldName(index)], indices => indices[TableIndices.Index.toString(index)]);
118
+ let entityFilters$1 = Primitive_option.valFromOption(entityFilters);
119
+ entityFilters$1.forEach(filter => {
120
+ let match = self.filterIndices[EntityFilter.toString(filter)];
130
121
  if (match !== undefined) {
131
122
  match[1].delete(entityId);
132
123
  }
133
- entityIndices$1.delete(index);
124
+ entityFilters$1.delete(filter);
134
125
  });
135
126
  }
136
127
 
@@ -181,26 +172,14 @@ function getUnsafe(inMemTable) {
181
172
  return key => mapChangeToEntity(inMemTable.latestEntityChangeById[key]);
182
173
  }
183
174
 
184
- function hasIndex(inMemTable, fieldName, operator) {
185
- return fieldValueHash => {
186
- let indicesSerializedToValue = inMemTable.fieldNameIndices[fieldName];
187
- if (indicesSerializedToValue === undefined) {
188
- return false;
189
- }
190
- let key = TableIndices.Index.toStringByParts(fieldName, operator, fieldValueHash);
191
- return indicesSerializedToValue[key] !== undefined;
192
- };
175
+ function hasIndex(inMemTable) {
176
+ return filterKey => inMemTable.filterIndices[filterKey] !== undefined;
193
177
  }
194
178
 
195
- function getUnsafeOnIndex(inMemTable, fieldName, operator) {
179
+ function getUnsafeOnIndex(inMemTable) {
196
180
  let getEntity = getUnsafe(inMemTable);
197
- return fieldValueHash => {
198
- let indicesSerializedToValue = inMemTable.fieldNameIndices[fieldName];
199
- if (indicesSerializedToValue === undefined) {
200
- return Stdlib_JsError.throwWithMessage(`Unexpected error. Must have an index on field ` + fieldName);
201
- }
202
- let key = TableIndices.Index.toStringByParts(fieldName, operator, fieldValueHash);
203
- let match = indicesSerializedToValue[key];
181
+ return filterKey => {
182
+ let match = inMemTable.filterIndices[filterKey];
204
183
  if (match !== undefined) {
205
184
  return Stdlib_Array.filterMap(Array.from(match[1]), entityId => {
206
185
  if (entityId in inMemTable.latestEntityChangeById) {
@@ -208,48 +187,40 @@ function getUnsafeOnIndex(inMemTable, fieldName, operator) {
208
187
  }
209
188
  });
210
189
  } else {
211
- return Stdlib_JsError.throwWithMessage(`Unexpected error. Must have an index for the value ` + fieldValueHash + ` on field ` + fieldName);
190
+ return Stdlib_JsError.throwWithMessage(`Unexpected error. Must have an index for the filter ` + filterKey);
212
191
  }
213
192
  };
214
193
  }
215
194
 
216
- function addEmptyIndex(inMemTable, index) {
217
- let fieldName = TableIndices.Index.getFieldName(index);
195
+ function addEmptyIndex(inMemTable, filter) {
196
+ let filterKey = EntityFilter.toString(filter);
197
+ let match = inMemTable.filterIndices[filterKey];
198
+ if (match !== undefined) {
199
+ return;
200
+ }
218
201
  let relatedEntityIds = new Set();
219
202
  Utils.Dict.forEach(inMemTable.latestEntityChangeById, change => {
220
203
  let entity = mapChangeToEntity(change);
221
204
  if (entity === undefined) {
222
205
  return;
223
206
  }
224
- let fieldValue = entity[fieldName];
225
- if (!TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
207
+ if (!EntityFilter.matches(filter, entity)) {
226
208
  return;
227
209
  }
228
210
  let entityId = getEntityIdUnsafe(entity);
229
- getOrCreateEntityIndices(inMemTable, entityId).add(index);
211
+ getOrCreateEntityFilters(inMemTable, entityId).add(filter);
230
212
  relatedEntityIds.add(entityId);
231
213
  });
232
- let indicesSerializedToValue = inMemTable.fieldNameIndices[fieldName];
233
- if (indicesSerializedToValue !== undefined) {
234
- let match = indicesSerializedToValue[TableIndices.Index.toString(index)];
235
- if (match !== undefined) {
236
- return;
237
- } else {
238
- indicesSerializedToValue[TableIndices.Index.toString(index)] = [
239
- index,
240
- relatedEntityIds
241
- ];
242
- return;
243
- }
244
- }
245
- inMemTable.fieldNameIndices[fieldName] = makeIndicesSerializedToValue(index, Primitive_option.some(relatedEntityIds));
214
+ inMemTable.filterIndices[filterKey] = [
215
+ filter,
216
+ relatedEntityIds
217
+ ];
246
218
  }
247
219
 
248
220
  let Entity = {
249
221
  UnexpectedIdNotDefinedOnEntity: UnexpectedIdNotDefinedOnEntity,
250
222
  getEntityIdUnsafe: getEntityIdUnsafe,
251
- getOrCreateEntityIndices: getOrCreateEntityIndices,
252
- makeIndicesSerializedToValue: makeIndicesSerializedToValue,
223
+ getOrCreateEntityFilters: getOrCreateEntityFilters,
253
224
  make: make,
254
225
  snapshotChanges: snapshotChanges,
255
226
  dropCommittedChanges: dropCommittedChanges,
package/src/Internal.res CHANGED
@@ -265,10 +265,17 @@ type genericEvent<'params, 'block, 'transaction> = {
265
265
  block: 'block,
266
266
  }
267
267
 
268
- type event = genericEvent<eventParams, eventBlock, eventTransaction>
268
+ // Opaque internally block-level values needed by the runtime
269
+ // (blockNumber, timestamp, blockHash) live on the item instead.
270
+ type event
269
271
 
270
272
  external fromGenericEvent: genericEvent<'a, 'b, 'c> => event = "%identity"
271
273
 
274
+ // Escape hatch for serialization boundaries (raw events, logging)
275
+ // which genuinely need the runtime shape.
276
+ external toGenericEvent: event => genericEvent<eventParams, eventBlock, eventTransaction> =
277
+ "%identity"
278
+
272
279
  type genericLoaderArgs<'event, 'context> = {
273
280
  event: 'event,
274
281
  context: 'context,
@@ -337,6 +344,16 @@ type rec paramMeta = {
337
344
  components?: array<paramMeta>,
338
345
  }
339
346
 
347
+ // Fetch-state registry value for an indexed contract address.
348
+ // `effectiveStartBlock` is derived from the registration block and the
349
+ // contract's configured start block (see `FetchState.deriveEffectiveStartBlock`).
350
+ type indexingContract = {
351
+ address: Address.t,
352
+ contractName: string,
353
+ registrationBlock: int,
354
+ effectiveStartBlock: int,
355
+ }
356
+
340
357
  // This is private so it's not manually constructed internally
341
358
  // The idea is that it can only be coerced from fuel/evmEventConfig
342
359
  // and it can include their fields. We prevent manual creation,
@@ -351,6 +368,12 @@ type eventConfig = private {
351
368
  // Usually always false for wildcard events
352
369
  // But might be true for wildcard event with dynamic event filter by addresses
353
370
  dependsOnAddresses: bool,
371
+ // Precompiled predicate (EVM only) for events that filter an indexed address
372
+ // param by registered addresses. Given the decoded event and the log's block
373
+ // number, drops an event whose param-address isn't registered at/before that
374
+ // block — the param-level analogue of EventRouter's srcAddress
375
+ // `effectiveStartBlock` check. Absent otherwise.
376
+ clientAddressFilter?: (event, int, dict<indexingContract>) => bool,
354
377
  handler: option<handler>,
355
378
  contractRegister: option<contractRegister>,
356
379
  paramsRawEventSchema: S.schema<eventParams>,
@@ -412,6 +435,50 @@ type evmContractConfig = {
412
435
  events: array<evmEventConfig>,
413
436
  }
414
437
 
438
+ type svmAccountFilter = {
439
+ position: int,
440
+ values: array<SvmTypes.Pubkey.t>,
441
+ }
442
+
443
+ /** AND-group: every entry must match the same instruction. */
444
+ type svmAccountFilterGroup = array<svmAccountFilter>
445
+
446
+ type svmInstructionEventConfig = {
447
+ ...eventConfig,
448
+ /** Base58 Solana program id this instruction belongs to. */
449
+ programId: SvmTypes.Pubkey.t,
450
+ /** Hex-encoded discriminator. `None` matches every instruction in the program. */
451
+ discriminator: option<string>,
452
+ /** Length of the discriminator in bytes (0 / 1 / 2 / 4 / 8). Drives the
453
+ `dN` selector at query time and the dispatch-key precomputation in the
454
+ router. */
455
+ discriminatorByteLen: int,
456
+ includeTransaction: bool,
457
+ includeLogs: bool,
458
+ includeTokenBalances: bool,
459
+ /** Disjunctive normal form: outer array is OR of AND-groups, inner array is
460
+ AND across positions. Empty outer array means "no account filter". */
461
+ accountFilters: array<svmAccountFilterGroup>,
462
+ /** `None` matches both outer and inner (CPI-invoked) instructions. */
463
+ isInner: option<bool>,
464
+ /** Positional account names from the Borsh schema, in declared order.
465
+ `[]` means no schema is attached for this instruction. */
466
+ accounts: array<string>,
467
+ /** Borsh args layout as `Vec<ArgDef>` JSON (see `human_config::svm::ArgDef`
468
+ on the Rust side). `JSON.Null` means no schema is attached. */
469
+ args: JSON.t,
470
+ /** Program-level nominal-type registry (`BTreeMap<String, ArgType>` JSON).
471
+ Duplicated on every event of the same program — the runtime dedups by
472
+ `programId` when registering. `JSON.Null` when empty. */
473
+ definedTypes: JSON.t,
474
+ }
475
+
476
+ type svmProgramConfig = {
477
+ name: string,
478
+ programId: SvmTypes.Pubkey.t,
479
+ instructions: array<svmInstructionEventConfig>,
480
+ }
481
+
415
482
  type indexingAddress = {
416
483
  address: Address.t,
417
484
  contractName: string,
@@ -430,6 +497,7 @@ type eventItem = private {
430
497
  timestamp: int,
431
498
  chain: ChainMap.Chain.t,
432
499
  blockNumber: int,
500
+ blockHash: string,
433
501
  logIndex: int,
434
502
  event: event,
435
503
  }
@@ -463,6 +531,7 @@ type item =
463
531
  timestamp: int,
464
532
  chain: ChainMap.Chain.t,
465
533
  blockNumber: int,
534
+ blockHash: string,
466
535
  logIndex: int,
467
536
  event: event,
468
537
  })
@@ -523,7 +592,6 @@ type genericEntityConfig<'entity> = {
523
592
  name: string,
524
593
  index: int,
525
594
  schema: S.t<'entity>,
526
- rowsSchema: S.t<array<'entity>>,
527
595
  table: Table.table,
528
596
  storage: entityStorage,
529
597
  }
@@ -569,9 +637,6 @@ type effect = {
569
637
  }
570
638
  let cacheTablePrefix = "envio_effect_"
571
639
  let cacheOutputSchema = S.json(~validate=false)->(Utils.magic: S.t<JSON.t> => S.t<effectOutput>)
572
- let effectCacheItemRowsSchema = S.array(
573
- S.schema(s => {id: s.matches(S.string), output: s.matches(cacheOutputSchema)}),
574
- )
575
640
  let makeCacheTable = (~effectName) => {
576
641
  Table.mkTable(
577
642
  cacheTablePrefix ++ effectName,
@@ -144,15 +144,10 @@ let cacheTablePrefix = "envio_effect_";
144
144
 
145
145
  let cacheOutputSchema = S$RescriptSchema.json(false);
146
146
 
147
- let effectCacheItemRowsSchema = S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
148
- id: s.m(S$RescriptSchema.string),
149
- output: s.m(cacheOutputSchema)
150
- })));
151
-
152
147
  function makeCacheTable(effectName) {
153
148
  return Table.mkTable(cacheTablePrefix + effectName, undefined, [
154
- Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined),
155
- Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined)
149
+ Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
150
+ Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined)
156
151
  ], undefined);
157
152
  }
158
153
 
@@ -173,7 +168,6 @@ export {
173
168
  fuelTransferParamsSchema,
174
169
  cacheTablePrefix,
175
170
  cacheOutputSchema,
176
- effectCacheItemRowsSchema,
177
171
  makeCacheTable,
178
172
  loadedFromDbCheckpointId,
179
173
  initialCheckpointId,
@@ -54,7 +54,7 @@ let deleteKey: (dict<'value>, string) => unit = (_obj, _k) => %raw(`delete _obj[
54
54
  let timeoutAfter = timeoutMillis =>
55
55
  Utils.delay(timeoutMillis)->Promise.then(() =>
56
56
  Promise.reject(
57
- LoaderTimeout(`Query took longer than ${Belt.Int.toString(timeoutMillis / 1000)} seconds`),
57
+ LoaderTimeout(`Query took longer than ${Int.toString(timeoutMillis / 1000)} seconds`),
58
58
  )
59
59
  )
60
60
 
@@ -67,7 +67,7 @@ let rec loadNext = async (am: asyncMap<'key, 'value>, k: 'key) => {
67
67
  // Resolve the external promise
68
68
  am.resolvers
69
69
  ->Utils.Map.get(k)
70
- ->Belt.Option.forEach(r => {
70
+ ->Option.forEach(r => {
71
71
  let _ = am.resolvers->Utils.Map.delete(k)
72
72
  r(val)
73
73
  })
@@ -2,7 +2,7 @@
2
2
 
3
3
  import * as Utils from "./Utils.res.mjs";
4
4
  import * as JsSdsl from "js-sdsl";
5
- import * as Belt_Option from "@rescript/runtime/lib/es6/Belt_Option.js";
5
+ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
6
6
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
7
7
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
8
8
 
@@ -35,7 +35,7 @@ function deleteKey(_obj, _k) {
35
35
  function timeoutAfter(timeoutMillis) {
36
36
  return Utils.delay(timeoutMillis).then(() => Promise.reject({
37
37
  RE_EXN_ID: LoaderTimeout,
38
- _1: `Query took longer than ` + String(timeoutMillis / 1000 | 0) + ` seconds`
38
+ _1: `Query took longer than ` + (timeoutMillis / 1000 | 0).toString() + ` seconds`
39
39
  }));
40
40
  }
41
41
 
@@ -46,7 +46,7 @@ async function loadNext(am, k) {
46
46
  am.loaderFn(k),
47
47
  timeoutAfter(am._timeoutMillis)
48
48
  ]);
49
- Belt_Option.forEach(am.resolvers.get(k), r => {
49
+ Stdlib_Option.forEach(am.resolvers.get(k), r => {
50
50
  am.resolvers.delete(k);
51
51
  r(val);
52
52
  });
package/src/LoadLayer.res CHANGED
@@ -17,11 +17,15 @@ let loadById = (
17
17
  // Since LoadManager.call prevents registerign entities already existing in the inMemoryStore,
18
18
  // we can be sure that we load only the new ones.
19
19
  let dbEntities = try {
20
- await storage.loadByIdsOrThrow(
21
- ~table=entityConfig.table,
22
- ~rowsSchema=entityConfig.rowsSchema,
23
- ~ids=idsToLoad,
24
- )
20
+ (
21
+ await storage.loadOrThrow(
22
+ ~table=entityConfig.table,
23
+ ~filter=EntityFilter.In({
24
+ fieldName: Table.idFieldName,
25
+ fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
26
+ }),
27
+ )
28
+ )->(Utils.magic: array<unknown> => array<Internal.entity>)
25
29
  } catch {
26
30
  | Persistence.StorageError({message, reason}) =>
27
31
  reason->ErrorHandling.mkLogAndRaise(~logger=item->Logging.getItemLogger, ~msg=message)
@@ -161,8 +165,8 @@ let rec executeWithRateLimit = (
161
165
 
162
166
  // Split into immediate and queued
163
167
  let immediateCount = Math.Int.min(state.availableCalls, effectArgs->Array.length)
164
- let immediateArgs = effectArgs->Belt.Array.slice(~offset=0, ~len=immediateCount)
165
- let queuedArgs = effectArgs->Belt.Array.sliceToEnd(immediateCount)
168
+ let immediateArgs = effectArgs->Array.slice(~start=0, ~end=immediateCount)
169
+ let queuedArgs = effectArgs->Array.slice(~start=immediateCount)
166
170
 
167
171
  // Update available calls
168
172
  state.availableCalls = state.availableCalls - immediateCount
@@ -264,11 +268,15 @@ let loadEffect = (
264
268
  let {table, outputSchema} = effect.storageMeta
265
269
 
266
270
  let dbEntities = try {
267
- await storage.loadByIdsOrThrow(
268
- ~table,
269
- ~rowsSchema=Internal.effectCacheItemRowsSchema,
270
- ~ids=idsToLoad,
271
- )
271
+ (
272
+ await storage.loadOrThrow(
273
+ ~table,
274
+ ~filter=EntityFilter.In({
275
+ fieldName: Table.idFieldName,
276
+ fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
277
+ }),
278
+ )
279
+ )->(Utils.magic: array<unknown> => array<Internal.effectCacheItem>)
272
280
  } catch {
273
281
  | exn =>
274
282
  item
@@ -342,60 +350,38 @@ let loadEffect = (
342
350
  )
343
351
  }
344
352
 
345
- let loadByField = (
353
+ let loadByFilter = (
346
354
  ~loadManager,
347
355
  ~persistence: Persistence.t,
348
- ~operator: TableIndices.Operator.t,
349
356
  ~entityConfig: Internal.entityConfig,
350
357
  ~inMemoryStore,
351
- ~fieldName,
352
- ~fieldValueSchema,
353
358
  ~shouldGroup,
354
359
  ~item,
355
- ~fieldValue,
360
+ ~filter: EntityFilter.t,
356
361
  ) => {
357
- let operatorCallName = switch operator {
358
- | Eq => "eq"
359
- | Gt => "gt"
360
- | Lt => "lt"
361
- }
362
- let key = `${entityConfig.name}.getWhere.${fieldName}.${operatorCallName}`
362
+ let key = filter->EntityFilter.toOperationKey(~entityName=entityConfig.name)
363
363
  let inMemTable = inMemoryStore->InMemoryStore.getInMemTable(~entityConfig)
364
364
 
365
- let load = async (fieldValues: array<'fieldValue>, ~onError as _) => {
365
+ let load = async (filters: array<EntityFilter.t>, ~onError as _) => {
366
366
  let storage = persistence->Persistence.getInitializedStorageOrThrow
367
367
  let timerRef = Prometheus.StorageLoad.startOperation(~storage=storage.name, ~operation=key)
368
368
 
369
369
  let size = ref(0)
370
370
 
371
- let indiciesToLoad = fieldValues->Array.map((fieldValue): TableIndices.Index.t => {
372
- Single({
373
- fieldName,
374
- fieldValue: TableIndices.FieldValue.castFrom(fieldValue),
375
- operator,
376
- })
377
- })
371
+ filters->Array.forEach(filter => inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter))
372
+
373
+ // Loading a superset of rows via a merged query is safe: every loaded
374
+ // entity is matched against all registered indices, not only the
375
+ // query's own filter.
376
+ let queries = filters->EntityFilter.merge
378
377
 
379
- let _ = await indiciesToLoad
380
- ->Array.map(async index => {
381
- inMemTable->InMemoryTable.Entity.addEmptyIndex(~index)
378
+ let _ = await queries
379
+ ->Array.map(async filter => {
382
380
  try {
383
- let entities = await storage.loadByFieldOrThrow(
384
- ~operator=switch index {
385
- | Single({operator: Gt}) => #">"
386
- | Single({operator: Eq}) => #"="
387
- | Single({operator: Lt}) => #"<"
388
- },
389
- ~table=entityConfig.table,
390
- ~rowsSchema=entityConfig.rowsSchema,
391
- ~fieldName=index->TableIndices.Index.getFieldName,
392
- ~fieldValue=switch index {
393
- | Single({fieldValue}) => fieldValue
394
- },
395
- ~fieldSchema=fieldValueSchema->(
396
- Utils.magic: S.t<'fieldValue> => S.t<TableIndices.FieldValue.t>
397
- ),
398
- )
381
+ let entities =
382
+ (await storage.loadOrThrow(~table=entityConfig.table, ~filter))->(
383
+ Utils.magic: array<unknown> => array<Internal.entity>
384
+ )
399
385
 
400
386
  entities->Array.forEach(entity => {
401
387
  inMemTable->InMemoryTable.Entity.initValue(
@@ -411,11 +397,13 @@ let loadByField = (
411
397
  reason->ErrorHandling.mkLogAndRaise(
412
398
  ~logger=Logging.createChildFrom(
413
399
  ~logger=item->Logging.getItemLogger,
400
+ // The executed query might be merged from multiple getWhere
401
+ // calls, so report it as the operation users write with the
402
+ // values bound to its placeholders, instead of an internal
403
+ // filter representation they never constructed.
414
404
  ~params={
415
- "operator": operatorCallName,
416
- "tableName": entityConfig.table.tableName,
417
- "fieldName": fieldName,
418
- "fieldValue": fieldValue,
405
+ "operation": key,
406
+ "params": filter->EntityFilter.getParams,
419
407
  },
420
408
  ),
421
409
  ~msg=message,
@@ -427,7 +415,7 @@ let loadByField = (
427
415
  timerRef->Prometheus.StorageLoad.endOperation(
428
416
  ~storage=storage.name,
429
417
  ~operation=key,
430
- ~whereSize=fieldValues->Array.length,
418
+ ~whereSize=queries->Array.reduce(0, (acc, query) => acc + query->EntityFilter.valuesCount),
431
419
  ~size=size.contents,
432
420
  )
433
421
  }
@@ -435,11 +423,10 @@ let loadByField = (
435
423
  loadManager->LoadManager.call(
436
424
  ~key,
437
425
  ~load,
438
- ~input=fieldValue,
426
+ ~input=filter,
439
427
  ~shouldGroup,
440
- ~hasher=fieldValue =>
441
- fieldValue->TableIndices.FieldValue.castFrom->TableIndices.FieldValue.toString,
442
- ~getUnsafeInMemory=inMemTable->InMemoryTable.Entity.getUnsafeOnIndex(~fieldName, ~operator),
443
- ~hasInMemory=inMemTable->InMemoryTable.Entity.hasIndex(~fieldName, ~operator),
428
+ ~hasher=EntityFilter.toString,
429
+ ~getUnsafeInMemory=inMemTable->InMemoryTable.Entity.getUnsafeOnIndex,
430
+ ~hasInMemory=inMemTable->InMemoryTable.Entity.hasIndex,
444
431
  )
445
432
  }