envio 3.5.0-alpha.0 → 3.5.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/index.d.ts +10 -6
  2. package/package.json +6 -6
  3. package/src/BatchProcessing.res +13 -0
  4. package/src/BatchProcessing.res.mjs +4 -0
  5. package/src/Bin.res +2 -2
  6. package/src/ChainFetching.res +5 -5
  7. package/src/ChainFetching.res.mjs +3 -4
  8. package/src/ChainState.res +44 -36
  9. package/src/ChainState.res.mjs +36 -31
  10. package/src/ChainState.resi +2 -12
  11. package/src/Change.res +3 -3
  12. package/src/Config.res +4 -4
  13. package/src/Config.res.mjs +1 -10
  14. package/src/Core.res +4 -35
  15. package/src/Core.res.mjs +1 -31
  16. package/src/CrossChainState.res +2 -20
  17. package/src/CrossChainState.res.mjs +2 -6
  18. package/src/EntityId.res +15 -0
  19. package/src/EntityId.res.mjs +9 -0
  20. package/src/Env.res +16 -0
  21. package/src/Env.res.mjs +6 -0
  22. package/src/EventConfigBuilder.res +1 -58
  23. package/src/EventConfigBuilder.res.mjs +1 -33
  24. package/src/FetchState.res +663 -484
  25. package/src/FetchState.res.mjs +491 -440
  26. package/src/HandlerRegister.res +45 -0
  27. package/src/HandlerRegister.res.mjs +43 -0
  28. package/src/InMemoryStore.res +11 -4
  29. package/src/InMemoryTable.res +12 -10
  30. package/src/InMemoryTable.res.mjs +6 -5
  31. package/src/IndexerState.res +59 -2
  32. package/src/IndexerState.res.mjs +48 -3
  33. package/src/IndexerState.resi +2 -0
  34. package/src/Internal.res +17 -8
  35. package/src/LogSelection.res +2 -1
  36. package/src/Metrics.res +37 -6
  37. package/src/Metrics.res.mjs +5 -1
  38. package/src/Persistence.res +1 -1
  39. package/src/PgStorage.res +55 -22
  40. package/src/PgStorage.res.mjs +25 -17
  41. package/src/SimulateItems.res +2 -2
  42. package/src/TestIndexer.res +6 -5
  43. package/src/TestIndexer.res.mjs +3 -2
  44. package/src/UserContext.res +3 -3
  45. package/src/Writing.res +12 -2
  46. package/src/Writing.res.mjs +13 -2
  47. package/src/bindings/ClickHouse.res +5 -3
  48. package/src/bindings/ClickHouse.res.mjs +2 -5
  49. package/src/bindings/Vitest.res +22 -1
  50. package/src/bindings/Vitest.res.mjs +15 -0
  51. package/src/db/EntityHistory.res +16 -7
  52. package/src/db/EntityHistory.res.mjs +7 -6
  53. package/src/db/Table.res +40 -9
  54. package/src/db/Table.res.mjs +44 -6
  55. package/src/sources/AddressSet.res +48 -0
  56. package/src/sources/AddressSet.res.mjs +11 -0
  57. package/src/sources/AddressStore.res +152 -0
  58. package/src/sources/AddressStore.res.mjs +97 -0
  59. package/src/sources/EvmChain.res +3 -0
  60. package/src/sources/EvmChain.res.mjs +4 -2
  61. package/src/sources/EvmHyperSyncSource.res +9 -9
  62. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  63. package/src/sources/EvmRpcClient.res +10 -3
  64. package/src/sources/EvmRpcClient.res.mjs +3 -3
  65. package/src/sources/FuelHyperSync.res +4 -3
  66. package/src/sources/FuelHyperSync.res.mjs +3 -3
  67. package/src/sources/FuelHyperSync.resi +2 -1
  68. package/src/sources/FuelHyperSyncClient.res +12 -6
  69. package/src/sources/FuelHyperSyncClient.res.mjs +2 -2
  70. package/src/sources/FuelHyperSyncSource.res +7 -4
  71. package/src/sources/FuelHyperSyncSource.res.mjs +3 -3
  72. package/src/sources/HyperSync.res +9 -5
  73. package/src/sources/HyperSync.res.mjs +3 -3
  74. package/src/sources/HyperSync.resi +3 -2
  75. package/src/sources/HyperSyncClient.res +24 -10
  76. package/src/sources/HyperSyncClient.res.mjs +4 -4
  77. package/src/sources/RpcSource.res +71 -67
  78. package/src/sources/RpcSource.res.mjs +4 -4
  79. package/src/sources/SimulateSource.res +26 -11
  80. package/src/sources/SimulateSource.res.mjs +13 -5
  81. package/src/sources/Source.res +8 -4
  82. package/src/sources/SourceManager.res +2 -3
  83. package/src/sources/SourceManager.res.mjs +2 -3
  84. package/src/sources/Svm.res +1 -2
  85. package/src/sources/Svm.res.mjs +1 -1
  86. package/src/sources/SvmHyperSyncClient.res +14 -4
  87. package/src/sources/SvmHyperSyncClient.res.mjs +2 -2
  88. package/src/sources/SvmHyperSyncSource.res +11 -6
  89. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  90. package/src/IndexingAddresses.res +0 -108
  91. package/src/IndexingAddresses.res.mjs +0 -101
  92. package/src/IndexingAddresses.resi +0 -34
@@ -1,26 +1,34 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
+ import * as Env from "./Env.res.mjs";
3
4
  import * as Utils from "./Utils.res.mjs";
4
5
  import * as Js_math from "@rescript/runtime/lib/es6/Js_math.js";
5
6
  import * as Logging from "./Logging.res.mjs";
7
+ import * as AddressSet from "./sources/AddressSet.res.mjs";
6
8
  import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
9
+ import * as AddressStore from "./sources/AddressStore.res.mjs";
7
10
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
8
11
  import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
9
12
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
10
13
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
11
14
  import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
12
15
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
13
- import * as IndexingAddresses from "./IndexingAddresses.res.mjs";
14
16
 
15
- let deriveContractNameByAddress = Utils.$$WeakMap.memoize(addressesByContractName => {
16
- let result = {};
17
- Utils.Dict.forEachWithKey(addressesByContractName, (addresses, contractName) => {
18
- for (let i = 0, i_finish = addresses.length; i < i_finish; ++i) {
19
- result[addresses[i]] = contractName;
20
- }
21
- });
22
- return result;
23
- });
17
+ function withAddresses(p, addresses) {
18
+ return {
19
+ id: p.id,
20
+ latestFetchedBlock: p.latestFetchedBlock,
21
+ selection: p.selection,
22
+ addresses: addresses,
23
+ mergeBlock: p.mergeBlock,
24
+ dynamicContract: p.dynamicContract,
25
+ mutPendingQueries: p.mutPendingQueries,
26
+ sourceRangeCapacity: p.sourceRangeCapacity,
27
+ prevSourceRangeCapacity: p.prevSourceRangeCapacity,
28
+ eventDensity: p.eventDensity,
29
+ latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
30
+ };
31
+ }
24
32
 
25
33
  function densityItemsTarget(density, fromBlock, toBlock, chainTargetBlock) {
26
34
  return Primitive_int.max(1, Math.ceil(((Stdlib_Option.getOr(toBlock, chainTargetBlock) - fromBlock | 0) + 1 | 0) * density) | 0);
@@ -68,7 +76,7 @@ function getOrThrow(optimizedPartitions, partitionId) {
68
76
  }
69
77
 
70
78
  function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAddrInPartition, nextPartitionIndexRef) {
71
- let combinedAddresses = p1.addressesByContractName[contractName].concat(p2.addressesByContractName[contractName]);
79
+ let combinedAddresses = p1.addresses.merge(p2.addresses);
72
80
  let p1Below = p1.latestFetchedBlock.blockNumber < potentialMergeBlock;
73
81
  let p2Below = p2.latestFetchedBlock.blockNumber < potentialMergeBlock;
74
82
  let completed = [];
@@ -79,7 +87,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
79
87
  id: p1.id,
80
88
  latestFetchedBlock: p1.latestFetchedBlock,
81
89
  selection: p1.selection,
82
- addressesByContractName: p1.addressesByContractName,
90
+ addresses: p1.addresses,
83
91
  mergeBlock: potentialMergeBlock,
84
92
  dynamicContract: p1.dynamicContract,
85
93
  mutPendingQueries: p1.mutPendingQueries,
@@ -92,7 +100,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
92
100
  id: p2.id,
93
101
  latestFetchedBlock: p2.latestFetchedBlock,
94
102
  selection: p2.selection,
95
- addressesByContractName: p2.addressesByContractName,
103
+ addresses: p2.addresses,
96
104
  mergeBlock: potentialMergeBlock,
97
105
  dynamicContract: p2.dynamicContract,
98
106
  mutPendingQueries: p2.mutPendingQueries,
@@ -121,7 +129,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
121
129
  blockTimestamp: 0
122
130
  },
123
131
  selection: p1.selection,
124
- addressesByContractName: {},
132
+ addresses: p1.addresses,
125
133
  mergeBlock: undefined,
126
134
  dynamicContract: contractName,
127
135
  mutPendingQueries: [],
@@ -135,7 +143,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
135
143
  id: p1.id,
136
144
  latestFetchedBlock: p1.latestFetchedBlock,
137
145
  selection: p1.selection,
138
- addressesByContractName: p1.addressesByContractName,
146
+ addresses: p1.addresses,
139
147
  mergeBlock: potentialMergeBlock,
140
148
  dynamicContract: p1.dynamicContract,
141
149
  mutPendingQueries: p1.mutPendingQueries,
@@ -151,7 +159,7 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
151
159
  id: p2.id,
152
160
  latestFetchedBlock: p2.latestFetchedBlock,
153
161
  selection: p2.selection,
154
- addressesByContractName: p2.addressesByContractName,
162
+ addresses: p2.addresses,
155
163
  mergeBlock: potentialMergeBlock,
156
164
  dynamicContract: p2.dynamicContract,
157
165
  mutPendingQueries: p2.mutPendingQueries,
@@ -164,58 +172,27 @@ function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAd
164
172
  } else {
165
173
  continuingBase = p1;
166
174
  }
167
- if (combinedAddresses.length > maxAddrInPartition) {
168
- let addressesFull = combinedAddresses.slice(0, maxAddrInPartition);
169
- let addressesRest = combinedAddresses.slice(maxAddrInPartition);
170
- let abcFull = {};
171
- abcFull[contractName] = addressesFull;
172
- let abcRest = {};
173
- abcRest[contractName] = addressesRest;
174
- completed.push({
175
- id: continuingBase.id,
176
- latestFetchedBlock: continuingBase.latestFetchedBlock,
177
- selection: continuingBase.selection,
178
- addressesByContractName: abcFull,
179
- mergeBlock: continuingBase.mergeBlock,
180
- dynamicContract: continuingBase.dynamicContract,
181
- mutPendingQueries: continuingBase.mutPendingQueries,
182
- sourceRangeCapacity: continuingBase.sourceRangeCapacity,
183
- prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
184
- eventDensity: continuingBase.eventDensity,
185
- latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
186
- });
175
+ if (combinedAddresses.size() > maxAddrInPartition) {
176
+ completed.push(withAddresses(continuingBase, combinedAddresses.slice(0, maxAddrInPartition)));
187
177
  let restId = nextPartitionIndexRef.contents.toString();
188
178
  nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
179
+ let init = withAddresses(continuingBase, combinedAddresses.slice(maxAddrInPartition, undefined));
189
180
  completed.push({
190
181
  id: restId,
191
- latestFetchedBlock: continuingBase.latestFetchedBlock,
192
- selection: continuingBase.selection,
193
- addressesByContractName: abcRest,
194
- mergeBlock: continuingBase.mergeBlock,
195
- dynamicContract: continuingBase.dynamicContract,
182
+ latestFetchedBlock: init.latestFetchedBlock,
183
+ selection: init.selection,
184
+ addresses: init.addresses,
185
+ mergeBlock: init.mergeBlock,
186
+ dynamicContract: init.dynamicContract,
196
187
  mutPendingQueries: [],
197
- sourceRangeCapacity: continuingBase.sourceRangeCapacity,
198
- prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
199
- eventDensity: continuingBase.eventDensity,
200
- latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
188
+ sourceRangeCapacity: init.sourceRangeCapacity,
189
+ prevSourceRangeCapacity: init.prevSourceRangeCapacity,
190
+ eventDensity: init.eventDensity,
191
+ latestSourceRangeCapacityUpdateBlock: init.latestSourceRangeCapacityUpdateBlock
201
192
  });
202
193
  return completed;
203
194
  }
204
- let abc = {};
205
- abc[contractName] = combinedAddresses;
206
- completed.push({
207
- id: continuingBase.id,
208
- latestFetchedBlock: continuingBase.latestFetchedBlock,
209
- selection: continuingBase.selection,
210
- addressesByContractName: abc,
211
- mergeBlock: continuingBase.mergeBlock,
212
- dynamicContract: continuingBase.dynamicContract,
213
- mutPendingQueries: continuingBase.mutPendingQueries,
214
- sourceRangeCapacity: continuingBase.sourceRangeCapacity,
215
- prevSourceRangeCapacity: continuingBase.prevSourceRangeCapacity,
216
- eventDensity: continuingBase.eventDensity,
217
- latestSourceRangeCapacityUpdateBlock: continuingBase.latestSourceRangeCapacityUpdateBlock
218
- });
195
+ completed.push(withAddresses(continuingBase, combinedAddresses));
219
196
  return completed;
220
197
  }
221
198
 
@@ -223,7 +200,7 @@ function ascSortFn(a, b) {
223
200
  return Primitive_int.compare(a.latestFetchedBlock.blockNumber, b.latestFetchedBlock.blockNumber);
224
201
  }
225
202
 
226
- function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContracts) {
203
+ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContracts, clientFilteredContracts) {
227
204
  let newPartitions = [];
228
205
  let mergingPartitions = {};
229
206
  let nextPartitionIndexRef = {
@@ -234,7 +211,7 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
234
211
  let exit = 0;
235
212
  if (p.dynamicContract !== undefined && p.mergeBlock === undefined && p.selection.dependsOnAddresses) {
236
213
  let contractName = p.dynamicContract;
237
- let pAddressesCount = p.addressesByContractName[contractName].length;
214
+ let pAddressesCount = p.addresses.countFor(contractName);
238
215
  let match = Utils.$$Array.last(p.mutPendingQueries);
239
216
  let potentialMergeBlock;
240
217
  if (match !== undefined) {
@@ -313,7 +290,8 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
313
290
  entities: entities,
314
291
  maxAddrInPartition: maxAddrInPartition,
315
292
  nextPartitionIndex: nextPartitionIndexRef.contents,
316
- dynamicContracts: dynamicContracts
293
+ dynamicContracts: dynamicContracts,
294
+ clientFilteredContracts: clientFilteredContracts
317
295
  };
318
296
  }
319
297
 
@@ -359,26 +337,24 @@ function getPendingQueryOrThrow(p, fromBlock) {
359
337
  }
360
338
  }
361
339
 
362
- function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount, latestFetchedBlock) {
363
- let partitionId = query.partitionId;
364
- let p = optimizedPartitions.entities[partitionId];
365
- let p$1 = p !== undefined ? p : Stdlib_JsError.throwWithMessage(`Unexpected case: Couldn't find partition ` + partitionId);
340
+ function handleQueryResponseForPartition(optimizedPartitions, p, query, knownHeight, itemsCount, latestFetchedBlock) {
366
341
  let mutEntities = Utils.Dict.shallowCopy(optimizedPartitions.entities);
367
- let pendingQuery = getPendingQueryOrThrow(p$1, query.fromBlock);
342
+ let pendingQuery = getPendingQueryOrThrow(p, query.fromBlock);
368
343
  pendingQuery.fetchedBlock = latestFetchedBlock;
369
344
  let blockRange = (latestFetchedBlock.blockNumber - query.fromBlock | 0) + 1 | 0;
370
345
  let observedEventDensity = itemsCount / blockRange;
371
- let eventDensity = p$1.eventDensity;
346
+ let eventDensity = p.eventDensity;
372
347
  let updatedEventDensity = eventDensity !== undefined ? (eventDensity + observedEventDensity) / 2 : observedEventDensity;
373
348
  let shouldUpdateSourceRangeCapacity = false;
374
- if (latestFetchedBlock.blockNumber > p$1.latestSourceRangeCapacityUpdateBlock) {
349
+ if (latestFetchedBlock.blockNumber > p.latestSourceRangeCapacityUpdateBlock) {
375
350
  let queryToBlock = query.toBlock;
376
351
  let tmp;
377
352
  if (queryToBlock !== undefined) {
378
353
  if (latestFetchedBlock.blockNumber < queryToBlock) {
379
- tmp = itemsCount < query.itemsTarget;
354
+ let itemsTarget = query.itemsTarget;
355
+ tmp = itemsTarget !== undefined ? itemsCount < itemsTarget : true;
380
356
  } else {
381
- let minHistoryRange = getMinHistoryRange(p$1);
357
+ let minHistoryRange = getMinHistoryRange(p);
382
358
  tmp = minHistoryRange !== undefined ? ((queryToBlock - query.fromBlock | 0) + 1 | 0) >= minHistoryRange : false;
383
359
  }
384
360
  } else {
@@ -386,10 +362,10 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
386
362
  }
387
363
  shouldUpdateSourceRangeCapacity = tmp;
388
364
  }
389
- let updatedSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? blockRange : p$1.sourceRangeCapacity;
390
- let updatedPrevSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? p$1.sourceRangeCapacity : p$1.prevSourceRangeCapacity;
391
- let mutPendingQueries = p$1.mutPendingQueries;
392
- let latestFetchedBlock$1 = p$1.latestFetchedBlock;
365
+ let updatedSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? blockRange : p.sourceRangeCapacity;
366
+ let updatedPrevSourceRangeCapacity = shouldUpdateSourceRangeCapacity ? p.sourceRangeCapacity : p.prevSourceRangeCapacity;
367
+ let mutPendingQueries = p.mutPendingQueries;
368
+ let latestFetchedBlock$1 = p.latestFetchedBlock;
393
369
  let consumedCount = 0;
394
370
  let canContinue = true;
395
371
  while (canContinue) {
@@ -410,24 +386,24 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
410
386
  mutPendingQueries.splice(0, consumedCount);
411
387
  }
412
388
  let updatedLatestFetchedBlock = latestFetchedBlock$1;
413
- let mergeBlock = p$1.mergeBlock;
389
+ let mergeBlock = p.mergeBlock;
414
390
  let partitionReachedMergeBlock = mergeBlock !== undefined ? updatedLatestFetchedBlock.blockNumber >= mergeBlock : false;
415
391
  if (partitionReachedMergeBlock) {
416
- Utils.Dict.deleteInPlace(mutEntities, p$1.id);
417
- return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts);
418
- }
419
- let updatedMainPartition_id = p$1.id;
420
- let updatedMainPartition_selection = p$1.selection;
421
- let updatedMainPartition_addressesByContractName = p$1.addressesByContractName;
422
- let updatedMainPartition_mergeBlock = p$1.mergeBlock;
423
- let updatedMainPartition_dynamicContract = p$1.dynamicContract;
424
- let updatedMainPartition_mutPendingQueries = p$1.mutPendingQueries;
425
- let updatedMainPartition_latestSourceRangeCapacityUpdateBlock = shouldUpdateSourceRangeCapacity ? latestFetchedBlock.blockNumber : p$1.latestSourceRangeCapacityUpdateBlock;
392
+ Utils.Dict.deleteInPlace(mutEntities, p.id);
393
+ return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts, optimizedPartitions.clientFilteredContracts);
394
+ }
395
+ let updatedMainPartition_id = p.id;
396
+ let updatedMainPartition_selection = p.selection;
397
+ let updatedMainPartition_addresses = p.addresses;
398
+ let updatedMainPartition_mergeBlock = p.mergeBlock;
399
+ let updatedMainPartition_dynamicContract = p.dynamicContract;
400
+ let updatedMainPartition_mutPendingQueries = p.mutPendingQueries;
401
+ let updatedMainPartition_latestSourceRangeCapacityUpdateBlock = shouldUpdateSourceRangeCapacity ? latestFetchedBlock.blockNumber : p.latestSourceRangeCapacityUpdateBlock;
426
402
  let updatedMainPartition = {
427
403
  id: updatedMainPartition_id,
428
404
  latestFetchedBlock: updatedLatestFetchedBlock,
429
405
  selection: updatedMainPartition_selection,
430
- addressesByContractName: updatedMainPartition_addressesByContractName,
406
+ addresses: updatedMainPartition_addresses,
431
407
  mergeBlock: updatedMainPartition_mergeBlock,
432
408
  dynamicContract: updatedMainPartition_dynamicContract,
433
409
  mutPendingQueries: updatedMainPartition_mutPendingQueries,
@@ -436,13 +412,13 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
436
412
  eventDensity: updatedEventDensity,
437
413
  latestSourceRangeCapacityUpdateBlock: updatedMainPartition_latestSourceRangeCapacityUpdateBlock
438
414
  };
439
- mutEntities[p$1.id] = updatedMainPartition;
415
+ mutEntities[p.id] = updatedMainPartition;
440
416
  if (optimizedPartitions.dynamicContracts.size !== 0) {
441
- return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts);
417
+ return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts, optimizedPartitions.clientFilteredContracts);
442
418
  }
443
419
  let ids = optimizedPartitions.idsInAscOrder;
444
420
  let count = ids.length;
445
- let idx = ids.indexOf(p$1.id);
421
+ let idx = ids.indexOf(p.id);
446
422
  let isAfter = jdx => {
447
423
  if (jdx < count) {
448
424
  return mutEntities[ids[jdx]].latestFetchedBlock.blockNumber < updatedLatestFetchedBlock.blockNumber;
@@ -456,7 +432,8 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
456
432
  entities: mutEntities,
457
433
  maxAddrInPartition: optimizedPartitions.maxAddrInPartition,
458
434
  nextPartitionIndex: optimizedPartitions.nextPartitionIndex,
459
- dynamicContracts: optimizedPartitions.dynamicContracts
435
+ dynamicContracts: optimizedPartitions.dynamicContracts,
436
+ clientFilteredContracts: optimizedPartitions.clientFilteredContracts
460
437
  };
461
438
  }
462
439
  let reordered = ids.slice();
@@ -465,16 +442,26 @@ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount
465
442
  reordered[jdx] = ids[jdx + 1 | 0];
466
443
  jdx = jdx + 1 | 0;
467
444
  };
468
- reordered[jdx] = p$1.id;
445
+ reordered[jdx] = p.id;
469
446
  return {
470
447
  idsInAscOrder: reordered,
471
448
  entities: mutEntities,
472
449
  maxAddrInPartition: optimizedPartitions.maxAddrInPartition,
473
450
  nextPartitionIndex: optimizedPartitions.nextPartitionIndex,
474
- dynamicContracts: optimizedPartitions.dynamicContracts
451
+ dynamicContracts: optimizedPartitions.dynamicContracts,
452
+ clientFilteredContracts: optimizedPartitions.clientFilteredContracts
475
453
  };
476
454
  }
477
455
 
456
+ function handleQueryResponse(optimizedPartitions, query, knownHeight, itemsCount, latestFetchedBlock) {
457
+ let p = optimizedPartitions.entities[query.partitionId];
458
+ if (p !== undefined) {
459
+ return handleQueryResponseForPartition(optimizedPartitions, p, query, knownHeight, itemsCount, latestFetchedBlock);
460
+ } else {
461
+ return optimizedPartitions;
462
+ }
463
+ }
464
+
478
465
  function getLatestFullyFetchedBlock(optimizedPartitions) {
479
466
  let id = optimizedPartitions.idsInAscOrder[0];
480
467
  if (id !== undefined) {
@@ -492,6 +479,7 @@ let OptimizedPartitions = {
492
479
  consumeFetchedQueries: consumeFetchedQueries,
493
480
  getPendingQueryOrThrow: getPendingQueryOrThrow,
494
481
  handleQueryResponse: handleQueryResponse,
482
+ handleQueryResponseForPartition: handleQueryResponseForPartition,
495
483
  getLatestFullyFetchedBlock: getLatestFullyFetchedBlock
496
484
  };
497
485
 
@@ -683,7 +671,6 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
683
671
  startBlock: fetchState.startBlock,
684
672
  endBlock: fetchState.endBlock,
685
673
  normalSelection: fetchState.normalSelection,
686
- contractConfigs: fetchState.contractConfigs,
687
674
  chainId: fetchState.chainId,
688
675
  latestOnBlockBlockNumber: latestOnBlockBlockNumber,
689
676
  blockLag: blockLag,
@@ -691,143 +678,254 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
691
678
  maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
692
679
  onBlockRegistrations: fetchState.onBlockRegistrations,
693
680
  knownHeight: knownHeight,
694
- firstEventBlock: fetchState.firstEventBlock
681
+ firstEventBlock: fetchState.firstEventBlock,
682
+ clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold
695
683
  };
696
684
  }
697
685
 
698
- function warnDifferentContractType(fetchState, existingContract, dc) {
699
- let logger = Logging.createChild({
700
- chainId: fetchState.chainId,
701
- contractAddress: dc.address,
702
- existingContractType: existingContract.contractName,
703
- newContractType: dc.contractName
704
- });
705
- Logging.childWarn(logger, `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`);
686
+ function addClientFilteredContract(clientFilteredContracts, contractName, chainId, addressCount, threshold) {
687
+ clientFilteredContracts.add(contractName);
688
+ Logging.childTrace(Logging.createChild({
689
+ chainId: chainId,
690
+ contractName: contractName,
691
+ addressCount: addressCount,
692
+ threshold: threshold
693
+ }), "Switching contract to client-side address filtering: registered address count crossed the server-side threshold.");
706
694
  }
707
695
 
708
- function addressesByContractNameCount(addressesByContractName) {
709
- let numAddresses = {
710
- contents: 0
711
- };
712
- Utils.Dict.forEach(addressesByContractName, addresses => {
713
- numAddresses.contents = numAddresses.contents + addresses.length | 0;
696
+ function claimedFetchedBlock(p, knownHeight) {
697
+ return Stdlib_Array.reduce(p.mutPendingQueries, p.latestFetchedBlock.blockNumber, (max, q) => {
698
+ let match = q.fetchedBlock;
699
+ return Primitive_int.max(max, match !== undefined ? match.blockNumber : Stdlib_Option.getOr(q.toBlock, knownHeight));
714
700
  });
715
- return numAddresses.contents;
716
701
  }
717
702
 
718
- function addressesByContractNameGetAll(addressesByContractName) {
719
- let all = [];
720
- Utils.Dict.forEach(addressesByContractName, addresses => {
721
- for (let idx = 0, idx_finish = addresses.length; idx < idx_finish; ++idx) {
722
- all.push(addresses[idx]);
703
+ function collapseClientFilteredContracts(partitions, clientFilteredContracts, normalSelection, nextPartitionIndexRef, addressStore, knownHeight) {
704
+ if (clientFilteredContracts.size === 0) {
705
+ return partitions;
706
+ }
707
+ let kept = [];
708
+ let standingRef = {
709
+ contents: undefined
710
+ };
711
+ let backfills = [];
712
+ let absorbedPartitions = [];
713
+ let strippedFrontiers = [];
714
+ partitions.forEach(p => {
715
+ if (!p.selection.dependsOnAddresses) {
716
+ if (p.mergeBlock !== undefined) {
717
+ backfills.push(p);
718
+ } else if (Stdlib_Option.isNone(standingRef.contents)) {
719
+ standingRef.contents = p;
720
+ } else {
721
+ absorbedPartitions.push(p);
722
+ }
723
+ return;
724
+ }
725
+ let contractNames = p.addresses.contractNames();
726
+ let serverSideNames = contractNames.filter(c => !clientFilteredContracts.has(c));
727
+ if (serverSideNames.length === contractNames.length) {
728
+ kept.push(p);
729
+ } else if (Utils.$$Array.isEmpty(serverSideNames)) {
730
+ absorbedPartitions.push(p);
731
+ } else {
732
+ strippedFrontiers.push(p.latestFetchedBlock);
733
+ kept.push(withAddresses(p, p.addresses.filterByContracts(serverSideNames)));
734
+ }
735
+ });
736
+ let standing = standingRef.contents;
737
+ let selectionChanged = standing !== undefined ? Stdlib_Option.mapOr(standing.selection.clientFilteredContracts, 0, prim => prim.length) !== clientFilteredContracts.size : true;
738
+ if (Utils.$$Array.isEmpty(absorbedPartitions) && Utils.$$Array.isEmpty(strippedFrontiers) && !selectionChanged) {
739
+ return partitions;
740
+ }
741
+ let minFrontierRef = {
742
+ contents: undefined
743
+ };
744
+ let considerFrontier = b => {
745
+ let m = minFrontierRef.contents;
746
+ if (m !== undefined && m.blockNumber <= b.blockNumber) {
747
+ return;
748
+ } else {
749
+ minFrontierRef.contents = b;
750
+ return;
751
+ }
752
+ };
753
+ absorbedPartitions.forEach(p => considerFrontier(p.latestFetchedBlock));
754
+ backfills.forEach(p => considerFrontier(p.latestFetchedBlock));
755
+ strippedFrontiers.forEach(considerFrontier);
756
+ let regByIndex = {};
757
+ let addRegs = regs => {
758
+ regs.forEach(reg => {
759
+ regByIndex[reg.index.toString()] = reg;
760
+ });
761
+ };
762
+ let standing$1 = standingRef.contents;
763
+ if (standing$1 !== undefined) {
764
+ addRegs(standing$1.selection.onEventRegistrations);
765
+ }
766
+ absorbedPartitions.forEach(p => {
767
+ if (!p.selection.dependsOnAddresses) {
768
+ return addRegs(p.selection.onEventRegistrations);
769
+ }
770
+ });
771
+ addRegs(normalSelection.onEventRegistrations.filter(reg => clientFilteredContracts.has(reg.eventConfig.contractName)));
772
+ let newSelection_onEventRegistrations = Object.values(regByIndex);
773
+ let newSelection_clientFilteredContracts = Array.from(clientFilteredContracts);
774
+ let newSelection = {
775
+ onEventRegistrations: newSelection_onEventRegistrations,
776
+ dependsOnAddresses: false,
777
+ clientFilteredContracts: newSelection_clientFilteredContracts
778
+ };
779
+ let standing$2 = standingRef.contents;
780
+ if (standing$2 !== undefined) {
781
+ let standingOut;
782
+ if (selectionChanged) {
783
+ let id = nextPartitionIndexRef.contents.toString();
784
+ nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
785
+ standingOut = {
786
+ id: id,
787
+ latestFetchedBlock: standing$2.latestFetchedBlock,
788
+ selection: newSelection,
789
+ addresses: standing$2.addresses,
790
+ mergeBlock: standing$2.mergeBlock,
791
+ dynamicContract: standing$2.dynamicContract,
792
+ mutPendingQueries: [],
793
+ sourceRangeCapacity: standing$2.sourceRangeCapacity,
794
+ prevSourceRangeCapacity: standing$2.prevSourceRangeCapacity,
795
+ eventDensity: standing$2.eventDensity,
796
+ latestSourceRangeCapacityUpdateBlock: standing$2.latestSourceRangeCapacityUpdateBlock
797
+ };
798
+ } else {
799
+ standingOut = standing$2;
800
+ }
801
+ kept.push(standingOut);
802
+ let catchUpToBlock = claimedFetchedBlock(standingOut, knownHeight);
803
+ let minFrontier = minFrontierRef.contents;
804
+ if (minFrontier !== undefined && minFrontier.blockNumber < catchUpToBlock) {
805
+ let id$1 = nextPartitionIndexRef.contents.toString();
806
+ nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
807
+ kept.push({
808
+ id: id$1,
809
+ latestFetchedBlock: minFrontier,
810
+ selection: newSelection,
811
+ addresses: addressStore.emptySet(),
812
+ mergeBlock: catchUpToBlock,
813
+ dynamicContract: undefined,
814
+ mutPendingQueries: [],
815
+ sourceRangeCapacity: standing$2.sourceRangeCapacity,
816
+ prevSourceRangeCapacity: standing$2.prevSourceRangeCapacity,
817
+ eventDensity: standing$2.eventDensity,
818
+ latestSourceRangeCapacityUpdateBlock: 0
819
+ });
723
820
  }
821
+ return kept;
822
+ }
823
+ let minFrontier$1 = minFrontierRef.contents;
824
+ if (minFrontier$1 === undefined) {
825
+ return kept;
826
+ }
827
+ let minRange = getMinQueryRange(absorbedPartitions);
828
+ let id$2 = nextPartitionIndexRef.contents.toString();
829
+ nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
830
+ kept.push({
831
+ id: id$2,
832
+ latestFetchedBlock: minFrontier$1,
833
+ selection: newSelection,
834
+ addresses: addressStore.emptySet(),
835
+ mergeBlock: undefined,
836
+ dynamicContract: undefined,
837
+ mutPendingQueries: [],
838
+ sourceRangeCapacity: minRange,
839
+ prevSourceRangeCapacity: minRange,
840
+ eventDensity: undefined,
841
+ latestSourceRangeCapacityUpdateBlock: 0
724
842
  });
725
- return all;
843
+ return kept;
726
844
  }
727
845
 
728
- function createPartitionsFromIndexingAddresses(registeringContractsByContract, dynamicContracts, normalSelection, maxAddrInPartition, nextPartitionIndex, existingPartitions, progressBlockNumber) {
729
- let nextPartitionIndexRef = nextPartitionIndex;
846
+ function createPartitions(registeringSetsByContract, addressStore, dynamicContracts, clientFilteredContracts, normalSelection, maxAddrInPartition, nextPartitionIndex, existingPartitions, progressBlockNumber, knownHeight) {
847
+ let nextPartitionIndexRef = {
848
+ contents: nextPartitionIndex
849
+ };
730
850
  let dynamicPartitions = [];
731
851
  let nonDynamicPartitions = [];
732
- let contractNames = Object.keys(registeringContractsByContract);
852
+ let contractNames = Object.keys(registeringSetsByContract);
733
853
  for (let cIdx = 0, cIdx_finish = contractNames.length; cIdx < cIdx_finish; ++cIdx) {
734
854
  let contractName = contractNames[cIdx];
735
- let registeringContracts = registeringContractsByContract[contractName];
736
- let addresses = Object.keys(registeringContracts);
855
+ let contractSet = registeringSetsByContract[contractName];
737
856
  let isDynamic = dynamicContracts.has(contractName);
738
857
  let partitions = isDynamic ? dynamicPartitions : nonDynamicPartitions;
739
- let byStartBlock = {};
740
- for (let jdx = 0, jdx_finish = addresses.length; jdx < jdx_finish; ++jdx) {
741
- let address = addresses[jdx];
742
- let indexingContract = registeringContracts[address];
743
- Utils.Dict.push(byStartBlock, indexingContract.effectiveStartBlock.toString(), address);
744
- }
745
- let ascKeys = Object.keys(byStartBlock);
746
- let initialKey = Utils.$$Array.firstUnsafe(ascKeys);
747
- let startBlockRef = Stdlib_Int.fromString(initialKey, undefined);
748
- let addressesRef = byStartBlock[initialKey];
749
- for (let idx = 0, idx_finish = ascKeys.length; idx < idx_finish; ++idx) {
750
- let maybeNextStartBlockKey = ascKeys[idx + 1 | 0];
751
- let shouldAllocateNewPartition;
752
- if (maybeNextStartBlockKey !== undefined) {
753
- let nextStartBlock = Stdlib_Int.fromString(maybeNextStartBlockKey, undefined);
754
- let shouldJoinCurrentStartBlock = (nextStartBlock - startBlockRef | 0) < 20000;
755
- if (shouldJoinCurrentStartBlock) {
756
- addressesRef = addressesRef.concat(byStartBlock[maybeNextStartBlockKey]);
757
- shouldAllocateNewPartition = false;
858
+ let groups = contractSet.startBlockGroups();
859
+ let offsetRef = 0;
860
+ let groupIdx = 0;
861
+ while (groupIdx < groups.length) {
862
+ let startBlock = groups[groupIdx].startBlock;
863
+ let countRef = 0;
864
+ let nextIdx = groupIdx;
865
+ let joining = true;
866
+ while (joining && nextIdx < groups.length) {
867
+ let group = groups[nextIdx];
868
+ if ((group.startBlock - startBlock | 0) < 20000) {
869
+ countRef = countRef + group.count | 0;
870
+ nextIdx = nextIdx + 1 | 0;
758
871
  } else {
759
- shouldAllocateNewPartition = true;
760
- }
761
- } else {
762
- shouldAllocateNewPartition = true;
763
- }
764
- if (shouldAllocateNewPartition) {
765
- let latestFetchedBlock_blockNumber = Primitive_int.max(startBlockRef - 1 | 0, progressBlockNumber);
766
- let latestFetchedBlock = {
767
- blockNumber: latestFetchedBlock_blockNumber,
768
- blockTimestamp: 0
769
- };
770
- while (addressesRef.length !== 0) {
771
- let pAddresses = addressesRef.slice(0, maxAddrInPartition);
772
- addressesRef = addressesRef.slice(maxAddrInPartition);
773
- let addressesByContractName = {};
774
- addressesByContractName[contractName] = pAddresses;
775
- partitions.push({
776
- id: nextPartitionIndexRef.toString(),
777
- latestFetchedBlock: latestFetchedBlock,
778
- selection: normalSelection,
779
- addressesByContractName: addressesByContractName,
780
- mergeBlock: undefined,
781
- dynamicContract: isDynamic ? contractName : undefined,
782
- mutPendingQueries: [],
783
- sourceRangeCapacity: 0,
784
- prevSourceRangeCapacity: 0,
785
- eventDensity: undefined,
786
- latestSourceRangeCapacityUpdateBlock: 0
787
- });
788
- nextPartitionIndexRef = nextPartitionIndexRef + 1 | 0;
789
- };
790
- if (maybeNextStartBlockKey !== undefined) {
791
- startBlockRef = Stdlib_Int.fromString(maybeNextStartBlockKey, undefined);
792
- addressesRef = byStartBlock[maybeNextStartBlockKey];
872
+ joining = false;
793
873
  }
794
- }
795
- }
874
+ };
875
+ let latestFetchedBlock_blockNumber = Primitive_int.max(startBlock - 1 | 0, progressBlockNumber);
876
+ let latestFetchedBlock = {
877
+ blockNumber: latestFetchedBlock_blockNumber,
878
+ blockTimestamp: 0
879
+ };
880
+ let remainingRef = countRef;
881
+ let chunkOffsetRef = offsetRef;
882
+ while (remainingRef > 0) {
883
+ let take = Primitive_int.min(remainingRef, maxAddrInPartition);
884
+ let pAddresses = contractSet.slice(chunkOffsetRef, take);
885
+ partitions.push({
886
+ id: nextPartitionIndexRef.contents.toString(),
887
+ latestFetchedBlock: latestFetchedBlock,
888
+ selection: normalSelection,
889
+ addresses: pAddresses,
890
+ mergeBlock: undefined,
891
+ dynamicContract: isDynamic ? contractName : undefined,
892
+ mutPendingQueries: [],
893
+ sourceRangeCapacity: 0,
894
+ prevSourceRangeCapacity: 0,
895
+ eventDensity: undefined,
896
+ latestSourceRangeCapacityUpdateBlock: 0
897
+ });
898
+ nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0;
899
+ chunkOffsetRef = chunkOffsetRef + take | 0;
900
+ remainingRef = remainingRef - take | 0;
901
+ };
902
+ offsetRef = offsetRef + countRef | 0;
903
+ groupIdx = nextIdx;
904
+ };
796
905
  }
797
906
  let mergedNonDynamic = [];
798
907
  if (nonDynamicPartitions.length !== 0) {
799
908
  nonDynamicPartitions.sort(ascSortFn);
800
909
  let currentPRef = nonDynamicPartitions[0];
801
- let nextIdx = 1;
802
- while (nextIdx < nonDynamicPartitions.length) {
803
- let nextP = nonDynamicPartitions[nextIdx];
910
+ let nextIdx$1 = 1;
911
+ while (nextIdx$1 < nonDynamicPartitions.length) {
912
+ let nextP = nonDynamicPartitions[nextIdx$1];
804
913
  let currentP = currentPRef;
805
914
  let currentPBlock = currentP.latestFetchedBlock.blockNumber;
806
915
  let nextPBlock = nextP.latestFetchedBlock.blockNumber;
807
- let totalCount = addressesByContractNameCount(currentP.addressesByContractName) + addressesByContractNameCount(nextP.addressesByContractName) | 0;
916
+ let totalCount = currentP.addresses.size() + nextP.addresses.size() | 0;
808
917
  if (totalCount > maxAddrInPartition) {
809
918
  mergedNonDynamic.push(currentP);
810
919
  currentPRef = nextP;
811
920
  } else {
812
- let mergedAddresses = Utils.Dict.shallowCopy(nextP.addressesByContractName);
813
- let currentContractNames = Object.keys(currentP.addressesByContractName);
814
- for (let jdx$1 = 0, jdx_finish$1 = currentContractNames.length; jdx$1 < jdx_finish$1; ++jdx$1) {
815
- let cn = currentContractNames[jdx$1];
816
- let currentAddrs = currentP.addressesByContractName[cn];
817
- let existingAddrs = mergedAddresses[cn];
818
- if (existingAddrs !== undefined) {
819
- mergedAddresses[cn] = existingAddrs.concat(currentAddrs);
820
- } else {
821
- mergedAddresses[cn] = currentAddrs;
822
- }
823
- }
921
+ let mergedAddresses = nextP.addresses.merge(currentP.addresses);
824
922
  let isTooFar = (currentPBlock + 20000 | 0) < nextPBlock;
825
923
  if (isTooFar) {
826
924
  mergedNonDynamic.push({
827
925
  id: currentP.id,
828
926
  latestFetchedBlock: currentP.latestFetchedBlock,
829
927
  selection: currentP.selection,
830
- addressesByContractName: currentP.addressesByContractName,
928
+ addresses: currentP.addresses,
831
929
  mergeBlock: currentPBlock < nextPBlock ? nextPBlock : undefined,
832
930
  dynamicContract: currentP.dynamicContract,
833
931
  mutPendingQueries: currentP.mutPendingQueries,
@@ -836,190 +934,134 @@ function createPartitionsFromIndexingAddresses(registeringContractsByContract, d
836
934
  eventDensity: currentP.eventDensity,
837
935
  latestSourceRangeCapacityUpdateBlock: currentP.latestSourceRangeCapacityUpdateBlock
838
936
  });
839
- currentPRef = {
840
- id: nextP.id,
841
- latestFetchedBlock: nextP.latestFetchedBlock,
842
- selection: nextP.selection,
843
- addressesByContractName: mergedAddresses,
844
- mergeBlock: nextP.mergeBlock,
845
- dynamicContract: nextP.dynamicContract,
846
- mutPendingQueries: nextP.mutPendingQueries,
847
- sourceRangeCapacity: nextP.sourceRangeCapacity,
848
- prevSourceRangeCapacity: nextP.prevSourceRangeCapacity,
849
- eventDensity: nextP.eventDensity,
850
- latestSourceRangeCapacityUpdateBlock: nextP.latestSourceRangeCapacityUpdateBlock
851
- };
937
+ currentPRef = withAddresses(nextP, mergedAddresses);
852
938
  } else {
853
- currentPRef = {
854
- id: currentP.id,
855
- latestFetchedBlock: currentP.latestFetchedBlock,
856
- selection: currentP.selection,
857
- addressesByContractName: mergedAddresses,
858
- mergeBlock: currentP.mergeBlock,
859
- dynamicContract: currentP.dynamicContract,
860
- mutPendingQueries: currentP.mutPendingQueries,
861
- sourceRangeCapacity: currentP.sourceRangeCapacity,
862
- prevSourceRangeCapacity: currentP.prevSourceRangeCapacity,
863
- eventDensity: currentP.eventDensity,
864
- latestSourceRangeCapacityUpdateBlock: currentP.latestSourceRangeCapacityUpdateBlock
865
- };
939
+ currentPRef = withAddresses(currentP, mergedAddresses);
866
940
  }
867
941
  }
868
- nextIdx = nextIdx + 1 | 0;
942
+ nextIdx$1 = nextIdx$1 + 1 | 0;
869
943
  };
870
944
  mergedNonDynamic.push(currentPRef);
871
945
  }
872
946
  let mergedPartitions = mergedNonDynamic.concat(dynamicPartitions);
873
- return make(existingPartitions.concat(mergedPartitions), maxAddrInPartition, nextPartitionIndexRef, dynamicContracts);
947
+ let allPartitions = collapseClientFilteredContracts(existingPartitions.concat(mergedPartitions), clientFilteredContracts, normalSelection, nextPartitionIndexRef, addressStore, knownHeight);
948
+ return make(allPartitions, maxAddrInPartition, nextPartitionIndexRef.contents, dynamicContracts, clientFilteredContracts);
874
949
  }
875
950
 
876
- function registerDynamicContracts(fetchState, indexingAddresses, items) {
951
+ function registerDynamicContracts(fetchState, addressStore, claimCeilingOpt, items) {
952
+ let claimCeiling = claimCeilingOpt !== undefined ? claimCeilingOpt : fetchState.knownHeight;
877
953
  if (Utils.$$Array.isEmpty(fetchState.normalSelection.onEventRegistrations)) {
878
954
  Stdlib_JsError.throwWithMessage("Invalid configuration. No events to fetch for the dynamic contract registration.");
879
955
  }
880
- let registeringContractsByContract = {};
881
- let earliestRegisteringEventBlockNumber = Infinity;
882
- let noEventsAddresses = {};
883
- let registeringAddresses = {};
956
+ let registrations = [];
957
+ let sourceDcs = [];
958
+ let sourceIndexes = [];
884
959
  for (let itemIdx = 0, itemIdx_finish = items.length; itemIdx < itemIdx_finish; ++itemIdx) {
885
- let item = items[itemIdx];
886
- let dcs = item.dcs;
960
+ let dcs = items[itemIdx].dcs;
887
961
  if (dcs !== undefined) {
888
- let idx = 0;
889
- while (idx < dcs.length) {
890
- let dc = dcs[idx];
891
- let shouldRemove = false;
892
- let match = fetchState.contractConfigs[dc.contractName];
893
- if (match !== undefined) {
894
- let dcWithStartBlock_address = dc.address;
895
- let dcWithStartBlock_contractName = dc.contractName;
896
- let dcWithStartBlock_registrationBlock = dc.registrationBlock;
897
- let dcWithStartBlock_effectiveStartBlock = IndexingAddresses.deriveEffectiveStartBlock(dc.registrationBlock, match.startBlock);
898
- let dcWithStartBlock = {
899
- address: dcWithStartBlock_address,
900
- contractName: dcWithStartBlock_contractName,
901
- registrationBlock: dcWithStartBlock_registrationBlock,
902
- effectiveStartBlock: dcWithStartBlock_effectiveStartBlock
903
- };
904
- let existingContract = IndexingAddresses.get(indexingAddresses, dc.address);
905
- if (existingContract !== undefined) {
906
- if (existingContract.contractName !== dc.contractName) {
907
- warnDifferentContractType(fetchState, existingContract, dcWithStartBlock);
908
- } else if (existingContract.effectiveStartBlock > dcWithStartBlock_effectiveStartBlock) {
909
- let logger = Logging.createChild({
910
- chainId: fetchState.chainId,
911
- contractAddress: dc.address,
912
- existingBlockNumber: existingContract.effectiveStartBlock,
913
- newBlockNumber: dcWithStartBlock_effectiveStartBlock
914
- });
915
- Logging.childWarn(logger, `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`);
916
- }
917
- shouldRemove = true;
918
- } else {
919
- let registeringContract = registeringAddresses[dc.address];
920
- let shouldUpdate = registeringContract !== undefined ? (
921
- registeringContract.contractName !== dc.contractName ? (warnDifferentContractType(fetchState, registeringContract, dcWithStartBlock), false) : false
922
- ) : true;
923
- if (shouldUpdate) {
924
- earliestRegisteringEventBlockNumber = Primitive_int.min(earliestRegisteringEventBlockNumber, dcWithStartBlock_effectiveStartBlock);
925
- Utils.Dict.getOrInsertEmptyDict(registeringContractsByContract, dc.contractName)[dc.address] = dcWithStartBlock;
926
- registeringAddresses[dc.address] = dcWithStartBlock;
927
- } else {
928
- shouldRemove = true;
929
- }
962
+ for (let dcIdx = 0, dcIdx_finish = dcs.length; dcIdx < dcIdx_finish; ++dcIdx) {
963
+ let dc = dcs[dcIdx];
964
+ registrations.push({
965
+ address: dc.address,
966
+ contractName: dc.contractName,
967
+ registrationBlock: dc.registrationBlock
968
+ });
969
+ sourceDcs.push(dcs);
970
+ sourceIndexes.push(dcIdx);
971
+ }
972
+ }
973
+ }
974
+ let idCursor = addressStore.nextId();
975
+ let verdicts = AddressStore.registerBatch(addressStore, registrations);
976
+ let warn = (contractAddress, params, message) => Logging.childWarn(Logging.createChild({
977
+ chainId: fetchState.chainId,
978
+ contractAddress: contractAddress,
979
+ details: params
980
+ }), message);
981
+ let registeringContractNames = [];
982
+ let hasNoEventsUpdatesRef = false;
983
+ for (let idx = 0, idx_finish = verdicts.length; idx < idx_finish; ++idx) {
984
+ let registration = registrations[idx];
985
+ let contractAddress = registration.address;
986
+ let match = verdicts[idx];
987
+ let keep;
988
+ if (typeof match !== "object") {
989
+ warn(contractAddress, {
990
+ contractName: registration.contractName
991
+ }, `Skipping contract registration: Not a valid address for this chain's ecosystem.`);
992
+ keep = false;
993
+ } else {
994
+ switch (match.TAG) {
995
+ case "Added" :
996
+ if (!registeringContractNames.includes(registration.contractName)) {
997
+ registeringContractNames.push(registration.contractName);
930
998
  }
931
- } else {
932
- let dcAsIndexingAddress_address = dc.address;
933
- let dcAsIndexingAddress_contractName = dc.contractName;
934
- let dcAsIndexingAddress_registrationBlock = dc.registrationBlock;
935
- let dcAsIndexingAddress_effectiveStartBlock = IndexingAddresses.deriveEffectiveStartBlock(dc.registrationBlock, undefined);
936
- let dcAsIndexingAddress = {
937
- address: dcAsIndexingAddress_address,
938
- contractName: dcAsIndexingAddress_contractName,
939
- registrationBlock: dcAsIndexingAddress_registrationBlock,
940
- effectiveStartBlock: dcAsIndexingAddress_effectiveStartBlock
941
- };
942
- let existingContract$1 = IndexingAddresses.get(indexingAddresses, dc.address);
943
- if (existingContract$1 !== undefined) {
944
- if (existingContract$1.contractName !== dc.contractName) {
945
- warnDifferentContractType(fetchState, existingContract$1, dcAsIndexingAddress);
946
- }
947
- shouldRemove = true;
948
- } else {
949
- let existingContract$2 = registeringAddresses[dc.address];
950
- if (existingContract$2 !== undefined) {
951
- if (existingContract$2.contractName !== dc.contractName) {
952
- warnDifferentContractType(fetchState, existingContract$2, dcAsIndexingAddress);
953
- }
954
- shouldRemove = true;
955
- } else {
956
- let logger$1 = Logging.createChild({
957
- chainId: fetchState.chainId,
958
- contractAddress: dc.address,
959
- contractName: dc.contractName
960
- });
961
- Logging.childWarn(logger$1, `Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`);
962
- noEventsAddresses[dc.address] = dcAsIndexingAddress;
963
- registeringAddresses[dc.address] = dcAsIndexingAddress;
964
- }
999
+ keep = true;
1000
+ break;
1001
+ case "NoEvents" :
1002
+ hasNoEventsUpdatesRef = true;
1003
+ warn(contractAddress, {
1004
+ contractName: registration.contractName
1005
+ }, `Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`);
1006
+ keep = true;
1007
+ break;
1008
+ case "Duplicate" :
1009
+ let existingEffectiveStartBlock = match.existingEffectiveStartBlock;
1010
+ let effectiveStartBlock = match.effectiveStartBlock;
1011
+ if (existingEffectiveStartBlock > effectiveStartBlock) {
1012
+ warn(contractAddress, {
1013
+ existingBlockNumber: existingEffectiveStartBlock,
1014
+ newBlockNumber: effectiveStartBlock
1015
+ }, `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`);
965
1016
  }
966
- }
967
- if (shouldRemove) {
968
- dcs.splice(idx, 1);
969
- } else {
970
- idx = idx + 1 | 0;
971
- }
972
- };
1017
+ keep = false;
1018
+ break;
1019
+ case "Conflict" :
1020
+ warn(contractAddress, {
1021
+ existingContractType: match.existingContractName,
1022
+ newContractType: registration.contractName
1023
+ }, `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`);
1024
+ keep = false;
1025
+ break;
1026
+ }
1027
+ }
1028
+ if (!keep) {
1029
+ sourceIndexes[idx] = -1 - sourceIndexes[idx] | 0;
973
1030
  }
974
1031
  }
975
- let dcContractNamesToStore = Object.keys(registeringContractsByContract);
976
- let hasNoEventsUpdates = !Utils.Dict.isEmpty(noEventsAddresses);
977
- if (dcContractNamesToStore.length === 0) {
978
- if (hasNoEventsUpdates) {
979
- IndexingAddresses.register(indexingAddresses, noEventsAddresses);
980
- return fetchState;
981
- } else {
982
- return fetchState;
1032
+ for (let idx$1 = verdicts.length - 1 | 0; idx$1 >= 0; --idx$1) {
1033
+ let marked = sourceIndexes[idx$1];
1034
+ if (marked < 0) {
1035
+ sourceDcs[idx$1].splice(-1 - marked | 0, 1);
983
1036
  }
984
1037
  }
1038
+ if (registeringContractNames.length === 0) {
1039
+ return fetchState;
1040
+ }
985
1041
  let newPartitions = [];
986
1042
  let dynamicContractsRef = fetchState.optimizedPartitions.dynamicContracts;
987
1043
  let mutExistingPartitions = Object.values(fetchState.optimizedPartitions.entities);
988
- for (let idx$1 = 0, idx_finish = dcContractNamesToStore.length; idx$1 < idx_finish; ++idx$1) {
989
- let contractName = dcContractNamesToStore[idx$1];
1044
+ for (let idx$2 = 0, idx_finish$1 = registeringContractNames.length; idx$2 < idx_finish$1; ++idx$2) {
1045
+ let contractName = registeringContractNames[idx$2];
990
1046
  if (!dynamicContractsRef.has(contractName)) {
991
1047
  dynamicContractsRef = Utils.$$Set.immutableAdd(dynamicContractsRef, contractName);
992
- for (let idx$2 = 0, idx_finish$1 = mutExistingPartitions.length; idx$2 < idx_finish$1; ++idx$2) {
993
- let p = mutExistingPartitions[idx$2];
994
- let addresses = p.addressesByContractName[contractName];
995
- if (addresses !== undefined && p.selection.dependsOnAddresses && p.mergeBlock === undefined) {
996
- let allPartitionContractNames = Object.keys(p.addressesByContractName);
1048
+ for (let idx$3 = 0, idx_finish$2 = mutExistingPartitions.length; idx$3 < idx_finish$2; ++idx$3) {
1049
+ let p = mutExistingPartitions[idx$3];
1050
+ let match$1 = p.addresses.countFor(contractName);
1051
+ if (match$1 !== 0 && p.selection.dependsOnAddresses && p.mergeBlock === undefined) {
1052
+ let allPartitionContractNames = p.addresses.contractNames();
997
1053
  if (allPartitionContractNames.length !== 1) {
998
1054
  let isFetching = p.mutPendingQueries.length !== 0;
999
1055
  if (!isFetching) {
1000
1056
  let newPartitionId = (fetchState.optimizedPartitions.nextPartitionIndex + newPartitions.length | 0).toString();
1001
- let restAddressesByContractName = Utils.Dict.shallowCopy(p.addressesByContractName);
1002
- Utils.Dict.deleteInPlace(restAddressesByContractName, contractName);
1003
- mutExistingPartitions[idx$2] = {
1004
- id: p.id,
1005
- latestFetchedBlock: p.latestFetchedBlock,
1006
- selection: p.selection,
1007
- addressesByContractName: restAddressesByContractName,
1008
- mergeBlock: p.mergeBlock,
1009
- dynamicContract: p.dynamicContract,
1010
- mutPendingQueries: p.mutPendingQueries,
1011
- sourceRangeCapacity: p.sourceRangeCapacity,
1012
- prevSourceRangeCapacity: p.prevSourceRangeCapacity,
1013
- eventDensity: p.eventDensity,
1014
- latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
1015
- };
1016
- let addressesByContractName = {};
1017
- addressesByContractName[contractName] = addresses;
1057
+ let restNames = allPartitionContractNames.filter(name => name !== contractName);
1058
+ mutExistingPartitions[idx$3] = withAddresses(p, p.addresses.filterByContracts(restNames));
1059
+ let splitAddresses = p.addresses.filterByContracts([contractName]);
1018
1060
  newPartitions.push({
1019
1061
  id: newPartitionId,
1020
1062
  latestFetchedBlock: p.latestFetchedBlock,
1021
1063
  selection: fetchState.normalSelection,
1022
- addressesByContractName: addressesByContractName,
1064
+ addresses: splitAddresses,
1023
1065
  mergeBlock: undefined,
1024
1066
  dynamicContract: contractName,
1025
1067
  mutPendingQueries: p.mutPendingQueries,
@@ -1030,11 +1072,11 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
1030
1072
  });
1031
1073
  }
1032
1074
  } else {
1033
- mutExistingPartitions[idx$2] = {
1075
+ mutExistingPartitions[idx$3] = {
1034
1076
  id: p.id,
1035
1077
  latestFetchedBlock: p.latestFetchedBlock,
1036
1078
  selection: p.selection,
1037
- addressesByContractName: p.addressesByContractName,
1079
+ addresses: p.addresses,
1038
1080
  mergeBlock: p.mergeBlock,
1039
1081
  dynamicContract: contractName,
1040
1082
  mutPendingQueries: p.mutPendingQueries,
@@ -1047,26 +1089,29 @@ function registerDynamicContracts(fetchState, indexingAddresses, items) {
1047
1089
  }
1048
1090
  }
1049
1091
  }
1050
- let registeringContracts = registeringContractsByContract[contractName];
1051
- IndexingAddresses.register(indexingAddresses, registeringContracts);
1052
1092
  }
1053
- IndexingAddresses.register(indexingAddresses, noEventsAddresses);
1054
- let optimizedPartitions = createPartitionsFromIndexingAddresses(registeringContractsByContract, dynamicContractsRef, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, fetchState.optimizedPartitions.nextPartitionIndex + newPartitions.length | 0, mutExistingPartitions.concat(newPartitions), 0);
1055
- return updateInternal(fetchState, optimizedPartitions, undefined, undefined, undefined, undefined);
1056
- }
1057
-
1058
- function filterByClientAddress(items, indexingAddresses) {
1059
- return items.filter(item => {
1060
- if (item.kind !== 0) {
1061
- return true;
1062
- }
1063
- let filter = item.onEventRegistration.clientAddressFilter;
1064
- if (filter !== undefined) {
1065
- return filter(item.payload, item.blockNumber, IndexingAddresses.rawForFilter(indexingAddresses));
1066
- } else {
1067
- return true;
1068
- }
1093
+ let threshold = fetchState.clientFilterAddressThreshold;
1094
+ let clientFilteredContracts;
1095
+ if (threshold !== undefined) {
1096
+ let clientFilteredContracts$1 = new Set(Array.from(fetchState.optimizedPartitions.clientFilteredContracts));
1097
+ Array.from(dynamicContractsRef).forEach(contractName => {
1098
+ let addressCount = addressStore.contractCount(contractName);
1099
+ if (!clientFilteredContracts$1.has(contractName) && addressCount > threshold) {
1100
+ return addClientFilteredContract(clientFilteredContracts$1, contractName, fetchState.chainId, addressCount, threshold);
1101
+ }
1102
+ });
1103
+ clientFilteredContracts = clientFilteredContracts$1;
1104
+ } else {
1105
+ clientFilteredContracts = fetchState.optimizedPartitions.clientFilteredContracts;
1106
+ }
1107
+ let registeringSetsByContract = {};
1108
+ registeringContractNames.forEach(contractName => {
1109
+ registeringSetsByContract[contractName] = AddressStore.makeSet(addressStore, contractName, {
1110
+ minId: idCursor
1111
+ });
1069
1112
  });
1113
+ let optimizedPartitions = createPartitions(registeringSetsByContract, addressStore, dynamicContractsRef, clientFilteredContracts, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, fetchState.optimizedPartitions.nextPartitionIndex + newPartitions.length | 0, mutExistingPartitions.concat(newPartitions), 0, Primitive_int.max(claimCeiling, fetchState.knownHeight));
1114
+ return updateInternal(fetchState, optimizedPartitions, undefined, undefined, undefined, undefined);
1070
1115
  }
1071
1116
 
1072
1117
  function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) {
@@ -1103,23 +1148,22 @@ function startFetchingQueries(param, queries) {
1103
1148
  }
1104
1149
  }
1105
1150
 
1106
- function pushDensityPricedQuery(queries, partitionId, fromBlock, toBlock, isChunk, density, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, selection, addressesByContractName) {
1151
+ function pushDensityPricedQuery(queries, partitionId, fromBlock, toBlock, isChunk, density, chainTargetBlock, selection, addresses) {
1107
1152
  let itemsEst = densityItemsTarget(density, fromBlock, toBlock, chainTargetBlock);
1108
- let itemsTarget = densityItemsTarget(density * chunkItemsMultiplier, fromBlock, toBlock, chainTargetBlock);
1109
1153
  queries.push({
1110
1154
  partitionId: partitionId,
1111
1155
  fromBlock: fromBlock,
1112
1156
  toBlock: toBlock,
1113
1157
  isChunk: isChunk,
1114
- itemsTarget: toBlock !== undefined ? Primitive_int.max(itemsTarget, itemsTargetFloor) : itemsTarget,
1158
+ itemsTarget: toBlock !== undefined ? undefined : itemsEst,
1115
1159
  itemsEst: itemsEst,
1116
1160
  selection: selection,
1117
- addressesByContractName: addressesByContractName
1161
+ addresses: addresses
1118
1162
  });
1119
1163
  return itemsEst;
1120
1164
  }
1121
1165
 
1122
- function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock, headBlockNumber, chainTargetBlock, maybeChunkRange, maxChunks, partition, partitionBudget, chunkItemsMultiplier, itemsTargetFloor, selection, addressesByContractName) {
1166
+ function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock, headBlockNumber, chainTargetBlock, maybeChunkRange, maxChunks, partition, partitionBudget, selection, addresses) {
1123
1167
  if (!(rangeFromBlock <= Primitive_int.min(headBlockNumber, chainTargetBlock) && maxChunks > 0)) {
1124
1168
  return;
1125
1169
  }
@@ -1135,7 +1179,7 @@ function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock,
1135
1179
  if (exit === 1) {
1136
1180
  let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : chainTargetBlock;
1137
1181
  let pushSingleQuery = (density, isChunk) => {
1138
- pushDensityPricedQuery(queries, partitionId, rangeFromBlock, rangeEndBlock, isChunk, density, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, selection, addressesByContractName);
1182
+ pushDensityPricedQuery(queries, partitionId, rangeFromBlock, rangeEndBlock, isChunk, density, chainTargetBlock, selection, addresses);
1139
1183
  };
1140
1184
  let match = partition.eventDensity;
1141
1185
  let exit$1 = 0;
@@ -1150,7 +1194,7 @@ function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock,
1150
1194
  let chunkIdx = 0;
1151
1195
  while (chunkIdx < maxChunks && ((chunkFromBlock + chunkSize | 0) - 1 | 0) <= maxBlock) {
1152
1196
  let chunkToBlock = (chunkFromBlock + chunkSize | 0) - 1 | 0;
1153
- pushDensityPricedQuery(queries, partitionId, chunkFromBlock, chunkToBlock, true, match, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, selection, addressesByContractName);
1197
+ pushDensityPricedQuery(queries, partitionId, chunkFromBlock, chunkToBlock, true, match, chainTargetBlock, selection, addresses);
1154
1198
  chunkFromBlock = chunkToBlock + 1 | 0;
1155
1199
  chunkIdx = chunkIdx + 1 | 0;
1156
1200
  };
@@ -1173,7 +1217,7 @@ function pushGapFillQueries(queries, partitionId, rangeFromBlock, rangeEndBlock,
1173
1217
  }
1174
1218
  }
1175
1219
 
1176
- function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlockNumber, chainTargetBlock, chunkItemsMultiplier, itemsTargetFloor, partitionBudget, queryEndBlock) {
1220
+ function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlockNumber, chainTargetBlock, partitionBudget, queryEndBlock) {
1177
1221
  let maybeChunkRange = getMinHistoryRange(p);
1178
1222
  let pendingCount = p.mutPendingQueries.length;
1179
1223
  let cursor = p.latestFetchedBlock.blockNumber + 1 | 0;
@@ -1184,7 +1228,7 @@ function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlo
1184
1228
  let pq = p.mutPendingQueries[pqIdx];
1185
1229
  if (pq.fromBlock > cursor) {
1186
1230
  let beforeLen = candidates.length;
1187
- pushGapFillQueries(candidates, partitionId, cursor, Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock), headBlockNumber, chainTargetBlock, maybeChunkRange, (12 - inFlightCount | 0) - chunksUsedThisCall | 0, p, partitionBudget, chunkItemsMultiplier, itemsTargetFloor, p.selection, p.addressesByContractName);
1231
+ pushGapFillQueries(candidates, partitionId, cursor, Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock), headBlockNumber, chainTargetBlock, maybeChunkRange, (12 - inFlightCount | 0) - chunksUsedThisCall | 0, p, partitionBudget, p.selection, p.addresses);
1188
1232
  chunksUsedThisCall = chunksUsedThisCall + (candidates.length - beforeLen | 0) | 0;
1189
1233
  }
1190
1234
  let toBlock = pq.toBlock;
@@ -1214,7 +1258,7 @@ function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlo
1214
1258
  }
1215
1259
  }
1216
1260
 
1217
- function pushForwardCandidates(candidates, inRangeStates, inRangeCount, chainTargetBlock, freshBudget, chunkItemsMultiplier, itemsTargetFloor) {
1261
+ function pushForwardCandidates(candidates, inRangeStates, inRangeCount, chainTargetBlock, freshBudget) {
1218
1262
  let probeShare = inRangeCount === 0 ? 0 : freshBudget / inRangeCount;
1219
1263
  let frontierCursor = Stdlib_Array.reduce(inRangeStates, chainTargetBlock, (min, fs) => {
1220
1264
  if (fs.cursor < min) {
@@ -1240,23 +1284,23 @@ function pushForwardCandidates(candidates, inRangeStates, inRangeCount, chainTar
1240
1284
  let generatedItems = 0;
1241
1285
  while (created < maxChunksRemaining && chunkFromBlock <= chunkStartCeiling && generatedItems <= freshBudget) {
1242
1286
  let chunkToBlock = Primitive_int.min((chunkFromBlock + chunkSize | 0) - 1 | 0, maxBlock);
1243
- let itemsEst = pushDensityPricedQuery(candidates, fs.partitionId, chunkFromBlock, chunkToBlock, true, match$1, chunkItemsMultiplier, chainTargetBlock, itemsTargetFloor, p.selection, p.addressesByContractName);
1287
+ let itemsEst = pushDensityPricedQuery(candidates, fs.partitionId, chunkFromBlock, chunkToBlock, true, match$1, chainTargetBlock, p.selection, p.addresses);
1244
1288
  generatedItems = generatedItems + itemsEst;
1245
1289
  chunkFromBlock = chunkToBlock + 1 | 0;
1246
1290
  created = created + 1 | 0;
1247
1291
  };
1248
1292
  return;
1249
1293
  }
1250
- let itemsTarget = rangeToTarget > 0 ? Primitive_int.max(1, Math.round(rangeTargetDensity * ((chainTargetBlock - fs.cursor | 0) + 1 | 0) / inRangeCount) | 0) : Primitive_int.max(1, Math.round(probeShare) | 0);
1294
+ let itemsEst$1 = rangeToTarget > 0 ? Primitive_int.max(1, Math.round(rangeTargetDensity * ((chainTargetBlock - fs.cursor | 0) + 1 | 0) / inRangeCount) | 0) : Primitive_int.max(1, Math.round(probeShare) | 0);
1251
1295
  candidates.push({
1252
1296
  partitionId: fs.partitionId,
1253
1297
  fromBlock: fs.cursor,
1254
1298
  toBlock: fs.queryEndBlock,
1255
1299
  isChunk: false,
1256
- itemsTarget: itemsTarget,
1257
- itemsEst: itemsTarget,
1300
+ itemsTarget: itemsEst$1,
1301
+ itemsEst: itemsEst$1,
1258
1302
  selection: p.selection,
1259
- addressesByContractName: p.addressesByContractName
1303
+ addresses: p.addresses
1260
1304
  });
1261
1305
  });
1262
1306
  }
@@ -1300,7 +1344,7 @@ function acceptCandidates(candidates, reservations, chainTargetItems, partitionI
1300
1344
  let remainingBudget = chainTargetItems;
1301
1345
  let acceptIdx = 0;
1302
1346
  let usedConcurrency = reservations.length;
1303
- while (remainingBudget > 0 && acceptIdx < streamCount && usedConcurrency < 100) {
1347
+ while (remainingBudget > 0 && acceptIdx < streamCount && usedConcurrency < Env.maxChainConcurrency) {
1304
1348
  let match = acceptanceStream[acceptIdx];
1305
1349
  let maybeQuery = match[2];
1306
1350
  if (maybeQuery !== undefined) {
@@ -1313,12 +1357,10 @@ function acceptCandidates(candidates, reservations, chainTargetItems, partitionI
1313
1357
  };
1314
1358
  }
1315
1359
 
1316
- function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star, $staropt$star$1) {
1360
+ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
1317
1361
  let blockLag = param.blockLag;
1318
1362
  let endBlock = param.endBlock;
1319
1363
  let optimizedPartitions = param.optimizedPartitions;
1320
- let chunkItemsMultiplier = $staropt$star !== undefined ? $staropt$star : 1;
1321
- let itemsTargetFloor = $staropt$star$1 !== undefined ? $staropt$star$1 : 0;
1322
1364
  let headBlockNumber = param.knownHeight - blockLag | 0;
1323
1365
  if (headBlockNumber <= 0) {
1324
1366
  return "WaitingForNewBlock";
@@ -1353,7 +1395,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star,
1353
1395
  shouldWaitForNewBlock = false;
1354
1396
  }
1355
1397
  }
1356
- let availableConcurrency = 100 - reservations.length | 0;
1398
+ let availableConcurrency = Env.maxChainConcurrency - reservations.length | 0;
1357
1399
  let partitionsCount$1 = chainTargetItems <= 0 || shouldWaitForNewBlock || availableConcurrency <= 0 ? 0 : partitionsCount;
1358
1400
  let queriesByPartitionIndex = Stdlib_Array.fromInitializer(partitionsCount$1, param => []);
1359
1401
  let computeQueryEndBlock = p => {
@@ -1370,7 +1412,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star,
1370
1412
  for (let idx$1 = 0; idx$1 < partitionsCount$1; ++idx$1) {
1371
1413
  let partitionId$1 = optimizedPartitions.idsInAscOrder[idx$1];
1372
1414
  let p$1 = optimizedPartitions.entities[partitionId$1];
1373
- let fillState = walkPartitionPending(p$1, partitionId$1, inFlightCounts[idx$1], candidates, headBlockNumber, chainTargetBlock, chunkItemsMultiplier, itemsTargetFloor, partitionBudget, computeQueryEndBlock(p$1));
1415
+ let fillState = walkPartitionPending(p$1, partitionId$1, inFlightCounts[idx$1], candidates, headBlockNumber, chainTargetBlock, partitionBudget, computeQueryEndBlock(p$1));
1374
1416
  if (fillState !== undefined) {
1375
1417
  fillStates.push(fillState);
1376
1418
  }
@@ -1391,7 +1433,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star,
1391
1433
  let inRangeStates = fillStates.filter(isInRange);
1392
1434
  let inRangeCount = inRangeStates.length;
1393
1435
  let inRangeStates$1 = inRangeCount > availableConcurrency ? (inRangeStates.sort((a, b) => Primitive_int.compare(a.cursor, b.cursor)), inRangeStates.slice(0, availableConcurrency)) : inRangeStates;
1394
- pushForwardCandidates(candidates, inRangeStates$1, inRangeCount, chainTargetBlock, freshBudget, chunkItemsMultiplier, itemsTargetFloor);
1436
+ pushForwardCandidates(candidates, inRangeStates$1, inRangeCount, chainTargetBlock, freshBudget);
1395
1437
  acceptCandidates(candidates, reservations, chainTargetItems, partitionIndexById, queriesByPartitionIndex);
1396
1438
  let queries = queriesByPartitionIndex.flat();
1397
1439
  if (Utils.$$Array.isEmpty(queries)) {
@@ -1440,11 +1482,12 @@ function getReadyItemsCount(fetchState, targetSize, fromItem) {
1440
1482
  return acc;
1441
1483
  }
1442
1484
 
1443
- function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, addresses, maxAddrInPartition, chainId, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockRegistrationsOpt, blockLagOpt, firstEventBlockOpt) {
1485
+ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addresses, maxAddrInPartition, chainId, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockRegistrationsOpt, blockLagOpt, firstEventBlockOpt, clientFilterAddressThresholdOpt) {
1444
1486
  let progressBlockNumber = progressBlockNumberOpt !== undefined ? progressBlockNumberOpt : startBlock - 1 | 0;
1445
1487
  let onBlockRegistrations = onBlockRegistrationsOpt !== undefined ? onBlockRegistrationsOpt : [];
1446
1488
  let blockLag = blockLagOpt !== undefined ? blockLagOpt : 0;
1447
1489
  let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
1490
+ let clientFilterAddressThreshold = clientFilterAddressThresholdOpt !== undefined ? Primitive_option.valFromOption(clientFilterAddressThresholdOpt) : undefined;
1448
1491
  let latestFetchedBlock = {
1449
1492
  blockNumber: progressBlockNumber,
1450
1493
  blockTimestamp: 0
@@ -1469,7 +1512,7 @@ function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, add
1469
1512
  onEventRegistrations: notDependingOnAddresses,
1470
1513
  dependsOnAddresses: false
1471
1514
  },
1472
- addressesByContractName: {},
1515
+ addresses: addressStore.emptySet(),
1473
1516
  mergeBlock: undefined,
1474
1517
  dynamicContract: undefined,
1475
1518
  mutPendingQueries: [],
@@ -1483,12 +1526,20 @@ function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, add
1483
1526
  onEventRegistrations: normalRegistrations,
1484
1527
  dependsOnAddresses: true
1485
1528
  };
1486
- let registeringContractsByContract = {};
1529
+ AddressStore.registerBatch(addressStore, addresses.map(contract => ({
1530
+ address: contract.address,
1531
+ contractName: contract.contractName,
1532
+ registrationBlock: contract.registrationBlock
1533
+ })));
1487
1534
  let dynamicContracts = new Set();
1535
+ let clientFilteredContracts = new Set();
1536
+ let registeringSetsByContract = {};
1488
1537
  addresses.forEach(contract => {
1489
1538
  let contractName = contract.contractName;
1490
1539
  if (contractNamesWithNormalEvents.has(contractName)) {
1491
- Utils.Dict.getOrInsertEmptyDict(registeringContractsByContract, contractName)[contract.address] = IndexingAddresses.makeIndexingAddress(contract, contractConfigs);
1540
+ if (!(contractName in registeringSetsByContract)) {
1541
+ registeringSetsByContract[contractName] = AddressStore.makeSet(addressStore, contractName, undefined);
1542
+ }
1492
1543
  if (contract.registrationBlock !== -1) {
1493
1544
  dynamicContracts.add(contractName);
1494
1545
  return;
@@ -1497,7 +1548,15 @@ function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, add
1497
1548
  }
1498
1549
  }
1499
1550
  });
1500
- let optimizedPartitions = createPartitionsFromIndexingAddresses(registeringContractsByContract, dynamicContracts, normalSelection, maxAddrInPartition, partitions.length, partitions, progressBlockNumber);
1551
+ if (clientFilterAddressThreshold !== undefined) {
1552
+ Utils.Dict.forEachWithKey(registeringSetsByContract, (set, contractName) => {
1553
+ let addressCount = set.size();
1554
+ if (addressCount > clientFilterAddressThreshold) {
1555
+ return addClientFilteredContract(clientFilteredContracts, contractName, chainId, addressCount, clientFilterAddressThreshold);
1556
+ }
1557
+ });
1558
+ }
1559
+ let optimizedPartitions = createPartitions(registeringSetsByContract, addressStore, dynamicContracts, clientFilteredContracts, normalSelection, maxAddrInPartition, partitions.length, partitions, progressBlockNumber, knownHeight);
1501
1560
  if (optimizedPartitions.idsInAscOrder.length === 0 && Utils.$$Array.isEmpty(onBlockRegistrations)) {
1502
1561
  Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + chainId.toString() + `. ` + (`addresses=` + addresses.length.toString() + `, `) + (`onEventRegistrations=` + onEventRegistrations.length.toString() + `, `) + (`normalRegistrations=` + normalRegistrations.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
1503
1562
  }
@@ -1516,7 +1575,6 @@ function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, add
1516
1575
  startBlock: startBlock,
1517
1576
  endBlock: endBlock,
1518
1577
  normalSelection: normalSelection,
1519
- contractConfigs: contractConfigs,
1520
1578
  chainId: chainId,
1521
1579
  latestOnBlockBlockNumber: latestOnBlockBlockNumber,
1522
1580
  blockLag: blockLag,
@@ -1524,7 +1582,8 @@ function make$1(startBlock, endBlock, onEventRegistrations, contractConfigs, add
1524
1582
  maxOnBlockBufferSize: maxOnBlockBufferSize,
1525
1583
  onBlockRegistrations: onBlockRegistrations,
1526
1584
  knownHeight: knownHeight,
1527
- firstEventBlock: firstEventBlock
1585
+ firstEventBlock: firstEventBlock,
1586
+ clientFilterAddressThreshold: clientFilterAddressThreshold
1528
1587
  };
1529
1588
  }
1530
1589
 
@@ -1566,52 +1625,44 @@ function rollbackPendingQueries(mutPendingQueries, targetBlockNumber) {
1566
1625
  return adjusted;
1567
1626
  }
1568
1627
 
1569
- function rollback(fetchState, indexingAddresses, targetBlockNumber) {
1570
- IndexingAddresses.rollbackInPlace(indexingAddresses, targetBlockNumber);
1628
+ function rollback(fetchState, addressStore, targetBlockNumber) {
1629
+ addressStore.rollback(targetBlockNumber);
1571
1630
  let keptPartitions = [];
1572
1631
  let nextKeptIdRef = 0;
1573
- let registeringContractsByContract = {};
1632
+ let registeringSetsByContract = {};
1633
+ let collectForRecreation = set => {
1634
+ set.contractNames().forEach(contractName => {
1635
+ let contractSet = set.filterByContracts([contractName]);
1636
+ let existing = registeringSetsByContract[contractName];
1637
+ registeringSetsByContract[contractName] = existing !== undefined ? Primitive_option.valFromOption(existing).merge(contractSet) : contractSet;
1638
+ });
1639
+ };
1574
1640
  let partitions = Object.values(fetchState.optimizedPartitions.entities);
1575
1641
  for (let idx = 0, idx_finish = partitions.length; idx < idx_finish; ++idx) {
1576
1642
  let p = partitions[idx];
1577
1643
  if (p.selection.dependsOnAddresses) {
1578
1644
  if (p.latestFetchedBlock.blockNumber > targetBlockNumber) {
1579
- Utils.Dict.forEachWithKey(p.addressesByContractName, (addresses, contractName) => {
1580
- addresses.forEach(address => {
1581
- let indexingContract = IndexingAddresses.get(indexingAddresses, address);
1582
- if (indexingContract === undefined) {
1583
- return;
1584
- }
1585
- let registeringContracts = Utils.Dict.getOrInsertEmptyDict(registeringContractsByContract, contractName);
1586
- registeringContracts[address] = indexingContract;
1587
- });
1588
- });
1645
+ collectForRecreation(p.addresses.filterByRegistrationBlock(targetBlockNumber));
1589
1646
  } else {
1590
1647
  let mergeBlock = p.mergeBlock;
1591
1648
  let mergeBlock$1 = mergeBlock !== undefined && mergeBlock > targetBlockNumber ? targetBlockNumber : mergeBlock;
1592
- let rollbackedAddressesByContractName = {};
1593
- Utils.Dict.forEachWithKey(p.addressesByContractName, (addresses, contractName) => {
1594
- let keptAddresses = addresses.filter(address => Stdlib_Option.isSome(IndexingAddresses.get(indexingAddresses, address)));
1595
- if (keptAddresses.length !== 0) {
1596
- rollbackedAddressesByContractName[contractName] = keptAddresses;
1597
- return;
1598
- }
1599
- });
1600
- if (!Utils.Dict.isEmpty(rollbackedAddressesByContractName)) {
1649
+ let rollbackedAddresses = p.addresses.filterByRegistrationBlock(targetBlockNumber);
1650
+ if (!AddressSet.isEmpty(rollbackedAddresses)) {
1601
1651
  let id = nextKeptIdRef.toString();
1602
1652
  nextKeptIdRef = nextKeptIdRef + 1 | 0;
1653
+ let init = withAddresses(p, rollbackedAddresses);
1603
1654
  keptPartitions.push({
1604
1655
  id: id,
1605
- latestFetchedBlock: p.latestFetchedBlock,
1606
- selection: p.selection,
1607
- addressesByContractName: rollbackedAddressesByContractName,
1656
+ latestFetchedBlock: init.latestFetchedBlock,
1657
+ selection: init.selection,
1658
+ addresses: init.addresses,
1608
1659
  mergeBlock: mergeBlock$1,
1609
- dynamicContract: p.dynamicContract,
1660
+ dynamicContract: init.dynamicContract,
1610
1661
  mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
1611
- sourceRangeCapacity: p.sourceRangeCapacity,
1612
- prevSourceRangeCapacity: p.prevSourceRangeCapacity,
1613
- eventDensity: p.eventDensity,
1614
- latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
1662
+ sourceRangeCapacity: init.sourceRangeCapacity,
1663
+ prevSourceRangeCapacity: init.prevSourceRangeCapacity,
1664
+ eventDensity: init.eventDensity,
1665
+ latestSourceRangeCapacityUpdateBlock: init.latestSourceRangeCapacityUpdateBlock
1615
1666
  });
1616
1667
  }
1617
1668
  }
@@ -1625,7 +1676,7 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
1625
1676
  blockTimestamp: 0
1626
1677
  }) : p.latestFetchedBlock,
1627
1678
  selection: p.selection,
1628
- addressesByContractName: p.addressesByContractName,
1679
+ addresses: p.addresses,
1629
1680
  mergeBlock: p.mergeBlock,
1630
1681
  dynamicContract: p.dynamicContract,
1631
1682
  mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, targetBlockNumber),
@@ -1636,13 +1687,12 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
1636
1687
  });
1637
1688
  }
1638
1689
  }
1639
- let optimizedPartitions = createPartitionsFromIndexingAddresses(registeringContractsByContract, fetchState.optimizedPartitions.dynamicContracts, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, nextKeptIdRef, keptPartitions, targetBlockNumber);
1690
+ let optimizedPartitions = createPartitions(registeringSetsByContract, addressStore, fetchState.optimizedPartitions.dynamicContracts, fetchState.optimizedPartitions.clientFilteredContracts, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, nextKeptIdRef, keptPartitions, targetBlockNumber, fetchState.knownHeight);
1640
1691
  return updateInternal({
1641
1692
  optimizedPartitions: fetchState.optimizedPartitions,
1642
1693
  startBlock: fetchState.startBlock,
1643
1694
  endBlock: fetchState.endBlock,
1644
1695
  normalSelection: fetchState.normalSelection,
1645
- contractConfigs: fetchState.contractConfigs,
1646
1696
  chainId: fetchState.chainId,
1647
1697
  latestOnBlockBlockNumber: Primitive_int.min(fetchState.latestOnBlockBlockNumber, targetBlockNumber),
1648
1698
  blockLag: fetchState.blockLag,
@@ -1650,7 +1700,8 @@ function rollback(fetchState, indexingAddresses, targetBlockNumber) {
1650
1700
  maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
1651
1701
  onBlockRegistrations: fetchState.onBlockRegistrations,
1652
1702
  knownHeight: fetchState.knownHeight,
1653
- firstEventBlock: fetchState.firstEventBlock
1703
+ firstEventBlock: fetchState.firstEventBlock,
1704
+ clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold
1654
1705
  }, optimizedPartitions, fetchState.buffer.filter(item => {
1655
1706
  let tmp;
1656
1707
  tmp = item.kind === 0 ? item.blockNumber : item.blockNumber;
@@ -1669,7 +1720,7 @@ function resetPendingQueries(fetchState) {
1669
1720
  id: partition.id,
1670
1721
  latestFetchedBlock: partition.latestFetchedBlock,
1671
1722
  selection: partition.selection,
1672
- addressesByContractName: partition.addressesByContractName,
1723
+ addresses: partition.addresses,
1673
1724
  mergeBlock: partition.mergeBlock,
1674
1725
  dynamicContract: partition.dynamicContract,
1675
1726
  mutPendingQueries: kept,
@@ -1687,12 +1738,12 @@ function resetPendingQueries(fetchState) {
1687
1738
  entities: newEntities,
1688
1739
  maxAddrInPartition: init.maxAddrInPartition,
1689
1740
  nextPartitionIndex: init.nextPartitionIndex,
1690
- dynamicContracts: init.dynamicContracts
1741
+ dynamicContracts: init.dynamicContracts,
1742
+ clientFilteredContracts: init.clientFilteredContracts
1691
1743
  },
1692
1744
  startBlock: fetchState.startBlock,
1693
1745
  endBlock: fetchState.endBlock,
1694
1746
  normalSelection: fetchState.normalSelection,
1695
- contractConfigs: fetchState.contractConfigs,
1696
1747
  chainId: fetchState.chainId,
1697
1748
  latestOnBlockBlockNumber: fetchState.latestOnBlockBlockNumber,
1698
1749
  blockLag: fetchState.blockLag,
@@ -1700,7 +1751,8 @@ function resetPendingQueries(fetchState) {
1700
1751
  maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
1701
1752
  onBlockRegistrations: fetchState.onBlockRegistrations,
1702
1753
  knownHeight: fetchState.knownHeight,
1703
- firstEventBlock: fetchState.firstEventBlock
1754
+ firstEventBlock: fetchState.firstEventBlock,
1755
+ clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold
1704
1756
  };
1705
1757
  }
1706
1758
 
@@ -1817,12 +1869,12 @@ let blockItemLogIndex = 16777216;
1817
1869
 
1818
1870
  let maxInFlightChunksPerPartition = 12;
1819
1871
 
1820
- let maxChainConcurrency = 100;
1872
+ let maxChainConcurrency = Env.maxChainConcurrency;
1821
1873
 
1822
1874
  let chunkRangeGrowthFactor = 1.8;
1823
1875
 
1824
1876
  export {
1825
- deriveContractNameByAddress,
1877
+ withAddresses,
1826
1878
  densityItemsTarget,
1827
1879
  getMinHistoryRange,
1828
1880
  getTrustedDensity,
@@ -1837,12 +1889,11 @@ export {
1837
1889
  blockItemLogIndex,
1838
1890
  appendOnBlockItems,
1839
1891
  updateInternal,
1840
- warnDifferentContractType,
1841
- addressesByContractNameCount,
1842
- addressesByContractNameGetAll,
1843
- createPartitionsFromIndexingAddresses,
1892
+ addClientFilteredContract,
1893
+ claimedFetchedBlock,
1894
+ collapseClientFilteredContracts,
1895
+ createPartitions,
1844
1896
  registerDynamicContracts,
1845
- filterByClientAddress,
1846
1897
  handleQueryResult,
1847
1898
  startFetchingQueries,
1848
1899
  maxInFlightChunksPerPartition,
@@ -1870,4 +1921,4 @@ export {
1870
1921
  getProgressBlockNumberAt,
1871
1922
  updateKnownHeight,
1872
1923
  }
1873
- /* deriveContractNameByAddress Not a pure module */
1924
+ /* Env Not a pure module */