envio 3.15.0 → 3.16.0-subgraph

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +40 -34
  2. package/evm.schema.json +11 -2
  3. package/fuel.schema.json +10 -1
  4. package/index.d.ts +6 -0
  5. package/licenses/README.md +1 -1
  6. package/package.json +10 -9
  7. package/src/Batch.res.mjs +5 -5
  8. package/src/BatchProcessing.res +1 -0
  9. package/src/BatchProcessing.res.mjs +1 -0
  10. package/src/Bin.res +7 -0
  11. package/src/Bin.res.mjs +8 -0
  12. package/src/ChainFetching.res +450 -16
  13. package/src/ChainFetching.res.mjs +403 -32
  14. package/src/ChainState.res +124 -7
  15. package/src/ChainState.res.mjs +128 -67
  16. package/src/ChainState.resi +26 -0
  17. package/src/Clock.res +57 -0
  18. package/src/Clock.res.mjs +72 -0
  19. package/src/Config.res +31 -3
  20. package/src/Config.res.mjs +15 -4
  21. package/src/ContractRegisterContext.res +3 -6
  22. package/src/ContractRegisterContext.res.mjs +4 -2
  23. package/src/Core.res +95 -36
  24. package/src/Core.res.mjs +90 -1
  25. package/src/CrossChainState.res +12 -2
  26. package/src/CrossChainState.res.mjs +3 -3
  27. package/src/Ecosystem.res +7 -4
  28. package/src/Ecosystem.res.mjs +8 -8
  29. package/src/EffectCacheLookup.res +98 -0
  30. package/src/EffectCacheLookup.res.mjs +116 -0
  31. package/src/EffectState.res +10 -2
  32. package/src/EffectState.res.mjs +8 -4
  33. package/src/EffectState.resi +3 -1
  34. package/src/EntityId.res +2 -7
  35. package/src/EntityId.res.mjs +1 -8
  36. package/src/EntityTables.res +4 -1
  37. package/src/EntityTables.res.mjs +1 -1
  38. package/src/Env.res +20 -0
  39. package/src/Env.res.mjs +16 -0
  40. package/src/EventProcessing.res +38 -1
  41. package/src/EventProcessing.res.mjs +28 -1
  42. package/src/FetchState.res +408 -73
  43. package/src/FetchState.res.mjs +361 -91
  44. package/src/FinalizeBackfill.res +2 -4
  45. package/src/FinalizeBackfill.res.mjs +3 -4
  46. package/src/HandlerLoader.res +39 -15
  47. package/src/HandlerLoader.res.mjs +27 -8
  48. package/src/HandlerRegister.res +4 -0
  49. package/src/HandlerRegister.res.mjs +5 -2
  50. package/src/HandlerRegister.resi +2 -0
  51. package/src/InMemoryTable.res +17 -16
  52. package/src/InMemoryTable.res.mjs +14 -15
  53. package/src/IndexerState.res +28 -4
  54. package/src/IndexerState.res.mjs +19 -6
  55. package/src/IndexerState.resi +4 -0
  56. package/src/Internal.res +61 -49
  57. package/src/Internal.res.mjs +36 -80
  58. package/src/LoadLayer.res +54 -15
  59. package/src/LoadLayer.res.mjs +75 -45
  60. package/src/LoadLayer.resi +26 -0
  61. package/src/Main.res +14 -1
  62. package/src/Main.res.mjs +4 -3
  63. package/src/Persistence.res +3 -23
  64. package/src/Persistence.res.mjs +3 -3
  65. package/src/PreloadLookahead.res +202 -0
  66. package/src/PreloadLookahead.res.mjs +218 -0
  67. package/src/Rollback.res +0 -1
  68. package/src/Rollback.res.mjs +1 -1
  69. package/src/RunScope.res +3 -0
  70. package/src/SafeCheckpointTracking.res +4 -4
  71. package/src/SafeCheckpointTracking.res.mjs +1 -1
  72. package/src/SimulateItems.res +42 -26
  73. package/src/SimulateItems.res.mjs +40 -16
  74. package/src/StagedRows.res +106 -0
  75. package/src/StagedRows.res.mjs +102 -0
  76. package/src/Staging.res +32 -14
  77. package/src/Staging.res.mjs +17 -20
  78. package/src/Storage.res +303 -948
  79. package/src/Storage.res.mjs +229 -708
  80. package/src/SubgraphLog.res +19 -0
  81. package/src/SubgraphLog.res.mjs +26 -0
  82. package/src/Supervisor.res +75 -56
  83. package/src/Supervisor.res.mjs +55 -26
  84. package/src/SyncWrites.res +107 -0
  85. package/src/SyncWrites.res.mjs +98 -0
  86. package/src/TestIndexer.res +22 -27
  87. package/src/TestIndexer.res.mjs +20 -17
  88. package/src/Throttler.res +15 -8
  89. package/src/Throttler.res.mjs +9 -6
  90. package/src/Throttler.resi +1 -1
  91. package/src/UserContext.res +419 -46
  92. package/src/UserContext.res.mjs +318 -30
  93. package/src/Utils.res +17 -6
  94. package/src/Utils.res.mjs +12 -6
  95. package/src/Writing.res +0 -1
  96. package/src/Writing.res.mjs +1 -1
  97. package/src/bindings/ClickHouse.res +0 -322
  98. package/src/bindings/ClickHouse.res.mjs +0 -338
  99. package/src/bindings/StorageClient.res +108 -174
  100. package/src/bindings/StorageClient.res.mjs +19 -8
  101. package/src/bindings/Vitest.res +0 -26
  102. package/src/bindings/Vitest.res.mjs +0 -3
  103. package/src/db/Table.res +24 -150
  104. package/src/db/Table.res.mjs +38 -272
  105. package/src/sources/AddressSet.res +2 -0
  106. package/src/sources/AddressStore.res +3 -0
  107. package/src/sources/CallSource.res +274 -0
  108. package/src/sources/CallSource.res.mjs +196 -0
  109. package/src/sources/ChainSources.res +2 -0
  110. package/src/sources/ChainSources.res.mjs +2 -2
  111. package/src/sources/EvmChain.res +10 -1
  112. package/src/sources/EvmChain.res.mjs +6 -3
  113. package/src/sources/EvmHyperSyncSource.res +2 -2
  114. package/src/sources/EvmHyperSyncSource.res.mjs +1 -1
  115. package/src/sources/EvmRpcClient.res +13 -3
  116. package/src/sources/EvmRpcClient.res.mjs +4 -3
  117. package/src/sources/EvmRpcWs.res +4 -3
  118. package/src/sources/EvmRpcWs.res.mjs +2 -2
  119. package/src/sources/HeightFeed.res +31 -10
  120. package/src/sources/HeightFeed.res.mjs +10 -6
  121. package/src/sources/HeightStream.res +32 -17
  122. package/src/sources/HeightStream.res.mjs +10 -10
  123. package/src/sources/HyperSync.res +2 -1
  124. package/src/sources/HyperSync.res.mjs +2 -1
  125. package/src/sources/HyperSyncSSE.res +4 -6
  126. package/src/sources/HyperSyncSSE.res.mjs +2 -2
  127. package/src/sources/RpcSource.res +10 -1
  128. package/src/sources/RpcSource.res.mjs +4 -3
  129. package/src/sources/Source.res +3 -0
  130. package/src/sources/SourceManager.res +84 -77
  131. package/src/sources/SourceManager.res.mjs +17 -20
  132. package/src/sources/SourceManager.resi +3 -4
  133. package/src/sources/StartBlockResolver.res +1 -0
  134. package/src/sources/StartBlockResolver.res.mjs +3 -2
  135. package/src/sources/SvmHyperSyncSource.res +2 -2
  136. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
  137. package/src/subgraph/abi-types.ts +84 -0
  138. package/src/subgraph/aggregations.ts +745 -0
  139. package/src/subgraph/as-package.ts +143 -0
  140. package/src/subgraph/assemblyscript-regex.ts +67 -0
  141. package/src/subgraph/assemblyscript.ts +731 -0
  142. package/src/subgraph/blocks.ts +398 -0
  143. package/src/subgraph/calls-source.ts +702 -0
  144. package/src/subgraph/calls.ts +258 -0
  145. package/src/subgraph/conformance.ts +106 -0
  146. package/src/subgraph/errors.ts +85 -0
  147. package/src/subgraph/graft.ts +73 -0
  148. package/src/subgraph/graph-cli-worker.mjs +35 -0
  149. package/src/subgraph/graph-cli.ts +900 -0
  150. package/src/subgraph/graph-node-decimal.ts +199 -0
  151. package/src/subgraph/graph-ts-enums.ts +28 -0
  152. package/src/subgraph/graph-ts-types/VERSION +2 -0
  153. package/src/subgraph/graph-ts-types/chain/arweave.d.ts +70 -0
  154. package/src/subgraph/graph-ts-types/chain/cosmos.d.ts +327 -0
  155. package/src/subgraph/graph-ts-types/chain/ethereum.d.ts +233 -0
  156. package/src/subgraph/graph-ts-types/chain/near.d.ts +253 -0
  157. package/src/subgraph/graph-ts-types/chain/starknet.d.ts +32 -0
  158. package/src/subgraph/graph-ts-types/common/collections.d.ts +136 -0
  159. package/src/subgraph/graph-ts-types/common/conversion.d.ts +11 -0
  160. package/src/subgraph/graph-ts-types/common/datasource.d.ts +30 -0
  161. package/src/subgraph/graph-ts-types/common/eager-offset.d.ts +0 -0
  162. package/src/subgraph/graph-ts-types/common/json.d.ts +17 -0
  163. package/src/subgraph/graph-ts-types/common/numbers.d.ts +120 -0
  164. package/src/subgraph/graph-ts-types/common/value.d.ts +120 -0
  165. package/src/subgraph/graph-ts-types/common/yaml.d.ts +90 -0
  166. package/src/subgraph/graph-ts-types/global/global.d.ts +194 -0
  167. package/src/subgraph/graph-ts-types/helper-functions.d.ts +22 -0
  168. package/src/subgraph/graph-ts-types/index.d.ts +102 -0
  169. package/src/subgraph/graph-ts.ts +4339 -0
  170. package/src/subgraph/hosts.ts +554 -0
  171. package/src/subgraph/log.ts +28 -0
  172. package/src/subgraph/receipts.ts +184 -0
  173. package/src/subgraph/register-slots.ts +93 -0
  174. package/src/subgraph/rollup.ts +109 -0
  175. package/src/subgraph/rpc.ts +337 -0
  176. package/src/subgraph/runtime.ts +2059 -0
  177. package/src/subgraph/scope.ts +176 -0
  178. package/src/subgraph/stats.ts +42 -0
  179. package/src/subgraph/template-instances.ts +66 -0
  180. package/src/subgraph/toolchain.ts +327 -0
  181. package/src/subgraph/transient.ts +176 -0
  182. package/svm.schema.json +11 -2
  183. package/src/PgWriting.res +0 -36
  184. package/src/PgWriting.res.mjs +0 -34
  185. package/src/db/IndexDefinition.res +0 -47
  186. package/src/db/IndexDefinition.res.mjs +0 -67
@@ -1,14 +1,18 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
+ import * as Env from "./Env.res.mjs";
3
4
  import * as Utils from "./Utils.res.mjs";
4
5
  import * as Logger from "./Logger.res.mjs";
6
+ import * as Internal from "./Internal.res.mjs";
5
7
  import * as RunScope from "./RunScope.res.mjs";
6
8
  import * as Ecosystem from "./Ecosystem.res.mjs";
7
9
  import * as ChainState from "./ChainState.res.mjs";
10
+ import * as FetchState from "./FetchState.res.mjs";
8
11
  import * as IndexerState from "./IndexerState.res.mjs";
9
12
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
10
13
  import * as ChainMetadata from "./ChainMetadata.res.mjs";
11
14
  import * as ErrorHandling from "./ErrorHandling.res.mjs";
15
+ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
12
16
  import * as SourceManager from "./sources/SourceManager.res.mjs";
13
17
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
14
18
  import * as HandlerRuntime from "./HandlerRuntime.res.mjs";
@@ -20,32 +24,74 @@ function hasContractRegister(item) {
20
24
  if (item.kind === 0) {
21
25
  return item.onEventRegistration.contractRegister !== undefined;
22
26
  } else {
23
- return false;
27
+ return item.onBlockRegistration.contractRegister !== undefined;
24
28
  }
25
29
  }
26
30
 
27
31
  function invokeContractRegister(runtime, item) {
28
- let match = item.onEventRegistration;
29
- let contractRegister = match.contractRegister;
30
- let contractRegister$1 = contractRegister !== undefined ? contractRegister : Stdlib_JsError.throwWithMessage("Contract register is not set for event " + match.eventConfig.name);
31
- return contractRegister$1({
32
- event: Ecosystem.getItemEvent(item, HandlerRuntime.ecosystem(runtime)),
33
- context: runtime.contractRegisterContext
34
- });
32
+ if (item.kind === 0) {
33
+ let onEventRegistration = item.onEventRegistration;
34
+ let contractRegister = onEventRegistration.contractRegister;
35
+ let contractRegister$1 = contractRegister !== undefined ? contractRegister : Stdlib_JsError.throwWithMessage("Contract register is not set for event " + onEventRegistration.eventConfig.name);
36
+ return contractRegister$1({
37
+ event: Ecosystem.getItemEvent(item, HandlerRuntime.ecosystem(runtime)),
38
+ context: runtime.contractRegisterContext
39
+ });
40
+ }
41
+ let block = item.block;
42
+ let blockNumber = item.blockNumber;
43
+ let onBlockRegistration = item.onBlockRegistration;
44
+ let contractRegister$2 = onBlockRegistration.contractRegister;
45
+ let contractRegister$3 = contractRegister$2 !== undefined ? contractRegister$2 : Stdlib_JsError.throwWithMessage("Contract register is not set for block handler " + onBlockRegistration.name);
46
+ let ecosystem = HandlerRuntime.ecosystem(runtime);
47
+ let context = runtime.contractRegisterContext;
48
+ let match = ecosystem.name;
49
+ let tmp;
50
+ let exit = 0;
51
+ switch (match) {
52
+ case "evm" :
53
+ case "fuel" :
54
+ exit = 1;
55
+ break;
56
+ case "svm" :
57
+ tmp = {
58
+ slot: blockNumber,
59
+ block: block,
60
+ context: context
61
+ };
62
+ break;
63
+ }
64
+ if (exit === 1) {
65
+ tmp = {
66
+ block: Primitive_option.some(Stdlib_Option.getOr(block, Ecosystem.makeKeyOnlyBlock(blockNumber, ecosystem))),
67
+ context: context
68
+ };
69
+ }
70
+ return contractRegister$3(tmp);
35
71
  }
36
72
 
37
73
  async function runContractRegistersOrThrow(itemsWithContractRegister, runtime, chainState) {
38
- await ChainState.materializePageItems(itemsWithContractRegister, ChainState.transactionStore(chainState), ChainState.blockStore(chainState), ChainState.shouldChecksum(chainState));
74
+ await ChainState.materializePageItems(Env.subgraphMode.contents ? itemsWithContractRegister.filter(item => item.kind === 0) : itemsWithContractRegister, ChainState.transactionStore(chainState), ChainState.blockStore(chainState), ChainState.shouldChecksum(chainState));
39
75
  let registrations = [];
40
76
  let chainLogger = ChainState.logger(chainState);
41
77
  let ecosystem = HandlerRuntime.ecosystem(runtime);
78
+ let isSubgraph = Env.subgraphMode.contents;
42
79
  await Promise.all(itemsWithContractRegister.map(async item => {
80
+ let tmp;
81
+ if (isSubgraph) {
82
+ let tmp$1;
83
+ tmp$1 = item.kind === 0 ? FetchState.creationRoundOfRegistration(ChainState.fetchState(chainState), item) : 1;
84
+ tmp = tmp$1;
85
+ } else {
86
+ tmp = undefined;
87
+ }
43
88
  let run = {
44
89
  item: item,
45
90
  isResolved: false,
46
91
  chainLogger: chainLogger,
47
92
  ecosystem: ecosystem,
48
- registrations: registrations
93
+ registrations: registrations,
94
+ creationRound: tmp
49
95
  };
50
96
  let scope = {
51
97
  TAG: "ContractRegister",
@@ -57,16 +103,208 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, runtime, c
57
103
  } catch (raw_exn) {
58
104
  let exn = Primitive_exceptions.internalToException(raw_exn);
59
105
  RunScope.resolve(scope);
60
- return ErrorHandling.mkLogAndRaise(RunScope.contractRegisterLogger(run), "Event contractRegister failed, please fix the error to keep the indexer running smoothly", exn);
106
+ let tmp$2;
107
+ tmp$2 = item.kind === 0 ? "Event contractRegister failed, please fix the error to keep the indexer running smoothly" : "Block handler contractRegister failed, please fix the error to keep the indexer running smoothly";
108
+ return ErrorHandling.mkLogAndRaise(RunScope.contractRegisterLogger(run), tmp$2, exn);
61
109
  }
62
110
  return RunScope.resolve(scope);
63
111
  }));
64
112
  return registrations;
65
113
  }
66
114
 
115
+ function mergeRegisterItems(eventItems, blockItems) {
116
+ let merged = [];
117
+ let i = 0;
118
+ let j = 0;
119
+ while (i < eventItems.length || j < blockItems.length) {
120
+ let match = eventItems[i];
121
+ let match$1 = blockItems[j];
122
+ let takeBlock;
123
+ if (match !== undefined) {
124
+ if (match$1 !== undefined) {
125
+ let eventBlock = match.blockNumber;
126
+ let blockBlock = match$1.blockNumber;
127
+ takeBlock = blockBlock < eventBlock || blockBlock === eventBlock && Internal.itemRunsFirst(match$1);
128
+ } else {
129
+ takeBlock = false;
130
+ }
131
+ } else {
132
+ takeBlock = true;
133
+ }
134
+ if (takeBlock) {
135
+ merged.push(blockItems[j]);
136
+ j = j + 1 | 0;
137
+ } else {
138
+ merged.push(eventItems[i]);
139
+ i = i + 1 | 0;
140
+ }
141
+ };
142
+ return merged;
143
+ }
144
+
145
+ async function runRegisterPassesOrThrow(eventItems, blockItems, runtime, chainState) {
146
+ let items = mergeRegisterItems(eventItems, blockItems);
147
+ let registrations = [];
148
+ let start = 0;
149
+ while (start < items.length) {
150
+ let chunk = items.slice(start, start + 2048 | 0);
151
+ registrations.push(...await runContractRegistersOrThrow(chunk, runtime, chainState));
152
+ start = start + 2048 | 0;
153
+ };
154
+ return registrations;
155
+ }
156
+
157
+ function sortRegistrations(registrations) {
158
+ return registrations.toSorted((a, b) => {
159
+ let order = Primitive_int.compare(a.registrationBlock, b.registrationBlock);
160
+ if (order !== 0) {
161
+ return order;
162
+ } else {
163
+ return Primitive_int.compare(Stdlib_Option.getOr(a.creationRound, 1), Stdlib_Option.getOr(b.creationRound, 1));
164
+ }
165
+ });
166
+ }
167
+
168
+ async function runResponsePassesOrThrow(eventItems, through, runtime, chainState) {
169
+ let registrations = [];
170
+ let eventIdx = 0;
171
+ let passedThrough;
172
+ let after = Stdlib_Option.getOr(ChainState.blockRegisterThrough(chainState), through);
173
+ let isDone = false;
174
+ while (!isDone) {
175
+ let match = ChainState.blockRegisterItemsAfter(chainState, after, through, 10000);
176
+ let match$1 = match !== undefined ? [
177
+ match[0],
178
+ match[1]
179
+ ] : [
180
+ [],
181
+ undefined
182
+ ];
183
+ let covered = match$1[1];
184
+ let blockItems = match$1[0];
185
+ let eventEnd;
186
+ if (covered !== undefined) {
187
+ if (covered < through) {
188
+ let idx = eventIdx;
189
+ while (idx < eventItems.length && eventItems[idx].blockNumber <= covered) {
190
+ idx = idx + 1 | 0;
191
+ };
192
+ eventEnd = idx;
193
+ } else {
194
+ eventEnd = eventItems.length;
195
+ }
196
+ } else {
197
+ eventEnd = eventItems.length;
198
+ }
199
+ let startedAt = Date.now();
200
+ let windowRegistrations = await runRegisterPassesOrThrow(eventItems.slice(eventIdx, eventEnd), blockItems, runtime, chainState);
201
+ Logger.trace(ChainState.logger(chainState), "Ran register passes with block passes", {
202
+ after: after,
203
+ through: covered,
204
+ blockPasses: blockItems.length,
205
+ eventPasses: eventEnd - eventIdx | 0,
206
+ registrations: windowRegistrations.length,
207
+ ms: Date.now() - startedAt
208
+ });
209
+ registrations.push(...windowRegistrations);
210
+ eventIdx = eventEnd;
211
+ if (covered !== undefined) {
212
+ passedThrough = covered;
213
+ after = covered;
214
+ isDone = covered >= through && eventIdx >= eventItems.length;
215
+ } else {
216
+ isDone = true;
217
+ }
218
+ };
219
+ return [
220
+ sortRegistrations(registrations),
221
+ passedThrough
222
+ ];
223
+ }
224
+
225
+ let registerCallsPerItem = (() => {
226
+ const cost = globalThis.__envioSubgraphRegisterCostPerItem
227
+ const value = cost ? cost() : undefined
228
+ return value === undefined ? undefined : value
229
+ });
230
+
231
+ let reportRegisterFrontier = ((chainId, block) => {
232
+ const report = globalThis.__envioSubgraphRegisterFrontier
233
+ if (report) report(chainId, block)
234
+ });
235
+
236
+ function registerChunkLimit() {
237
+ if (!Env.subgraphMode.contents) {
238
+ return;
239
+ }
240
+ if (Env.subgraphRegisterChunk !== undefined) {
241
+ if (Env.subgraphRegisterChunk > 0) {
242
+ return Env.subgraphRegisterChunk;
243
+ } else {
244
+ return;
245
+ }
246
+ }
247
+ let cost = registerCallsPerItem();
248
+ if (cost !== undefined) {
249
+ if (cost < 0.5) {
250
+ return;
251
+ } else {
252
+ return Primitive_int.max(100, 3000 / cost | 0);
253
+ }
254
+ } else {
255
+ return 100;
256
+ }
257
+ }
258
+
259
+ function cutAtRegisterLimit(items, latestFetchedBlock, limit) {
260
+ let seen = 0;
261
+ let cutIdx = -1;
262
+ let idx = 0;
263
+ while (cutIdx < 0 && idx < items.length) {
264
+ if (hasContractRegister(items[idx])) {
265
+ seen = seen + 1 | 0;
266
+ if (seen >= limit) {
267
+ cutIdx = idx;
268
+ }
269
+ }
270
+ idx = idx + 1 | 0;
271
+ };
272
+ if (cutIdx < 0) {
273
+ return [
274
+ items,
275
+ latestFetchedBlock
276
+ ];
277
+ }
278
+ let cutBlock = items[cutIdx].blockNumber;
279
+ let last = cutIdx;
280
+ while ((last + 1 | 0) < items.length && items[last + 1 | 0].blockNumber <= cutBlock) {
281
+ last = last + 1 | 0;
282
+ };
283
+ if ((last + 1 | 0) >= items.length && cutBlock >= latestFetchedBlock) {
284
+ return [
285
+ items,
286
+ latestFetchedBlock
287
+ ];
288
+ } else {
289
+ return [
290
+ items.slice(0, last + 1 | 0),
291
+ cutBlock
292
+ ];
293
+ }
294
+ }
295
+
67
296
  function applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query) {
68
297
  let chainState = IndexerState.getChainState(state, chainId);
69
298
  ChainState.handleQueryResult(chainState, query, newItems, newRegistrations, latestFetchedBlock, knownHeight);
299
+ if (Env.subgraphMode.contents) {
300
+ let fetchState = ChainState.fetchState(chainState);
301
+ let frontier = FetchState.OptimizedPartitions.getLatestFullyFetchedBlock(fetchState.optimizedPartitions);
302
+ if (frontier !== undefined) {
303
+ let through = fetchState.blockRegisterThrough;
304
+ let frontier$1 = through !== undefined ? Primitive_int.min(frontier, through) : frontier;
305
+ reportRegisterFrontier(chainId, frontier$1);
306
+ }
307
+ }
70
308
  if (!IndexerState.exitAfterFirstEventBlock(state)) {
71
309
  return;
72
310
  }
@@ -89,8 +327,15 @@ async function onQueryResponse(state, param, runtime, stateId, scheduleFetch, sc
89
327
  let transactionStore = response.transactionStore;
90
328
  let parsedQueueItems = response.parsedQueueItems;
91
329
  let knownHeight = response.knownHeight;
92
- ChainState.recordBlockRangeFetch(chainState, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - query.fromBlock | 0) + 1 | 0);
93
- let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems, 0, (count, item) => {
330
+ let limit = registerChunkLimit();
331
+ let match = limit !== undefined ? cutAtRegisterLimit(parsedQueueItems, latestFetchedBlockNumber, limit) : [
332
+ parsedQueueItems,
333
+ latestFetchedBlockNumber
334
+ ];
335
+ let latestFetchedBlockNumber$1 = match[1];
336
+ let parsedQueueItems$1 = match[0];
337
+ ChainState.recordBlockRangeFetch(chainState, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems$1.length, (latestFetchedBlockNumber$1 - query.fromBlock | 0) + 1 | 0);
338
+ let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems$1, 0, (count, item) => {
94
339
  if (hasContractRegister(item)) {
95
340
  return count + 1 | 0;
96
341
  } else {
@@ -101,15 +346,15 @@ async function onQueryResponse(state, param, runtime, stateId, scheduleFetch, sc
101
346
  Logger.trace(ChainState.logger(chainState), "Finished querying", {
102
347
  partitionId: query.partitionId,
103
348
  fromBlock: query.fromBlock,
104
- toBlock: latestFetchedBlockNumber,
105
- numEvents: parsedQueueItems.length
349
+ toBlock: latestFetchedBlockNumber$1,
350
+ numEvents: parsedQueueItems$1.length
106
351
  });
107
352
  } else {
108
353
  Logger.trace(ChainState.logger(chainState), "Finished querying", {
109
354
  partitionId: query.partitionId,
110
355
  fromBlock: query.fromBlock,
111
- toBlock: latestFetchedBlockNumber,
112
- numEvents: parsedQueueItems.length,
356
+ toBlock: latestFetchedBlockNumber$1,
357
+ numEvents: parsedQueueItems$1.length,
113
358
  numContractRegisterEvents: numContractRegisterEvents
114
359
  });
115
360
  }
@@ -118,22 +363,22 @@ async function onQueryResponse(state, param, runtime, stateId, scheduleFetch, sc
118
363
  if (typeof reorgResult !== "object") {
119
364
  rollbackWithReorgDetectedBlockNumber = undefined;
120
365
  } else {
121
- let match = reorgResult._0;
122
- let scannedBlock = match.scannedBlock;
366
+ let match$1 = reorgResult._0;
367
+ let scannedBlock = match$1.scannedBlock;
123
368
  Logger.info(ChainState.logger(chainState), `Blockchain reorg detected. ` + (
124
369
  ChainState.shouldRollbackOnReorg(chainState) ? "Initiating indexer rollback" : "NOT initiating indexer rollback due to configuration"
125
370
  ) + `.`, {
126
371
  blockNumber: scannedBlock.blockNumber,
127
372
  indexedBlockHash: scannedBlock.blockHash,
128
- receivedBlockHash: match.receivedBlock.blockHash
373
+ receivedBlockHash: match$1.receivedBlock.blockHash
129
374
  });
130
375
  ChainState.recordReorgDetected(chainState, scannedBlock.blockNumber);
131
376
  rollbackWithReorgDetectedBlockNumber = ChainState.shouldRollbackOnReorg(chainState) ? scannedBlock.blockNumber : undefined;
132
377
  }
133
378
  if (rollbackWithReorgDetectedBlockNumber !== undefined) {
134
- let match$1 = IndexerState.rollbackState(state);
379
+ let match$2 = IndexerState.rollbackState(state);
135
380
  let eventsProcessedDiffByChain;
136
- eventsProcessedDiffByChain = typeof match$1 !== "object" || match$1.TAG !== "RollbackReady" ? undefined : match$1.eventsProcessedDiffByChain;
381
+ eventsProcessedDiffByChain = typeof match$2 !== "object" || match$2.TAG !== "RollbackReady" ? undefined : match$2.eventsProcessedDiffByChain;
137
382
  Utils.Dict.forEach(IndexerState.chainStates(state), cs => ChainState.prepareReorg(cs, eventsProcessedDiffByChain !== undefined ? eventsProcessedDiffByChain[ChainState.chainConfig(cs).id] : undefined));
138
383
  IndexerState.beginReorg(state, chainId, rollbackWithReorgDetectedBlockNumber);
139
384
  return scheduleRollback();
@@ -142,31 +387,86 @@ async function onQueryResponse(state, param, runtime, stateId, scheduleFetch, sc
142
387
  ChainState.transactionStore(chainState).merge(Primitive_option.valFromOption(transactionStore));
143
388
  }
144
389
  let itemsWithContractRegister = [];
145
- for (let idx = 0, idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx) {
146
- let item = parsedQueueItems[idx];
390
+ for (let idx = 0, idx_finish = parsedQueueItems$1.length; idx < idx_finish; ++idx) {
391
+ let item = parsedQueueItems$1[idx];
147
392
  if (hasContractRegister(item)) {
148
393
  itemsWithContractRegister.push(item);
149
394
  }
150
395
  }
151
- let proceed = newRegistrations => {
396
+ let proceed = (newRegistrations, blockRegisterThrough, newItemsOpt, latestFetchedBlockOpt) => {
397
+ let newItems = newItemsOpt !== undefined ? newItemsOpt : parsedQueueItems$1;
398
+ let latestFetchedBlock = latestFetchedBlockOpt !== undefined ? latestFetchedBlockOpt : latestFetchedBlockNumber$1;
152
399
  if (!IndexerState.isStale(state, stateId)) {
153
- applyQueryResponse(state, chainId, parsedQueueItems, newRegistrations, knownHeight, latestFetchedBlockNumber, query);
400
+ applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query);
401
+ if (blockRegisterThrough !== undefined) {
402
+ ChainState.markBlockRegistered(chainState, blockRegisterThrough);
403
+ }
154
404
  ChainMetadata.stage(state);
155
405
  scheduleFetch();
156
406
  return scheduleProcessing();
157
407
  }
158
408
  };
409
+ let blockRegisterCeiling = Primitive_int.min(latestFetchedBlockNumber$1, Primitive_int.max(knownHeight, ChainState.knownHeight(chainState)));
410
+ let withBlockPasses = ChainState.hasBlockRegisters(chainState) && (Utils.$$Array.notEmpty(itemsWithContractRegister) || ChainState.canAdvanceBlockRegisters(chainState, blockRegisterCeiling));
411
+ if (withBlockPasses) {
412
+ let match$3 = ChainState.queueBlockRegisterRun(chainState);
413
+ let finish = match$3[1];
414
+ let held = ChainState.holdExtraBudget(chainState, Primitive_int.max(0, parsedQueueItems$1.length - query.itemsEst | 0));
415
+ await match$3[0];
416
+ if (IndexerState.isStale(state, stateId)) {
417
+ ChainState.releaseExtraBudget(chainState, held);
418
+ finish();
419
+ return scheduleFetch();
420
+ }
421
+ let match$4 = ChainState.blockRegisterItemsAfter(chainState, Primitive_int.max(Stdlib_Option.getOr(ChainState.blockRegisterThrough(chainState), blockRegisterCeiling), query.fromBlock - 1 | 0), blockRegisterCeiling, 50000);
422
+ let cutAt;
423
+ if (match$4 !== undefined) {
424
+ let covered = match$4[1];
425
+ cutAt = covered < blockRegisterCeiling ? covered : undefined;
426
+ } else {
427
+ cutAt = undefined;
428
+ }
429
+ let through = Stdlib_Option.getOr(cutAt, blockRegisterCeiling);
430
+ let match$5;
431
+ if (cutAt !== undefined) {
432
+ let isKept = item => item.blockNumber <= through;
433
+ match$5 = [
434
+ parsedQueueItems$1.filter(isKept),
435
+ itemsWithContractRegister.filter(isKept),
436
+ through
437
+ ];
438
+ } else {
439
+ match$5 = [
440
+ parsedQueueItems$1,
441
+ itemsWithContractRegister,
442
+ latestFetchedBlockNumber$1
443
+ ];
444
+ }
445
+ let val;
446
+ try {
447
+ val = await runResponsePassesOrThrow(match$5[1], through, runtime, chainState);
448
+ } catch (raw_exn) {
449
+ let exn = Primitive_exceptions.internalToException(raw_exn);
450
+ finish();
451
+ return IndexerState.errorExit(state, ErrorHandling.make(exn, ChainState.logger(chainState), undefined));
452
+ }
453
+ ChainState.releaseExtraBudget(chainState, held);
454
+ proceed(val[0], val[1], match$5[0], match$5[2]);
455
+ return finish();
456
+ }
159
457
  if (itemsWithContractRegister.length === 0) {
160
- return proceed([]);
458
+ return proceed([], undefined, undefined, undefined);
161
459
  }
460
+ let held$1 = Env.subgraphMode.contents ? Primitive_option.some(ChainState.holdExtraBudget(chainState, Primitive_int.max(0, parsedQueueItems$1.length - query.itemsEst | 0))) : undefined;
162
461
  let newRegistrations;
163
462
  try {
164
463
  newRegistrations = await runContractRegistersOrThrow(itemsWithContractRegister, runtime, chainState);
165
- } catch (raw_exn) {
166
- let exn = Primitive_exceptions.internalToException(raw_exn);
167
- return IndexerState.errorExit(state, ErrorHandling.make(exn, ChainState.logger(chainState), undefined));
464
+ } catch (raw_exn$1) {
465
+ let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
466
+ return IndexerState.errorExit(state, ErrorHandling.make(exn$1, ChainState.logger(chainState), undefined));
168
467
  }
169
- return proceed(newRegistrations);
468
+ Stdlib_Option.forEach(held$1, held => ChainState.releaseExtraBudget(chainState, held));
469
+ return proceed(newRegistrations, undefined, undefined, undefined);
170
470
  }
171
471
 
172
472
  function finishWaitingForNewBlock(state, chainId, knownHeight, stateId, scheduleFetch, scheduleProcessing) {
@@ -179,11 +479,55 @@ function finishWaitingForNewBlock(state, chainId, knownHeight, stateId, schedule
179
479
  scheduleProcessing();
180
480
  }
181
481
 
482
+ async function catchUpBlockRegisters(state, chainId, runtime, stateId, scheduleFetch, scheduleProcessing) {
483
+ let chainState = IndexerState.getChainState(state, chainId);
484
+ if (!ChainState.isBlockRegisterIdle(chainState)) {
485
+ return;
486
+ }
487
+ let match = ChainState.nextBlockRegisterItems(chainState, ChainState.blockRegisterFrontier(chainState), 10000);
488
+ if (match === undefined) {
489
+ return;
490
+ }
491
+ let through = match[1];
492
+ let blockItems = match[0];
493
+ let match$1 = ChainState.queueBlockRegisterRun(chainState);
494
+ let finish = match$1[1];
495
+ let startedAt = Date.now();
496
+ let registrations;
497
+ try {
498
+ registrations = await runRegisterPassesOrThrow([], blockItems, runtime, chainState);
499
+ } catch (raw_exn) {
500
+ let exn = Primitive_exceptions.internalToException(raw_exn);
501
+ finish();
502
+ return IndexerState.errorExit(state, ErrorHandling.make(exn, ChainState.logger(chainState), undefined));
503
+ }
504
+ let isStale = IndexerState.isStale(state, stateId);
505
+ Logger.trace(ChainState.logger(chainState), "Ran block passes", {
506
+ through: through,
507
+ blockPasses: blockItems.length,
508
+ registrations: registrations.length,
509
+ stale: isStale,
510
+ ms: Date.now() - startedAt
511
+ });
512
+ if (!isStale) {
513
+ ChainState.applyBlockRegisters(chainState, sortRegistrations(registrations), through);
514
+ }
515
+ finish();
516
+ if (!isStale) {
517
+ ChainMetadata.stage(state);
518
+ scheduleProcessing();
519
+ }
520
+ return scheduleFetch();
521
+ }
522
+
182
523
  async function fetchChain(state, chainId, runtime, action, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
183
524
  let chainState = IndexerState.getChainState(state, chainId);
184
525
  if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
185
526
  return;
186
527
  }
528
+ if (ChainState.hasBlockRegisters(chainState)) {
529
+ catchUpBlockRegisters(state, chainId, runtime, stateId, scheduleFetch, scheduleProcessing);
530
+ }
187
531
  let isRealtime = IndexerState.isRealtime(state);
188
532
  let sourceManager = ChainState.sourceManager(chainState);
189
533
  let reducedPolling = !isRealtime && ChainState.knownHeight(chainState) > 0;
@@ -207,13 +551,40 @@ async function fetchChain(state, chainId, runtime, action, stateId, scheduleFetc
207
551
  }
208
552
  }
209
553
 
554
+ let blockRegisterChunk = 2048;
555
+
556
+ let blockRegisterWindow = 10000;
557
+
558
+ let maxResponseBlockPasses = 50000;
559
+
560
+ let registerCallsPerChunk = 3000;
561
+
562
+ let unknownCostChunk = 100;
563
+
564
+ let minRegisterChunk = 100;
565
+
210
566
  export {
211
567
  hasContractRegister,
212
568
  invokeContractRegister,
213
569
  runContractRegistersOrThrow,
570
+ mergeRegisterItems,
571
+ blockRegisterChunk,
572
+ blockRegisterWindow,
573
+ maxResponseBlockPasses,
574
+ runRegisterPassesOrThrow,
575
+ sortRegistrations,
576
+ runResponsePassesOrThrow,
577
+ registerCallsPerItem,
578
+ reportRegisterFrontier,
579
+ registerCallsPerChunk,
580
+ unknownCostChunk,
581
+ minRegisterChunk,
582
+ registerChunkLimit,
583
+ cutAtRegisterLimit,
214
584
  onQueryResponse,
215
585
  applyQueryResponse,
216
586
  finishWaitingForNewBlock,
587
+ catchUpBlockRegisters,
217
588
  fetchChain,
218
589
  }
219
- /* Utils Not a pure module */
590
+ /* Env Not a pure module */