envio 3.12.0 → 3.13.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/package.json +6 -6
  2. package/src/BatchProcessing.res +9 -9
  3. package/src/BatchProcessing.res.mjs +5 -5
  4. package/src/Bin.res +24 -17
  5. package/src/Bin.res.mjs +5 -0
  6. package/src/ChainFetching.res +31 -13
  7. package/src/ChainFetching.res.mjs +9 -4
  8. package/src/ChainState.res +55 -3
  9. package/src/ChainState.res.mjs +46 -3
  10. package/src/ChainState.resi +3 -1
  11. package/src/Config.res +35 -0
  12. package/src/Config.res.mjs +39 -0
  13. package/src/Core.res +4 -0
  14. package/src/Core.res.mjs +4 -0
  15. package/src/CrossChainState.res +61 -4
  16. package/src/CrossChainState.res.mjs +39 -5
  17. package/src/CrossChainState.resi +10 -1
  18. package/src/Env.res +4 -0
  19. package/src/FetchState.res +51 -46
  20. package/src/FetchState.res.mjs +54 -36
  21. package/src/InMemoryTable.res +155 -67
  22. package/src/InMemoryTable.res.mjs +151 -60
  23. package/src/IndexerLoop.res +2 -0
  24. package/src/IndexerLoop.res.mjs +1 -0
  25. package/src/IndexerState.res +41 -1
  26. package/src/IndexerState.res.mjs +43 -4
  27. package/src/IndexerState.resi +10 -0
  28. package/src/LoadLayer.res +57 -34
  29. package/src/LoadLayer.res.mjs +45 -62
  30. package/src/LoadLayer.resi +1 -1
  31. package/src/Logging.res +38 -6
  32. package/src/Logging.res.mjs +32 -5
  33. package/src/Main.res +131 -268
  34. package/src/Main.res.mjs +28 -152
  35. package/src/Metrics.res +263 -102
  36. package/src/Metrics.res.mjs +227 -48
  37. package/src/Persistence.res +27 -2
  38. package/src/Persistence.res.mjs +9 -2
  39. package/src/PgStorage.res +109 -71
  40. package/src/PgStorage.res.mjs +88 -58
  41. package/src/Server.res +181 -0
  42. package/src/Server.res.mjs +143 -0
  43. package/src/Supervisor.res +415 -0
  44. package/src/Supervisor.res.mjs +325 -0
  45. package/src/TestIndexer.res +9 -25
  46. package/src/TestIndexer.res.mjs +5 -4
  47. package/src/UserContext.res +13 -32
  48. package/src/UserContext.res.mjs +1 -7
  49. package/src/Utils.res +1 -4
  50. package/src/Utils.res.mjs +7 -16
  51. package/src/Worker.res +95 -0
  52. package/src/Worker.res.mjs +80 -0
  53. package/src/bindings/NodeJs.res +41 -0
  54. package/src/db/EntityFilter.res +487 -275
  55. package/src/db/EntityFilter.res.mjs +557 -309
  56. package/src/db/InternalTable.res +8 -1
  57. package/src/db/InternalTable.res.mjs +5 -1
  58. package/src/db/Table.res +21 -6
  59. package/src/db/Table.res.mjs +13 -4
  60. package/src/sources/BlockStore.res +7 -2
  61. package/src/sources/EvmHyperSyncSource.res +2 -0
  62. package/src/sources/EvmHyperSyncSource.res.mjs +2 -2
  63. package/src/sources/FuelHyperSyncSource.res +1 -0
  64. package/src/sources/FuelHyperSyncSource.res.mjs +1 -1
  65. package/src/sources/HyperSync.res +4 -0
  66. package/src/sources/HyperSync.res.mjs +4 -2
  67. package/src/sources/HyperSync.resi +1 -0
  68. package/src/sources/HyperSyncClient.res +3 -0
  69. package/src/sources/HyperSyncSSE.res +1 -1
  70. package/src/sources/HyperSyncSSE.res.mjs +4 -10
  71. package/src/sources/RpcSource.res +1 -0
  72. package/src/sources/RpcSource.res.mjs +1 -1
  73. package/src/sources/SimulateSource.res +1 -0
  74. package/src/sources/SimulateSource.res.mjs +1 -1
  75. package/src/sources/Source.res +7 -0
  76. package/src/sources/SourceManager.res +4 -3
  77. package/src/sources/SourceManager.res.mjs +2 -2
  78. package/src/sources/SvmHyperSyncClient.res +5 -0
  79. package/src/sources/SvmHyperSyncSource.res +3 -0
  80. package/src/sources/SvmHyperSyncSource.res.mjs +4 -2
  81. package/src/tui/Tui.res +24 -0
  82. package/src/tui/Tui.res.mjs +18 -0
  83. package/src/tui/components/SyncETA.res +12 -6
  84. package/src/tui/components/SyncETA.res.mjs +12 -8
@@ -0,0 +1,325 @@
1
+ // Generated by ReScript, PLEASE EDIT WITH CARE
2
+
3
+ import * as Env from "./Env.res.mjs";
4
+ import * as Tui from "./tui/Tui.res.mjs";
5
+ import * as Utils from "./Utils.res.mjs";
6
+ import * as Config from "./Config.res.mjs";
7
+ import * as Server from "./Server.res.mjs";
8
+ import * as Worker from "./Worker.res.mjs";
9
+ import * as ChainId from "./ChainId.res.mjs";
10
+ import * as Logging from "./Logging.res.mjs";
11
+ import * as Metrics from "./Metrics.res.mjs";
12
+ import * as Process from "process";
13
+ import * as ChainMap from "./ChainMap.res.mjs";
14
+ import * as PgStorage from "./PgStorage.res.mjs";
15
+ import * as Performance from "./bindings/Performance.res.mjs";
16
+ import * as Persistence from "./Persistence.res.mjs";
17
+ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
18
+ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
19
+ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
20
+ import * as Child_process from "child_process";
21
+ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
22
+ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
23
+ import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
24
+
25
+ function plan(chainIds, maxConnections) {
26
+ let workerCount = Primitive_int.min(chainIds.length, maxConnections / 2 | 0);
27
+ if (workerCount < 2) {
28
+ return;
29
+ }
30
+ let evenShare = Primitive_int.div(maxConnections, workerCount);
31
+ let remainder = Primitive_int.mod_(maxConnections, workerCount);
32
+ return Stdlib_Array.fromInitializer(workerCount, workerIndex => ({
33
+ chainIds: chainIds.filter((param, dealIndex) => {
34
+ let position = Primitive_int.mod_(dealIndex, workerCount);
35
+ let isReversePass = Primitive_int.div(dealIndex, workerCount) % 2 === 1;
36
+ return (
37
+ isReversePass ? (workerCount - 1 | 0) - position | 0 : position
38
+ ) === workerIndex;
39
+ }),
40
+ maxConnections: evenShare + (
41
+ workerIndex < remainder ? 1 : 0
42
+ ) | 0
43
+ }));
44
+ }
45
+
46
+ function planForRun(config, maxConnectionsOpt) {
47
+ let maxConnections = maxConnectionsOpt !== undefined ? maxConnectionsOpt : Env.Db.maxConnections;
48
+ if (config.isolated || !Config.isPerChain(config)) {
49
+ return;
50
+ } else {
51
+ return plan(ChainMap.values(config.chainMap).map(chain => chain.id), maxConnections);
52
+ }
53
+ }
54
+
55
+ function name(worker) {
56
+ return worker.chainIds.map(ChainId.toString).join(";");
57
+ }
58
+
59
+ function label(worker) {
60
+ return `[chain ` + name(worker) + `]`;
61
+ }
62
+
63
+ function logFilePath(workerIndex, pathOpt) {
64
+ let path = pathOpt !== undefined ? pathOpt : Env.logFilePath;
65
+ let suffix = `.worker-` + workerIndex.toString();
66
+ let dot = path.lastIndexOf(".");
67
+ if (dot > path.lastIndexOf("/")) {
68
+ return path.slice(0, dot) + suffix + path.slice(dot, path.length);
69
+ } else {
70
+ return path + suffix;
71
+ }
72
+ }
73
+
74
+ function readLines(onLine) {
75
+ let pending = {
76
+ contents: ""
77
+ };
78
+ let read = chunk => {
79
+ let parts = (pending.contents + chunk).split("\n");
80
+ pending.contents = Stdlib_Option.getOr(parts.pop(), "");
81
+ parts.forEach(onLine);
82
+ };
83
+ let flush = () => {
84
+ let line = pending.contents;
85
+ if (line === "") {
86
+ return;
87
+ }
88
+ pending.contents = "";
89
+ onLine(line);
90
+ };
91
+ return [
92
+ read,
93
+ flush
94
+ ];
95
+ }
96
+
97
+ function fork(worker, workerIndex, holdRealtime, entryPathOpt, pipeOutputOpt, onOutputOpt) {
98
+ let entryPath = entryPathOpt !== undefined ? entryPathOpt : process.argv[1];
99
+ let pipeOutput = pipeOutputOpt !== undefined ? pipeOutputOpt : false;
100
+ let onOutput = onOutputOpt !== undefined ? onOutputOpt : prim => {
101
+ console.log(prim);
102
+ };
103
+ let env = Object.assign({}, Process.env);
104
+ env[Worker.envVar] = S$RescriptSchema.reverseConvertToJsonStringOrThrow({
105
+ chainIds: worker.chainIds,
106
+ holdRealtime: holdRealtime
107
+ }, Worker.configSchema, undefined);
108
+ env["ENVIO_PG_MAX_CONNECTIONS"] = worker.maxConnections.toString();
109
+ env["LOG_FILE"] = logFilePath(workerIndex, undefined);
110
+ if (pipeOutput && Stdlib_Option.getOr((process.stdout.isTTY == null) ? undefined : Primitive_option.some(process.stdout.isTTY), false)) {
111
+ env["FORCE_COLOR"] = "1";
112
+ }
113
+ let child = Child_process.fork(entryPath, [], {
114
+ env: env,
115
+ serialization: "advanced",
116
+ stdio: pipeOutput ? [
117
+ "inherit",
118
+ "pipe",
119
+ "pipe",
120
+ "ipc"
121
+ ] : [
122
+ "inherit",
123
+ "inherit",
124
+ "inherit",
125
+ "ipc"
126
+ ]
127
+ });
128
+ if (pipeOutput) {
129
+ [
130
+ child.stdout,
131
+ child.stderr
132
+ ].forEach(stream => {
133
+ if (stream === null) {
134
+ return;
135
+ }
136
+ let match = readLines(onOutput);
137
+ stream.setEncoding("utf8");
138
+ stream.on("data", match[0]);
139
+ stream.on("end", match[1]);
140
+ });
141
+ }
142
+ let running = {
143
+ worker: worker,
144
+ child: child,
145
+ snapshot: undefined,
146
+ runtime: undefined,
147
+ settled: false
148
+ };
149
+ child.on("message", message => {
150
+ running.snapshot = message.metrics;
151
+ running.runtime = message.runtime;
152
+ });
153
+ return running;
154
+ }
155
+
156
+ function stop(group) {
157
+ group.stopping = true;
158
+ group.running.forEach(r => {
159
+ r.child.kill("SIGTERM");
160
+ });
161
+ }
162
+
163
+ let inFlight = {
164
+ contents: undefined
165
+ };
166
+
167
+ function syncCache(dump) {
168
+ let dumping = inFlight.contents;
169
+ if (dumping !== undefined) {
170
+ return dumping;
171
+ }
172
+ let dumping$1 = dump().finally(() => {
173
+ inFlight.contents = undefined;
174
+ });
175
+ inFlight.contents = dumping$1;
176
+ return dumping$1;
177
+ }
178
+
179
+ function dumpCache(config) {
180
+ let storage = PgStorage.makeStorageFromEnv(config, Primitive_option.some(PgStorage.makeClient(1)), undefined, undefined);
181
+ return storage.dumpEffectCache().finally(() => {
182
+ storage.close();
183
+ });
184
+ }
185
+
186
+ async function awaitExit(group) {
187
+ let failed = {
188
+ contents: false
189
+ };
190
+ let alive = {
191
+ contents: group.running.length
192
+ };
193
+ await new Promise((resolve, param) => {
194
+ let onGone = (r, failure) => {
195
+ if (!r.settled) {
196
+ r.settled = true;
197
+ if (failure) {
198
+ failed.contents = true;
199
+ if (!group.stopping) {
200
+ stop(group);
201
+ }
202
+ }
203
+ alive.contents = alive.contents - 1 | 0;
204
+ if (alive.contents === 0) {
205
+ return resolve();
206
+ } else {
207
+ return;
208
+ }
209
+ }
210
+ };
211
+ group.running.forEach(r => {
212
+ r.child.on("exit", (code, _signal) => onGone(r, !group.stopping && (
213
+ code === null ? undefined : Primitive_option.some(code)
214
+ ) !== 0));
215
+ r.child.on("error", exn => {
216
+ Logging.errorWithExn(exn, label(r.worker) + ` failed to start`);
217
+ onGone(r, true);
218
+ });
219
+ });
220
+ });
221
+ if (failed.contents) {
222
+ Stdlib_JsError.throwWithMessage("An indexer process exited with a failure. Stopped the others.");
223
+ }
224
+ if (group.stopping) {
225
+ return "Stopped";
226
+ } else {
227
+ return "Finished";
228
+ }
229
+ }
230
+
231
+ function isRunAtHead(snapshots, workerCount) {
232
+ if (snapshots.length === workerCount) {
233
+ return snapshots.every(snapshot => snapshot.hasArrivedAtHead);
234
+ } else {
235
+ return false;
236
+ }
237
+ }
238
+
239
+ async function run(config, workers, reset) {
240
+ let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
241
+ await Persistence.initForRun(persistence, config, reset, config.isDev, false);
242
+ await persistence.storage.close();
243
+ let startTime = new Date();
244
+ let startTimeRef = Performance.now();
245
+ Logging.info(`Splitting ` + ChainMap.values(config.chainMap).length.toString() + ` chains across ` + workers.length.toString() + ` processes, from a budget of ` + Env.Db.maxConnections.toString() + ` database connections.`);
246
+ let shouldUseTui = Tui.shouldUse(undefined);
247
+ let holdRealtime = Persistence.getInitializedState(persistence).chains.some(chain => Stdlib_Option.isNone(chain.timestampCaughtUpToHeadOrEndblock));
248
+ let group = {
249
+ running: workers.map((worker, workerIndex) => fork(worker, workerIndex, holdRealtime, undefined, shouldUseTui, undefined)),
250
+ stopping: false
251
+ };
252
+ let reported = () => Stdlib_Array.filterMap(group.running, r => r.snapshot);
253
+ let merge = snapshots => Metrics.merge(snapshots, startTime, new Date(), Performance.secondsSince(startTimeRef));
254
+ Server.startServer(() => {
255
+ let snapshots = reported();
256
+ if (snapshots.length !== 0) {
257
+ return merge(snapshots);
258
+ }
259
+ }, Utils.EnvioPackage.value.version, () => syncCache(() => dumpCache(config)), () => Metrics.renderRuntime(Stdlib_Array.filterMap(group.running, r => Stdlib_Option.map(r.runtime, runtime => [
260
+ `worker="` + name(r.worker) + `"`,
261
+ runtime
262
+ ]))), config.isDev);
263
+ let releaseCheck = {
264
+ contents: undefined
265
+ };
266
+ let stopReleaseCheck = () => {
267
+ Stdlib_Option.forEach(releaseCheck.contents, prim => {
268
+ clearInterval(prim);
269
+ });
270
+ releaseCheck.contents = undefined;
271
+ };
272
+ if (holdRealtime) {
273
+ releaseCheck.contents = Primitive_option.some(setInterval(() => {
274
+ if (isRunAtHead(reported(), group.running.length)) {
275
+ stopReleaseCheck();
276
+ group.running.forEach(r => {
277
+ r.child.send("release-realtime");
278
+ });
279
+ return Logging.info("Every chain has reached the head. Switching the run to realtime.");
280
+ }
281
+ }, 500));
282
+ }
283
+ if (shouldUseTui) {
284
+ Tui.start(config, () => merge(reported()));
285
+ }
286
+ process.on("SIGTERM", () => stop(group));
287
+ process.on("SIGINT", () => stop(group));
288
+ let outcome = await awaitExit(group);
289
+ stopReleaseCheck();
290
+ if (outcome === "Finished") {
291
+ if (shouldUseTui) {
292
+ process.on("SIGTERM", () => {
293
+ Process.exit(0);
294
+ });
295
+ process.on("SIGINT", () => {
296
+ Process.exit(0);
297
+ });
298
+ } else {
299
+ Logging.info("Exiting with success");
300
+ Process.exit(0);
301
+ }
302
+ return;
303
+ }
304
+ Process.exit(0);
305
+ }
306
+
307
+ let minConnectionsPerWorker = 2;
308
+
309
+ export {
310
+ minConnectionsPerWorker,
311
+ plan,
312
+ planForRun,
313
+ name,
314
+ label,
315
+ logFilePath,
316
+ readLines,
317
+ fork,
318
+ stop,
319
+ syncCache,
320
+ dumpCache,
321
+ awaitExit,
322
+ isRunAtHead,
323
+ run,
324
+ }
325
+ /* Env Not a pure module */
@@ -75,15 +75,10 @@ let handleLoad = (state: testIndexerState, ~tableName: string, ~filter: EntityFi
75
75
  | None => []
76
76
  | Some(entityConfig) =>
77
77
  let entityDict = state.entities->Dict.get(tableName)->Option.getOr(Dict.make())
78
- let matched =
79
- entityDict
80
- ->Dict.valuesToArray
81
- ->Array.filter(entity => {
82
- // The store holds decoded entities and the filter carries decoded values,
83
- // so compare directly (same approach as InMemoryTable) — no JSON round-trip.
84
- let entityAsDict = entity->(Utils.magic: Internal.entity => dict<EntityFilter.FieldValue.t>)
85
- filter->EntityFilter.matches(~entity=entityAsDict)
86
- })
78
+ // The store holds decoded entities and the filter carries decoded values,
79
+ // so compare directly (same approach as InMemoryTable) — no JSON round-trip.
80
+ let matcher = filter->EntityFilter.makeMatcher(~table=entityConfig.table)
81
+ let matched = entityDict->Dict.valuesToArray->Array.filter(matcher)
87
82
  // The chain is already fixed by the scope the load ran for, so the loaded
88
83
  // entity is handed back in the shape the handlers see.
89
84
  switch entityConfig.table->Table.getChainIdField {
@@ -544,23 +539,12 @@ let makeEntityGetWhere = (~state: testIndexerState, ~entityConfig: Internal.enti
544
539
  `Cannot call ${entityConfig.name}.getWhere() while indexer.process() is running. ` ++ "Wait for process() to complete before accessing entities directly.",
545
540
  )
546
541
  }
547
- let filters =
548
- filter->EntityFilter.parseGetWhereOrThrow(
549
- ~entityName=entityConfig.name,
550
- ~table=entityConfig.table,
551
- )
542
+ let matcher =
543
+ filter
544
+ ->EntityFilter.parseOrThrow(~entityName=entityConfig.name, ~table=entityConfig.table)
545
+ ->EntityFilter.makeMatcher(~table=entityConfig.table)
552
546
  let entityDict = state.entities->Dict.get(entityConfig.name)->Option.getOr(Dict.make())
553
- // parseGetWhereOrThrow expands an operator group into alternatives whose
554
- // matches are disjoint, so the union needs no dedup.
555
- Promise.resolve(
556
- entityDict
557
- ->Dict.valuesToArray
558
- ->Array.filter(entity => {
559
- let entityAsDict = entity->(Utils.magic: Internal.entity => dict<EntityFilter.FieldValue.t>)
560
- filters->Array.some(filter => filter->EntityFilter.matches(~entity=entityAsDict))
561
- })
562
- ->Array.map(copyEntity),
563
- )
547
+ Promise.resolve(entityDict->Dict.valuesToArray->Array.filter(matcher)->Array.map(copyEntity))
564
548
  }
565
549
  }
566
550
 
@@ -56,7 +56,8 @@ function handleLoad(state, tableName, filter) {
56
56
  return [];
57
57
  }
58
58
  let entityDict = Stdlib_Option.getOr(state.entities[tableName], {});
59
- let matched = Object.values(entityDict).filter(entity => EntityFilter.matches(filter, entity));
59
+ let matcher = EntityFilter.makeMatcher(filter, entityConfig.table);
60
+ let matched = Object.values(entityDict).filter(matcher);
60
61
  let field = Table.getChainIdField(entityConfig.table);
61
62
  if (field !== undefined) {
62
63
  return matched.map(entity => {
@@ -376,9 +377,9 @@ function makeEntityGetWhere(state, entityConfig) {
376
377
  if (state.processInProgress) {
377
378
  Stdlib_JsError.throwWithMessage(`Cannot call ` + entityConfig.name + `.getWhere() while indexer.process() is running. ` + "Wait for process() to complete before accessing entities directly.");
378
379
  }
379
- let filters = EntityFilter.parseGetWhereOrThrow(filter, entityConfig.name, entityConfig.table);
380
+ let matcher = EntityFilter.makeMatcher(EntityFilter.parseOrThrow(filter, entityConfig.name, entityConfig.table), entityConfig.table);
380
381
  let entityDict = Stdlib_Option.getOr(state.entities[entityConfig.name], {});
381
- return Promise.resolve(Object.values(entityDict).filter(entity => filters.some(filter => EntityFilter.matches(filter, entity))).map(copyEntity));
382
+ return Promise.resolve(Object.values(entityDict).filter(matcher).map(copyEntity));
382
383
  };
383
384
  }
384
385
 
@@ -638,7 +639,7 @@ function createTestIndexer() {
638
639
  };
639
640
  try {
640
641
  await new Promise((resolve, reject) => {
641
- let indexerState = IndexerState.makeFromDbState(runConfig, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, errHandler => {
642
+ let indexerState = IndexerState.makeFromDbState(runConfig, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, undefined, errHandler => {
642
643
  ErrorHandling.log(errHandler);
643
644
  reject(Utils.prettifyExn(errHandler.exn));
644
645
  }, () => resolve());
@@ -122,39 +122,20 @@ type entityContextParams = {
122
122
  // The handler context is always chain-scoped, so a per-chain entity resolves to
123
123
  // the chain the handler runs on and a cross-chain one to the shared partition.
124
124
  let entityScope = (params: entityContextParams) =>
125
- params.entityConfig->InMemoryStore.entityScope(
126
- ~chainId=params.item->Internal.getItemChainId,
127
- )
128
-
129
- let getWhereHandler = (params: entityContextParams, filter: dict<dict<unknown>>) => {
130
- let entityConfig = params.entityConfig
131
-
132
- @inline
133
- let loadWithFilter = filter =>
134
- LoadLayer.loadByFilter(
135
- ~loadManager=params.loadManager,
136
- ~persistence=params.persistence,
137
- ~entityConfig,
138
- ~scope=params->entityScope,
139
- ~indexerState=params.indexerState,
140
- ~shouldGroup=params.isPreload,
141
- ~item=params.item,
142
- ~ecosystem=params.config.ecosystem,
143
- ~filter,
144
- )
125
+ params.entityConfig->InMemoryStore.entityScope(~chainId=params.item->Internal.getItemChainId)
145
126
 
146
- switch filter->EntityFilter.parseGetWhereOrThrow(
147
- ~entityName=entityConfig.name,
148
- ~table=entityConfig.table,
149
- ) {
150
- | [single] => loadWithFilter(single)
151
- | filters =>
152
- filters
153
- ->Array.map(filter => loadWithFilter(filter))
154
- ->Promise.all
155
- ->Promise.thenResolve(results => results->Array.flat)
156
- }
157
- }
127
+ let getWhereHandler = (params: entityContextParams, filter: dict<dict<unknown>>) =>
128
+ LoadLayer.loadByFilter(
129
+ ~loadManager=params.loadManager,
130
+ ~persistence=params.persistence,
131
+ ~entityConfig=params.entityConfig,
132
+ ~scope=params->entityScope,
133
+ ~indexerState=params.indexerState,
134
+ ~shouldGroup=params.isPreload,
135
+ ~item=params.item,
136
+ ~ecosystem=params.config.ecosystem,
137
+ ~filter,
138
+ )
158
139
 
159
140
  let noopSet = (_entity: Internal.entity) => ()
160
141
  let noopDeleteUnsafe = (_entityId: EntityId.t) => ()
@@ -77,13 +77,7 @@ function entityScope(params) {
77
77
  }
78
78
 
79
79
  function getWhereHandler(params, filter) {
80
- let entityConfig = params.entityConfig;
81
- let filters = EntityFilter.parseGetWhereOrThrow(filter, entityConfig.name, entityConfig.table);
82
- if (filters.length !== 1) {
83
- return Promise.all(filters.map(filter => LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter))).then(results => results.flat());
84
- } else {
85
- return LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filters[0]);
86
- }
80
+ return LoadLayer.loadByFilter(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter);
87
81
  }
88
82
 
89
83
  function noopSet(_entity) {
package/src/Utils.res CHANGED
@@ -273,7 +273,7 @@ module Dict = {
273
273
  }
274
274
 
275
275
  module Math = {
276
- let minOptInt = (a, b) =>
276
+ let minOptInt = (a: option<int>, b: option<int>) =>
277
277
  switch (a, b) {
278
278
  | (Some(a), Some(b)) => Some(a < b ? a : b)
279
279
  | (Some(a), None) => Some(a)
@@ -374,9 +374,6 @@ module Array = {
374
374
  /**
375
375
  Helper to check if a value exists in an array
376
376
  */
377
- let includes = (arr: array<'a>, val: 'a) =>
378
- arr->Array.find(item => item == val)->Stdlib.Option.isSome
379
-
380
377
  let isEmpty = (arr: array<_>) =>
381
378
  switch arr {
382
379
  | [] => true
package/src/Utils.res.mjs CHANGED
@@ -6,11 +6,9 @@ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
6
6
  import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
7
7
  import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
8
8
  import * as Stdlib_BigInt from "@rescript/runtime/lib/es6/Stdlib_BigInt.js";
9
- import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
10
9
  import * as Stdlib_String from "@rescript/runtime/lib/es6/Stdlib_String.js";
11
10
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
12
11
  import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
13
- import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
14
12
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
15
13
  import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
16
14
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -221,19 +219,17 @@ let Dict = {
221
219
  };
222
220
 
223
221
  function minOptInt(a, b) {
224
- if (a === undefined) {
222
+ if (a !== undefined) {
225
223
  if (b !== undefined) {
226
- return Primitive_option.some(Primitive_option.valFromOption(b));
224
+ return a < b ? a : b;
227
225
  } else {
228
- return;
226
+ return a;
229
227
  }
228
+ } else if (b !== undefined) {
229
+ return b;
230
+ } else {
231
+ return;
230
232
  }
231
- let a$1 = Primitive_option.valFromOption(a);
232
- if (b === undefined) {
233
- return Primitive_option.some(a$1);
234
- }
235
- let b$1 = Primitive_option.valFromOption(b);
236
- return Primitive_option.some(Primitive_object.lessthan(a$1, b$1) ? a$1 : b$1);
237
233
  }
238
234
 
239
235
  let $$Math = {
@@ -324,10 +320,6 @@ function transposeResults(results) {
324
320
  };
325
321
  }
326
322
 
327
- function includes(arr, val) {
328
- return Stdlib_Option.isSome(arr.find(item => Primitive_object.equal(item, val)));
329
- }
330
-
331
323
  function isEmpty$1(arr) {
332
324
  return arr.length === 0;
333
325
  }
@@ -435,7 +427,6 @@ let $$Array$1 = {
435
427
  clearInPlace: clearInPlace$1,
436
428
  setIndexImmutable: setIndexImmutable,
437
429
  transposeResults: transposeResults,
438
- includes: includes,
439
430
  isEmpty: isEmpty$1,
440
431
  notEmpty: notEmpty,
441
432
  awaitEach: awaitEach,
package/src/Worker.res ADDED
@@ -0,0 +1,95 @@
1
+ // The worker side of a supervised run: a process the supervisor forked to drive
2
+ // a subset of the chains. It has no server and no TUI of its own — it reports
3
+ // through the IPC channel, and the supervisor is the one operational surface.
4
+
5
+ // Set by a supervisor in the environment of the workers it forks. Internal:
6
+ // it counts only together with the fork's own channel, so a copy left in a
7
+ // shell starts nothing, and an indexer a user starts themselves takes every
8
+ // path it takes today.
9
+ let envVar = "ENVIO_INTERNAL_WORKER"
10
+
11
+ // What the supervisor decided about this worker, handed over in the spawn
12
+ // environment rather than over the channel: it is settled before the process
13
+ // starts, and the worker needs it before it can load its own config.
14
+ type config = {
15
+ chainIds: array<ChainId.t>,
16
+ // The chains this worker drives may reach the head while chains in another
17
+ // process are still backfilling, and an indexer goes realtime as a whole or
18
+ // not at all. Cleared by the supervisor's `ReleaseRealtime`.
19
+ holdRealtime: bool,
20
+ }
21
+
22
+ let configSchema = S.object((s): config => {
23
+ chainIds: s.field("chainIds", S.array(ChainId.schema)),
24
+ holdRealtime: s.fieldOr("holdRealtime", S.bool, false),
25
+ })
26
+
27
+ // Read as this module loads, which is before anything that could catch a bare
28
+ // schema error and say where it came from.
29
+ let detect = (~env: dict<string>, ~hasChannel) =>
30
+ switch (hasChannel, env->Dict.get(envVar)) {
31
+ | (true, Some(json)) =>
32
+ switch json->S.parseJsonStringOrThrow(configSchema) {
33
+ | config => Some(config)
34
+ | exception S.Raised(error) =>
35
+ JsError.throwWithMessage(
36
+ `Invalid ${envVar}: ${error->S.Error.message}. It is set by an indexer supervisor for the processes it forks, and isn't meant to be set by hand.`,
37
+ )
38
+ }
39
+ | _ => None
40
+ }
41
+
42
+ let config = detect(
43
+ ~env=NodeJs.Process.process.env,
44
+ ~hasChannel=NodeJs.Process.channel->Nullable.toOption->Option.isSome,
45
+ )
46
+
47
+ let isEnabled = config->Option.isSome
48
+
49
+ @tag("kind")
50
+ type parentMessage =
51
+ // Every chain in the run has reached the head, so this worker may enter the
52
+ // reorg threshold and switch to realtime with the rest of them.
53
+ | @as("release-realtime") ReleaseRealtime
54
+
55
+ @tag("kind")
56
+ type workerMessage =
57
+ | @as("snapshot") Snapshot({metrics: Metrics.t, runtime: Metrics.runtimeSample})
58
+
59
+ // How often a worker reports. Matches the TUI's own refresh, so the supervised
60
+ // display moves at the same rate an unsplit run's does.
61
+ %%private(let snapshotIntervalMillis = 500)
62
+
63
+ // The supervisor is the one that stops a worker, and the one whose absence
64
+ // ends it. A terminal's interrupt reaches the whole group at once, so the
65
+ // worker leaves it to the supervisor, which stops every worker in turn; without
66
+ // that, a worker gone on its own would read as a failure to the supervisor
67
+ // still deciding what the interrupt meant. A supervisor that dies can't tear
68
+ // the group down, so losing the channel is what ends the worker then.
69
+ let bindToSupervisor = () => {
70
+ NodeJs.Process.onSignal("SIGINT", () => ())
71
+ NodeJs.Process.onDisconnect(() => {
72
+ Logging.error("The indexer supervisor is gone. Stopping this chain's process.")
73
+ NodeJs.process->NodeJs.exitWithCode(Failure)
74
+ })
75
+ }
76
+
77
+ %%private(let send = (message: workerMessage) => NodeJs.Process.sendToParent(message)->ignore)
78
+
79
+ // Reports this process's chains and its own runtime for as long as it runs, so
80
+ // the supervisor can merge every worker's into the one snapshot the run serves,
81
+ // and listens for the one decision the supervisor makes on the run's behalf.
82
+ // Does nothing in a process nobody forked.
83
+ let bindRun = (~getMetrics: unit => Metrics.t, ~onReleaseRealtime: unit => unit) =>
84
+ if isEnabled {
85
+ Metrics.startRuntimeCollectors()
86
+ let _intervalId = setInterval(
87
+ () => send(Snapshot({metrics: getMetrics(), runtime: Metrics.sampleRuntime()})),
88
+ snapshotIntervalMillis,
89
+ )
90
+ NodeJs.Process.onMessage((message: parentMessage) =>
91
+ switch message {
92
+ | ReleaseRealtime => onReleaseRealtime()
93
+ }
94
+ )
95
+ }