envio 3.13.0-alpha.0 → 3.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -8
- package/src/BatchProcessing.res +4 -5
- package/src/BatchProcessing.res.mjs +4 -11
- package/src/ChainFetching.res +18 -38
- package/src/ChainFetching.res.mjs +14 -20
- package/src/ChainState.res +113 -77
- package/src/ChainState.res.mjs +90 -73
- package/src/ChainState.resi +14 -5
- package/src/Config.res +14 -17
- package/src/Config.res.mjs +5 -20
- package/src/CrossChainState.res +68 -20
- package/src/CrossChainState.res.mjs +48 -5
- package/src/CrossChainState.resi +1 -0
- package/src/Env.res +2 -2
- package/src/ErrorHandling.res +15 -4
- package/src/ErrorHandling.res.mjs +16 -5
- package/src/FinalizeBackfill.res +21 -9
- package/src/FinalizeBackfill.res.mjs +17 -4
- package/src/HandlerRegister.res +0 -37
- package/src/HandlerRegister.res.mjs +0 -18
- package/src/IndexerState.res +2 -0
- package/src/IndexerState.res.mjs +5 -0
- package/src/IndexerState.resi +1 -0
- package/src/Logging.res +6 -38
- package/src/Logging.res.mjs +5 -32
- package/src/Main.res +6 -9
- package/src/Main.res.mjs +3 -5
- package/src/Metrics.res +9 -4
- package/src/Metrics.res.mjs +3 -3
- package/src/Persistence.res +6 -3
- package/src/Persistence.res.mjs +5 -3
- package/src/PgStorage.res +2 -2
- package/src/PgStorage.res.mjs +2 -2
- package/src/Rollback.res +3 -3
- package/src/Server.res +0 -3
- package/src/SimulateItems.res +1 -1
- package/src/SimulateItems.res.mjs +1 -1
- package/src/Supervisor.res +236 -109
- package/src/Supervisor.res.mjs +185 -55
- package/src/TestIndexer.res +6 -3
- package/src/TestIndexer.res.mjs +3 -3
- package/src/Worker.res +6 -0
- package/src/Worker.res.mjs +2 -1
- package/src/bindings/NodeJs.res +25 -16
- package/src/bindings/NodeJs.res.mjs +8 -1
- package/src/sources/AddressSet.res +5 -3
- package/src/sources/AddressStore.res +19 -11
- package/src/sources/AddressStore.res.mjs +2 -7
- package/src/sources/BlockStore.res +10 -9
- package/src/sources/BlockStore.res.mjs +4 -4
- package/src/sources/ChainSources.res +4 -0
- package/src/sources/ChainSources.res.mjs +2 -2
- package/src/sources/EvmChain.res +4 -0
- package/src/sources/EvmChain.res.mjs +3 -1
- package/src/sources/EvmEventItem.res +56 -0
- package/src/sources/EvmEventItem.res.mjs +32 -0
- package/src/sources/EvmHyperSyncSource.res +7 -37
- package/src/sources/EvmHyperSyncSource.res.mjs +8 -29
- package/src/sources/EvmRpcClient.res +41 -59
- package/src/sources/EvmRpcClient.res.mjs +4 -54
- package/src/sources/EvmRpcWs.res +4 -1
- package/src/sources/EvmRpcWs.res.mjs +6 -2
- package/src/sources/FuelHyperSyncSource.res +6 -4
- package/src/sources/FuelHyperSyncSource.res.mjs +7 -9
- package/src/sources/HeightFeed.res +7 -1
- package/src/sources/HeightFeed.res.mjs +9 -7
- package/src/sources/HyperSyncClient.res +1 -14
- package/src/sources/RequestStat.res +7 -0
- package/src/sources/RequestStat.res.mjs +14 -1
- package/src/sources/RpcSource.res +100 -1043
- package/src/sources/RpcSource.res.mjs +81 -1056
- package/src/sources/SimulateSource.res +2 -5
- package/src/sources/SimulateSource.res.mjs +2 -3
- package/src/sources/Source.res +34 -12
- package/src/sources/Source.res.mjs +11 -0
- package/src/sources/SourceManager.res +111 -31
- package/src/sources/SourceManager.res.mjs +65 -31
- package/src/sources/SourceManager.resi +4 -0
- package/src/sources/StartBlockResolver.res +8 -5
- package/src/sources/StartBlockResolver.res.mjs +4 -2
- package/src/sources/SvmHyperSyncSource.res +4 -2
- package/src/sources/SvmHyperSyncSource.res.mjs +4 -6
- package/src/sources/TransactionStore.res +9 -6
- package/src/sources/TransactionStore.res.mjs +2 -2
- package/src/tui/Tui.res +4 -18
- package/src/tui/Tui.res.mjs +3 -6
- package/src/LazyLoader.res +0 -135
- package/src/LazyLoader.res.mjs +0 -118
- package/src/bindings/SDSL.res +0 -12
- package/src/bindings/SDSL.res.mjs +0 -9
- package/src/bindings/Yargs.res +0 -8
- package/src/bindings/Yargs.res.mjs +0 -2
- package/src/sources/Rpc.res +0 -185
- package/src/sources/Rpc.res.mjs +0 -183
package/src/Supervisor.res.mjs
CHANGED
|
@@ -19,11 +19,16 @@ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
|
19
19
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
20
20
|
import * as Child_process from "child_process";
|
|
21
21
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
22
|
+
import * as CrossChainState from "./CrossChainState.res.mjs";
|
|
22
23
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
23
24
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
24
25
|
|
|
25
26
|
function plan(chainIds, maxConnections) {
|
|
26
|
-
let workerCount =
|
|
27
|
+
let workerCount = Stdlib_Array.reduce([
|
|
28
|
+
chainIds.length,
|
|
29
|
+
maxConnections / 2 | 0,
|
|
30
|
+
4
|
|
31
|
+
], 4, Primitive_int.min);
|
|
27
32
|
if (workerCount < 2) {
|
|
28
33
|
return;
|
|
29
34
|
}
|
|
@@ -94,18 +99,42 @@ function readLines(onLine) {
|
|
|
94
99
|
];
|
|
95
100
|
}
|
|
96
101
|
|
|
97
|
-
function
|
|
102
|
+
function memoryBudgets(workerCount) {
|
|
103
|
+
return [
|
|
104
|
+
[
|
|
105
|
+
"ENVIO_INDEXING_MAX_BUFFER_SIZE",
|
|
106
|
+
CrossChainState.calculateTargetBufferSize()
|
|
107
|
+
],
|
|
108
|
+
[
|
|
109
|
+
"ENVIO_IN_MEMORY_OBJECTS_TARGET",
|
|
110
|
+
Env.inMemoryObjectsTarget | 0
|
|
111
|
+
]
|
|
112
|
+
].map(param => [
|
|
113
|
+
param[0],
|
|
114
|
+
Primitive_int.max(1, Primitive_int.div(param[1], workerCount)).toString()
|
|
115
|
+
]);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function fork(worker, workerIndex, workerCount, holdRealtime, isDev, entryPathOpt, pipeOutputOpt, onOutputOpt, onErrorOutputOpt, onSnapshotOpt) {
|
|
98
119
|
let entryPath = entryPathOpt !== undefined ? entryPathOpt : process.argv[1];
|
|
99
120
|
let pipeOutput = pipeOutputOpt !== undefined ? pipeOutputOpt : false;
|
|
100
121
|
let onOutput = onOutputOpt !== undefined ? onOutputOpt : prim => {
|
|
101
122
|
console.log(prim);
|
|
102
123
|
};
|
|
124
|
+
let onErrorOutput = onErrorOutputOpt !== undefined ? onErrorOutputOpt : prim => {
|
|
125
|
+
console.error(prim);
|
|
126
|
+
};
|
|
127
|
+
let onSnapshot = onSnapshotOpt !== undefined ? onSnapshotOpt : () => {};
|
|
103
128
|
let env = Object.assign({}, Process.env);
|
|
104
129
|
env[Worker.envVar] = S$RescriptSchema.reverseConvertToJsonStringOrThrow({
|
|
105
130
|
chainIds: worker.chainIds,
|
|
106
|
-
holdRealtime: holdRealtime
|
|
131
|
+
holdRealtime: holdRealtime,
|
|
132
|
+
isDev: isDev
|
|
107
133
|
}, Worker.configSchema, undefined);
|
|
108
134
|
env["ENVIO_PG_MAX_CONNECTIONS"] = worker.maxConnections.toString();
|
|
135
|
+
memoryBudgets(workerCount).forEach(param => {
|
|
136
|
+
env[param[0]] = param[1];
|
|
137
|
+
});
|
|
109
138
|
env["LOG_FILE"] = logFilePath(workerIndex, undefined);
|
|
110
139
|
if (pipeOutput && Stdlib_Option.getOr((process.stdout.isTTY == null) ? undefined : Primitive_option.some(process.stdout.isTTY), false)) {
|
|
111
140
|
env["FORCE_COLOR"] = "1";
|
|
@@ -127,13 +156,20 @@ function fork(worker, workerIndex, holdRealtime, entryPathOpt, pipeOutputOpt, on
|
|
|
127
156
|
});
|
|
128
157
|
if (pipeOutput) {
|
|
129
158
|
[
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
159
|
+
[
|
|
160
|
+
child.stdout,
|
|
161
|
+
onOutput
|
|
162
|
+
],
|
|
163
|
+
[
|
|
164
|
+
child.stderr,
|
|
165
|
+
onErrorOutput
|
|
166
|
+
]
|
|
167
|
+
].forEach(param => {
|
|
168
|
+
let stream = param[0];
|
|
133
169
|
if (stream === null) {
|
|
134
170
|
return;
|
|
135
171
|
}
|
|
136
|
-
let match = readLines(
|
|
172
|
+
let match = readLines(param[1]);
|
|
137
173
|
stream.setEncoding("utf8");
|
|
138
174
|
stream.on("data", match[0]);
|
|
139
175
|
stream.on("end", match[1]);
|
|
@@ -149,6 +185,7 @@ function fork(worker, workerIndex, holdRealtime, entryPathOpt, pipeOutputOpt, on
|
|
|
149
185
|
child.on("message", message => {
|
|
150
186
|
running.snapshot = message.metrics;
|
|
151
187
|
running.runtime = message.runtime;
|
|
188
|
+
onSnapshot();
|
|
152
189
|
});
|
|
153
190
|
return running;
|
|
154
191
|
}
|
|
@@ -183,6 +220,20 @@ function dumpCache(config) {
|
|
|
183
220
|
});
|
|
184
221
|
}
|
|
185
222
|
|
|
223
|
+
function classifyExit(code, signal, stopping) {
|
|
224
|
+
if (stopping) {
|
|
225
|
+
return "Expected";
|
|
226
|
+
}
|
|
227
|
+
if (code === 0) {
|
|
228
|
+
return "Expected";
|
|
229
|
+
}
|
|
230
|
+
if (signal === "SIGTERM") {
|
|
231
|
+
return "Stopping";
|
|
232
|
+
} else {
|
|
233
|
+
return "Failed";
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
186
237
|
async function awaitExit(group) {
|
|
187
238
|
let failed = {
|
|
188
239
|
contents: false
|
|
@@ -191,30 +242,32 @@ async function awaitExit(group) {
|
|
|
191
242
|
contents: group.running.length
|
|
192
243
|
};
|
|
193
244
|
await new Promise((resolve, param) => {
|
|
194
|
-
let onGone = (r,
|
|
195
|
-
if (
|
|
196
|
-
|
|
197
|
-
|
|
245
|
+
let onGone = (r, ending) => {
|
|
246
|
+
if (r.settled) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
r.settled = true;
|
|
250
|
+
switch (ending) {
|
|
251
|
+
case "Expected" :
|
|
252
|
+
break;
|
|
253
|
+
case "Stopping" :
|
|
254
|
+
stop(group);
|
|
255
|
+
break;
|
|
256
|
+
case "Failed" :
|
|
198
257
|
failed.contents = true;
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
return resolve();
|
|
206
|
-
} else {
|
|
207
|
-
return;
|
|
208
|
-
}
|
|
258
|
+
stop(group);
|
|
259
|
+
break;
|
|
260
|
+
}
|
|
261
|
+
alive.contents = alive.contents - 1 | 0;
|
|
262
|
+
if (alive.contents === 0) {
|
|
263
|
+
return resolve();
|
|
209
264
|
}
|
|
210
265
|
};
|
|
211
266
|
group.running.forEach(r => {
|
|
212
|
-
r.child.on("exit", (code,
|
|
213
|
-
code === null ? undefined : Primitive_option.some(code)
|
|
214
|
-
) !== 0));
|
|
267
|
+
r.child.on("exit", (code, signal) => onGone(r, classifyExit(code, signal, group.stopping)));
|
|
215
268
|
r.child.on("error", exn => {
|
|
216
269
|
Logging.errorWithExn(exn, label(r.worker) + ` failed to start`);
|
|
217
|
-
onGone(r,
|
|
270
|
+
onGone(r, "Failed");
|
|
218
271
|
});
|
|
219
272
|
});
|
|
220
273
|
});
|
|
@@ -228,29 +281,112 @@ async function awaitExit(group) {
|
|
|
228
281
|
}
|
|
229
282
|
}
|
|
230
283
|
|
|
231
|
-
function isRunAtHead(
|
|
232
|
-
if (
|
|
233
|
-
return
|
|
284
|
+
function isRunAtHead(running) {
|
|
285
|
+
if (Utils.$$Array.notEmpty(running)) {
|
|
286
|
+
return running.every(r => {
|
|
287
|
+
if (r.child.connected) {
|
|
288
|
+
return Stdlib_Option.mapOr(r.snapshot, false, snapshot => snapshot.hasArrivedAtHead);
|
|
289
|
+
} else {
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
});
|
|
234
293
|
} else {
|
|
235
294
|
return false;
|
|
236
295
|
}
|
|
237
296
|
}
|
|
238
297
|
|
|
298
|
+
function releaseIfAtHead(group) {
|
|
299
|
+
if (group.holdingRealtime && !group.stopping && isRunAtHead(group.running)) {
|
|
300
|
+
group.holdingRealtime = false;
|
|
301
|
+
group.running.forEach(r => {
|
|
302
|
+
r.child.send("release-realtime");
|
|
303
|
+
});
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function configuredChains(config) {
|
|
309
|
+
return ChainMap.values(config.chainMap).map(chain => {
|
|
310
|
+
let match = chain.sourceConfig;
|
|
311
|
+
let tmp;
|
|
312
|
+
switch (match.TAG) {
|
|
313
|
+
case "EvmSourceConfig" :
|
|
314
|
+
tmp = Stdlib_Option.isSome(match.hypersync);
|
|
315
|
+
break;
|
|
316
|
+
case "FuelSourceConfig" :
|
|
317
|
+
case "SvmSourceConfig" :
|
|
318
|
+
tmp = true;
|
|
319
|
+
break;
|
|
320
|
+
case "SimulateSourceConfig" :
|
|
321
|
+
case "CustomSources" :
|
|
322
|
+
tmp = false;
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
let block = chain.startBlock;
|
|
326
|
+
let tmp$1;
|
|
327
|
+
tmp$1 = block === "latest" ? 0 : block;
|
|
328
|
+
return {
|
|
329
|
+
chainId: chain.id,
|
|
330
|
+
poweredByHyperSync: tmp,
|
|
331
|
+
firstEventBlockNumber: undefined,
|
|
332
|
+
latestProcessedBlock: undefined,
|
|
333
|
+
timestampCaughtUpToHeadOrEndblock: undefined,
|
|
334
|
+
numEventsProcessed: 0,
|
|
335
|
+
latestFetchedBlockNumber: 0,
|
|
336
|
+
knownHeight: 0,
|
|
337
|
+
numBatchesFetched: 0,
|
|
338
|
+
startBlock: tmp$1,
|
|
339
|
+
endBlock: chain.endBlock,
|
|
340
|
+
numAddresses: 0,
|
|
341
|
+
addressesByContract: [],
|
|
342
|
+
isReady: false,
|
|
343
|
+
sourceBlockNumber: 0,
|
|
344
|
+
progressBlockNumber: -1,
|
|
345
|
+
progressLatencyMs: undefined,
|
|
346
|
+
progressBlockTime: undefined,
|
|
347
|
+
concurrency: 0,
|
|
348
|
+
partitionsCount: 0,
|
|
349
|
+
bufferSize: 0,
|
|
350
|
+
bufferBlockNumber: -1,
|
|
351
|
+
idleSeconds: 0,
|
|
352
|
+
waitingForNewBlockSeconds: 0,
|
|
353
|
+
queryingSeconds: 0,
|
|
354
|
+
blockRangeFetchSeconds: 0,
|
|
355
|
+
blockRangeParseSeconds: 0,
|
|
356
|
+
blockRangeFetchCount: 0,
|
|
357
|
+
blockRangeFetchedEvents: 0,
|
|
358
|
+
blockRangeFetchedBlocks: 0,
|
|
359
|
+
reorgCount: 0,
|
|
360
|
+
reorgDetectedBlock: undefined,
|
|
361
|
+
rollbackTargetBlock: undefined,
|
|
362
|
+
rateLimitTimeMs: 0,
|
|
363
|
+
rateLimitResetInMs: undefined
|
|
364
|
+
};
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
|
|
239
368
|
async function run(config, workers, reset) {
|
|
240
369
|
let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
|
|
241
370
|
await Persistence.initForRun(persistence, config, reset, config.isDev, false);
|
|
242
371
|
await persistence.storage.close();
|
|
243
372
|
let startTime = new Date();
|
|
244
373
|
let startTimeRef = Performance.now();
|
|
245
|
-
Logging.info(
|
|
246
|
-
|
|
374
|
+
Logging.info({
|
|
375
|
+
msg: "Indexing will be split across multiple processes for faster and more reliable indexing.",
|
|
376
|
+
chains: ChainMap.values(config.chainMap).length,
|
|
377
|
+
processes: workers.length,
|
|
378
|
+
maxConnections: Env.Db.maxConnections
|
|
379
|
+
});
|
|
380
|
+
let shouldUseTui = Tui.shouldUse(undefined, undefined);
|
|
247
381
|
let holdRealtime = Persistence.getInitializedState(persistence).chains.some(chain => Stdlib_Option.isNone(chain.timestampCaughtUpToHeadOrEndblock));
|
|
248
382
|
let group = {
|
|
249
|
-
running:
|
|
250
|
-
stopping: false
|
|
383
|
+
running: [],
|
|
384
|
+
stopping: false,
|
|
385
|
+
holdingRealtime: holdRealtime
|
|
251
386
|
};
|
|
387
|
+
group.running = workers.map((worker, workerIndex) => fork(worker, workerIndex, workers.length, holdRealtime, config.isDev, undefined, shouldUseTui, undefined, undefined, () => releaseIfAtHead(group)));
|
|
252
388
|
let reported = () => Stdlib_Array.filterMap(group.running, r => r.snapshot);
|
|
253
|
-
let merge = snapshots => Metrics.merge(snapshots, startTime, new Date(), Performance.secondsSince(startTimeRef));
|
|
389
|
+
let merge = snapshots => Metrics.merge(snapshots, startTime, new Date(), Performance.secondsSince(startTimeRef), CrossChainState.calculateTargetBufferSize());
|
|
254
390
|
Server.startServer(() => {
|
|
255
391
|
let snapshots = reported();
|
|
256
392
|
if (snapshots.length !== 0) {
|
|
@@ -260,33 +396,20 @@ async function run(config, workers, reset) {
|
|
|
260
396
|
`worker="` + name(r.worker) + `"`,
|
|
261
397
|
runtime
|
|
262
398
|
]))), config.isDev);
|
|
263
|
-
let releaseCheck = {
|
|
264
|
-
contents: undefined
|
|
265
|
-
};
|
|
266
|
-
let stopReleaseCheck = () => {
|
|
267
|
-
Stdlib_Option.forEach(releaseCheck.contents, prim => {
|
|
268
|
-
clearInterval(prim);
|
|
269
|
-
});
|
|
270
|
-
releaseCheck.contents = undefined;
|
|
271
|
-
};
|
|
272
|
-
if (holdRealtime) {
|
|
273
|
-
releaseCheck.contents = Primitive_option.some(setInterval(() => {
|
|
274
|
-
if (isRunAtHead(reported(), group.running.length)) {
|
|
275
|
-
stopReleaseCheck();
|
|
276
|
-
group.running.forEach(r => {
|
|
277
|
-
r.child.send("release-realtime");
|
|
278
|
-
});
|
|
279
|
-
return Logging.info("Every chain has reached the head. Switching the run to realtime.");
|
|
280
|
-
}
|
|
281
|
-
}, 500));
|
|
282
|
-
}
|
|
283
399
|
if (shouldUseTui) {
|
|
284
|
-
Tui.start(config, () =>
|
|
400
|
+
Tui.start(config, () => {
|
|
401
|
+
let snapshots = reported();
|
|
402
|
+
if (snapshots.length !== 0) {
|
|
403
|
+
return merge(snapshots);
|
|
404
|
+
}
|
|
405
|
+
let newrecord = {...merge([])};
|
|
406
|
+
newrecord.chains = configuredChains(config);
|
|
407
|
+
return newrecord;
|
|
408
|
+
});
|
|
285
409
|
}
|
|
286
410
|
process.on("SIGTERM", () => stop(group));
|
|
287
411
|
process.on("SIGINT", () => stop(group));
|
|
288
412
|
let outcome = await awaitExit(group);
|
|
289
|
-
stopReleaseCheck();
|
|
290
413
|
if (outcome === "Finished") {
|
|
291
414
|
if (shouldUseTui) {
|
|
292
415
|
process.on("SIGTERM", () => {
|
|
@@ -306,20 +429,27 @@ async function run(config, workers, reset) {
|
|
|
306
429
|
|
|
307
430
|
let minConnectionsPerWorker = 2;
|
|
308
431
|
|
|
432
|
+
let maxWorkers = 4;
|
|
433
|
+
|
|
309
434
|
export {
|
|
310
435
|
minConnectionsPerWorker,
|
|
436
|
+
maxWorkers,
|
|
311
437
|
plan,
|
|
312
438
|
planForRun,
|
|
313
439
|
name,
|
|
314
440
|
label,
|
|
315
441
|
logFilePath,
|
|
316
442
|
readLines,
|
|
443
|
+
memoryBudgets,
|
|
317
444
|
fork,
|
|
318
445
|
stop,
|
|
319
446
|
syncCache,
|
|
320
447
|
dumpCache,
|
|
448
|
+
classifyExit,
|
|
321
449
|
awaitExit,
|
|
322
450
|
isRunAtHead,
|
|
451
|
+
releaseIfAtHead,
|
|
452
|
+
configuredChains,
|
|
323
453
|
run,
|
|
324
454
|
}
|
|
325
455
|
/* Env Not a pure module */
|
package/src/TestIndexer.res
CHANGED
|
@@ -254,13 +254,16 @@ let makeInitialState = (
|
|
|
254
254
|
addressRowsByChain
|
|
255
255
|
->Utils.Dict.dangerouslyGetNonOption(chainIdStr)
|
|
256
256
|
->Option.getOr(AddressRows.emptySeedRows())
|
|
257
|
+
// The pinned startBlock resumes the chain rather than moving its start, so
|
|
258
|
+
// contract start blocks and onBlock intervals stay what config.yaml says.
|
|
259
|
+
// https://github.com/enviodev/hyperindex/issues/1656
|
|
257
260
|
{
|
|
258
261
|
Persistence.id: chain,
|
|
259
|
-
startBlock:
|
|
262
|
+
startBlock: config.chainMap->ChainMap.get(chain)->Config.startBlockOrZero,
|
|
260
263
|
endBlock: processChainConfig.endBlock,
|
|
261
264
|
sourceBlockNumber: processChainConfig.endBlock->Option.getOr(0),
|
|
262
265
|
maxReorgDepth: 0, // No reorg support in test indexer
|
|
263
|
-
progressBlockNumber: -1,
|
|
266
|
+
progressBlockNumber: processChainConfig.startBlock - 1,
|
|
264
267
|
progressBlockTime: None,
|
|
265
268
|
numEventsProcessed: 0.,
|
|
266
269
|
firstEventBlockNumber: None,
|
|
@@ -948,7 +951,7 @@ let createTestIndexer = (): t<'processConfig> => {
|
|
|
948
951
|
~exitAfterFirstEventBlock,
|
|
949
952
|
~onError=errHandler => {
|
|
950
953
|
errHandler->ErrorHandling.log
|
|
951
|
-
reject(errHandler
|
|
954
|
+
reject(errHandler->ErrorHandling.toExn)
|
|
952
955
|
},
|
|
953
956
|
// Caught up: resolve the run instead of exiting the process.
|
|
954
957
|
~onExit=() => resolve(),
|
package/src/TestIndexer.res.mjs
CHANGED
|
@@ -180,10 +180,10 @@ function makeInitialState(config, processConfigChains, addressRowsByChain, contr
|
|
|
180
180
|
let addressRows = Stdlib_Option.getOr(addressRowsByChain[chainIdStr], AddressRows.emptySeedRows());
|
|
181
181
|
return {
|
|
182
182
|
id: chain,
|
|
183
|
-
startBlock:
|
|
183
|
+
startBlock: Config.startBlockOrZero(ChainMap.get(config.chainMap, chain)),
|
|
184
184
|
endBlock: processChainConfig.endBlock,
|
|
185
185
|
maxReorgDepth: 0,
|
|
186
|
-
progressBlockNumber: -1,
|
|
186
|
+
progressBlockNumber: processChainConfig.startBlock - 1 | 0,
|
|
187
187
|
progressBlockTime: undefined,
|
|
188
188
|
numEventsProcessed: 0,
|
|
189
189
|
firstEventBlockNumber: undefined,
|
|
@@ -641,7 +641,7 @@ function createTestIndexer() {
|
|
|
641
641
|
await new Promise((resolve, reject) => {
|
|
642
642
|
let indexerState = IndexerState.makeFromDbState(runConfig, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, undefined, errHandler => {
|
|
643
643
|
ErrorHandling.log(errHandler);
|
|
644
|
-
reject(
|
|
644
|
+
reject(ErrorHandling.toExn(errHandler));
|
|
645
645
|
}, () => resolve());
|
|
646
646
|
indexerStateRef.contents = Primitive_option.some(indexerState);
|
|
647
647
|
IndexerLoop.start(indexerState);
|
package/src/Worker.res
CHANGED
|
@@ -17,11 +17,17 @@ type config = {
|
|
|
17
17
|
// process are still backfilling, and an indexer goes realtime as a whole or
|
|
18
18
|
// not at all. Cleared by the supervisor's `ReleaseRealtime`.
|
|
19
19
|
holdRealtime: bool,
|
|
20
|
+
// Which command the run was started by. A worker re-parses the project's
|
|
21
|
+
// files, which say nothing about that, so it can only be told — and a worker
|
|
22
|
+
// that took a dev run for a plain one would exit at its end block and leave
|
|
23
|
+
// the console it was still serving with a chain missing.
|
|
24
|
+
isDev: bool,
|
|
20
25
|
}
|
|
21
26
|
|
|
22
27
|
let configSchema = S.object((s): config => {
|
|
23
28
|
chainIds: s.field("chainIds", S.array(ChainId.schema)),
|
|
24
29
|
holdRealtime: s.fieldOr("holdRealtime", S.bool, false),
|
|
30
|
+
isDev: s.field("isDev", S.bool),
|
|
25
31
|
})
|
|
26
32
|
|
|
27
33
|
// Read as this module loads, which is before anything that could catch a bare
|
package/src/Worker.res.mjs
CHANGED
|
@@ -14,7 +14,8 @@ let envVar = "ENVIO_INTERNAL_WORKER";
|
|
|
14
14
|
|
|
15
15
|
let configSchema = S$RescriptSchema.object(s => ({
|
|
16
16
|
chainIds: s.f("chainIds", S$RescriptSchema.array(ChainId.schema)),
|
|
17
|
-
holdRealtime: s.fieldOr("holdRealtime", S$RescriptSchema.bool, false)
|
|
17
|
+
holdRealtime: s.fieldOr("holdRealtime", S$RescriptSchema.bool, false),
|
|
18
|
+
isDev: s.f("isDev", S$RescriptSchema.bool)
|
|
18
19
|
}));
|
|
19
20
|
|
|
20
21
|
function detect(env, hasChannel) {
|
package/src/bindings/NodeJs.res
CHANGED
|
@@ -157,7 +157,30 @@ module ChildProcess = {
|
|
|
157
157
|
@module("child_process")
|
|
158
158
|
external execWithOptions: (string, execOptions, callback) => unit = "exec"
|
|
159
159
|
|
|
160
|
-
|
|
160
|
+
// One of a child's stdio slots, present only for a slot the parent asked to
|
|
161
|
+
// pipe rather than inherit.
|
|
162
|
+
module Stream = {
|
|
163
|
+
type t
|
|
164
|
+
@send external setEncoding: (t, string) => unit = "setEncoding"
|
|
165
|
+
@send external onData: (t, @as("data") _, string => unit) => unit = "on"
|
|
166
|
+
@send external onEnd: (t, @as("end") _, unit => unit) => unit = "on"
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
module Child = {
|
|
170
|
+
type t
|
|
171
|
+
@send external send: (t, 'msg) => bool = "send"
|
|
172
|
+
@send external onMessage: (t, @as("message") _, 'msg => unit) => unit = "on"
|
|
173
|
+
@send
|
|
174
|
+
external onExit: (t, @as("exit") _, (Null.t<int>, Null.t<string>) => unit) => unit = "on"
|
|
175
|
+
@send external onError: (t, @as("error") _, exn => unit) => unit = "on"
|
|
176
|
+
@send external kill: (t, string) => bool = "kill"
|
|
177
|
+
// Whether the IPC channel is still open. Node closes it before it reports
|
|
178
|
+
// the exit, so this goes false while the child is still running.
|
|
179
|
+
@get external connected: t => bool = "connected"
|
|
180
|
+
@get external stdout: t => Null.t<Stream.t> = "stdout"
|
|
181
|
+
@get external stderr: t => Null.t<Stream.t> = "stderr"
|
|
182
|
+
}
|
|
183
|
+
|
|
161
184
|
type forkOptions = {
|
|
162
185
|
cwd?: string,
|
|
163
186
|
env?: dict<string>,
|
|
@@ -167,21 +190,7 @@ module ChildProcess = {
|
|
|
167
190
|
stdio?: array<string>,
|
|
168
191
|
}
|
|
169
192
|
@module("child_process")
|
|
170
|
-
external fork: (string, array<string>, forkOptions) =>
|
|
171
|
-
@send external send: (child, 'msg) => bool = "send"
|
|
172
|
-
@send external onMessage: (child, @as("message") _, 'msg => unit) => unit = "on"
|
|
173
|
-
@send
|
|
174
|
-
external onExit: (child, @as("exit") _, (Null.t<int>, Null.t<string>) => unit) => unit = "on"
|
|
175
|
-
@send external onChildError: (child, @as("error") _, exn => unit) => unit = "on"
|
|
176
|
-
@send external kill: (child, string) => bool = "kill"
|
|
177
|
-
|
|
178
|
-
// Present only for a stdio slot the parent asked to pipe.
|
|
179
|
-
type stdioStream
|
|
180
|
-
@get external stdout: child => Null.t<stdioStream> = "stdout"
|
|
181
|
-
@get external stderr: child => Null.t<stdioStream> = "stderr"
|
|
182
|
-
@send external setEncoding: (stdioStream, string) => unit = "setEncoding"
|
|
183
|
-
@send external onData: (stdioStream, @as("data") _, string => unit) => unit = "on"
|
|
184
|
-
@send external onEnd: (stdioStream, @as("end") _, unit => unit) => unit = "on"
|
|
193
|
+
external fork: (string, array<string>, forkOptions) => Child.t = "fork"
|
|
185
194
|
}
|
|
186
195
|
|
|
187
196
|
module Url = {
|
|
@@ -20,8 +20,10 @@ type startBlockGroup = {startBlock: int, count: int}
|
|
|
20
20
|
|
|
21
21
|
@send external merge: (t, t) => t = "merge"
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
// For assertions only - nothing in the indexer reads it. A query gets its
|
|
24
|
+
// address filter from the set's cached per-contract slices and the runtime asks
|
|
25
|
+
// `containsAt`; this is the only way a test can compare two sets by what they
|
|
26
|
+
// hold rather than by handle identity.
|
|
27
|
+
@send external addressesForTest: t => array<Address.t> = "addressesForTest"
|
|
26
28
|
|
|
27
29
|
let isEmpty = (set: t) => set->size === 0
|
|
@@ -45,15 +45,14 @@ type makeSetOptions = {
|
|
|
45
45
|
limit?: int,
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
@send
|
|
49
|
-
external newEvm: (Core.addressStoreCtor, bool, array<contract>) => t = "newEvm"
|
|
48
|
+
@send external newEvm: (Core.addressStoreCtor, array<contract>) => t = "newEvm"
|
|
50
49
|
@send external newSvm: (Core.addressStoreCtor, array<contract>) => t = "newSvm"
|
|
51
50
|
@send external newFuel: (Core.addressStoreCtor, array<contract>) => t = "newFuel"
|
|
52
51
|
|
|
53
|
-
let make = (~ecosystem: Ecosystem.name, ~
|
|
52
|
+
let make = (~ecosystem: Ecosystem.name, ~contracts: array<contract>): t => {
|
|
54
53
|
let ctor = Core.getAddon().addressStore
|
|
55
54
|
switch ecosystem {
|
|
56
|
-
| Evm => ctor->newEvm(
|
|
55
|
+
| Evm => ctor->newEvm(contracts)
|
|
57
56
|
| Svm => ctor->newSvm(contracts)
|
|
58
57
|
| Fuel => ctor->newFuel(contracts)
|
|
59
58
|
}
|
|
@@ -105,7 +104,6 @@ let contractsOf = (
|
|
|
105
104
|
|
|
106
105
|
@send
|
|
107
106
|
external registerBatchRaw: (t, array<registration>) => array<rawVerdict> = "registerBatch"
|
|
108
|
-
@send external seedBatchRaw: (t, array<registration>) => array<rawVerdict> = "seedBatch"
|
|
109
107
|
|
|
110
108
|
@send
|
|
111
109
|
external seedRowsRaw: (
|
|
@@ -128,7 +126,12 @@ external drainForWrite: (t, int, array<int>) => array<drainedAddress> = "drainFo
|
|
|
128
126
|
|
|
129
127
|
@send external pendingCount: t => int = "pendingCount"
|
|
130
128
|
|
|
131
|
-
|
|
129
|
+
// For assertions only - nothing in the indexer reads it. The write path drains
|
|
130
|
+
// instead, and `pendingCount` is what the runtime asks; neither lets a test see
|
|
131
|
+
// which registrations are queued without consuming them.
|
|
132
|
+
@send
|
|
133
|
+
external pendingEntriesForTest: t => array<Internal.indexingContract> = "pendingEntriesForTest"
|
|
134
|
+
|
|
132
135
|
@send external makeSetRaw: (t, string, makeSetOptions) => AddressSet.t = "makeSet"
|
|
133
136
|
@send external contractCount: (t, string) => int = "contractCount"
|
|
134
137
|
|
|
@@ -145,11 +148,19 @@ type rolledBackAddress = {address: NodeJs.Buffer.t, contractId: int}
|
|
|
145
148
|
// still point at the right entries.
|
|
146
149
|
@send external rollback: (t, int) => array<rolledBackAddress> = "rollback"
|
|
147
150
|
|
|
148
|
-
|
|
151
|
+
// For assertions only - nothing in the indexer reads it. The runtime's gate is
|
|
152
|
+
// `isIndexedAt` and the user sees `contractAddresses`; neither exposes an
|
|
153
|
+
// entry's own start blocks or the order registrations sort into.
|
|
154
|
+
@send
|
|
155
|
+
external getAllForTest: (t, Address.t) => array<Internal.indexingContract> = "getAllForTest"
|
|
149
156
|
|
|
150
157
|
@send external dynamicContractNames: t => array<string> = "dynamicContractNames"
|
|
151
158
|
|
|
152
|
-
|
|
159
|
+
// The one place the store spells an address the way the chain shows it: the
|
|
160
|
+
// setting comes from the caller, since the store keeps keys, not spellings.
|
|
161
|
+
@send
|
|
162
|
+
external contractAddresses: (t, string, ~shouldChecksum: bool) => array<Address.t> =
|
|
163
|
+
"contractAddresses"
|
|
153
164
|
|
|
154
165
|
let toVerdict = (raw: rawVerdict): verdict =>
|
|
155
166
|
switch raw.kind {
|
|
@@ -167,8 +178,5 @@ let toVerdict = (raw: rawVerdict): verdict =>
|
|
|
167
178
|
let registerBatch = (store: t, registrations: array<registration>): array<verdict> =>
|
|
168
179
|
store->registerBatchRaw(registrations)->Array.map(toVerdict)
|
|
169
180
|
|
|
170
|
-
let seedBatch = (store: t, registrations: array<registration>): array<verdict> =>
|
|
171
|
-
store->seedBatchRaw(registrations)->Array.map(toVerdict)
|
|
172
|
-
|
|
173
181
|
let makeSet = (store: t, ~contractName, ~options={}: makeSetOptions) =>
|
|
174
182
|
store->makeSetRaw(contractName, options)
|
|
@@ -6,11 +6,11 @@ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
|
6
6
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
7
7
|
import * as ContractMapping from "../ContractMapping.res.mjs";
|
|
8
8
|
|
|
9
|
-
function make(ecosystem,
|
|
9
|
+
function make(ecosystem, contracts) {
|
|
10
10
|
let ctor = Core.getAddon().AddressStore;
|
|
11
11
|
switch (ecosystem) {
|
|
12
12
|
case "evm" :
|
|
13
|
-
return ctor.newEvm(
|
|
13
|
+
return ctor.newEvm(contracts);
|
|
14
14
|
case "fuel" :
|
|
15
15
|
return ctor.newFuel(contracts);
|
|
16
16
|
case "svm" :
|
|
@@ -74,10 +74,6 @@ function registerBatch(store, registrations) {
|
|
|
74
74
|
return store.registerBatch(registrations).map(toVerdict);
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
-
function seedBatch(store, registrations) {
|
|
78
|
-
return store.seedBatch(registrations).map(toVerdict);
|
|
79
|
-
}
|
|
80
|
-
|
|
81
77
|
function makeSet(store, contractName, optionsOpt) {
|
|
82
78
|
let options = optionsOpt !== undefined ? optionsOpt : ({});
|
|
83
79
|
return store.makeSet(contractName, options);
|
|
@@ -89,7 +85,6 @@ export {
|
|
|
89
85
|
seedRows,
|
|
90
86
|
toVerdict,
|
|
91
87
|
registerBatch,
|
|
92
|
-
seedBatch,
|
|
93
88
|
makeSet,
|
|
94
89
|
}
|
|
95
90
|
/* Core Not a pure module */
|