envio 3.12.1 → 3.13.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/package.json +6 -6
  2. package/src/BatchProcessing.res +9 -9
  3. package/src/BatchProcessing.res.mjs +5 -5
  4. package/src/Bin.res +24 -17
  5. package/src/Bin.res.mjs +5 -0
  6. package/src/ChainFetching.res +24 -10
  7. package/src/ChainFetching.res.mjs +8 -3
  8. package/src/ChainState.res +43 -0
  9. package/src/ChainState.res.mjs +43 -0
  10. package/src/ChainState.resi +2 -0
  11. package/src/Config.res +35 -0
  12. package/src/Config.res.mjs +39 -0
  13. package/src/Core.res +4 -0
  14. package/src/Core.res.mjs +4 -0
  15. package/src/CrossChainState.res +60 -3
  16. package/src/CrossChainState.res.mjs +38 -4
  17. package/src/CrossChainState.resi +10 -1
  18. package/src/Env.res +4 -0
  19. package/src/IndexerLoop.res +2 -0
  20. package/src/IndexerLoop.res.mjs +1 -0
  21. package/src/IndexerState.res +41 -1
  22. package/src/IndexerState.res.mjs +43 -4
  23. package/src/IndexerState.resi +10 -0
  24. package/src/Logging.res +38 -6
  25. package/src/Logging.res.mjs +32 -5
  26. package/src/Main.res +131 -268
  27. package/src/Main.res.mjs +28 -152
  28. package/src/Metrics.res +263 -102
  29. package/src/Metrics.res.mjs +227 -48
  30. package/src/Persistence.res +27 -2
  31. package/src/Persistence.res.mjs +9 -2
  32. package/src/PgStorage.res +17 -9
  33. package/src/PgStorage.res.mjs +11 -8
  34. package/src/Server.res +181 -0
  35. package/src/Server.res.mjs +143 -0
  36. package/src/Supervisor.res +415 -0
  37. package/src/Supervisor.res.mjs +325 -0
  38. package/src/TestIndexer.res.mjs +1 -1
  39. package/src/Worker.res +95 -0
  40. package/src/Worker.res.mjs +80 -0
  41. package/src/bindings/NodeJs.res +41 -0
  42. package/src/db/InternalTable.res +8 -1
  43. package/src/db/InternalTable.res.mjs +5 -1
  44. package/src/tui/Tui.res +24 -0
  45. package/src/tui/Tui.res.mjs +18 -0
  46. package/src/tui/components/SyncETA.res +12 -6
  47. package/src/tui/components/SyncETA.res.mjs +12 -8
@@ -6,6 +6,7 @@ import * as ChainId from "./ChainId.res.mjs";
6
6
  import * as Process from "process";
7
7
  import * as Perf_hooks from "perf_hooks";
8
8
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
9
+ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
9
10
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
10
11
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
11
12
 
@@ -18,6 +19,94 @@ function hasProcessedToEndblock(m) {
18
19
  }
19
20
  }
20
21
 
22
+ function sumByKey(items, key, add) {
23
+ let byKey = {};
24
+ let order = [];
25
+ items.forEach(item => {
26
+ let k = key(item);
27
+ let existing = byKey[k];
28
+ if (existing !== undefined) {
29
+ byKey[k] = add(Primitive_option.valFromOption(existing), item);
30
+ } else {
31
+ byKey[k] = item;
32
+ order.push(k);
33
+ }
34
+ });
35
+ return order.map(k => byKey[k]);
36
+ }
37
+
38
+ function merge(snapshots, startTime, metricTime, elapsedSeconds) {
39
+ let sumInt = select => Stdlib_Array.reduce(snapshots, 0, (acc, snapshot) => acc + select(snapshot) | 0);
40
+ let sumFloat = select => Stdlib_Array.reduce(snapshots, 0, (acc, snapshot) => acc + select(snapshot));
41
+ return {
42
+ startTime: startTime,
43
+ metricTime: metricTime,
44
+ elapsedSeconds: elapsedSeconds,
45
+ targetBufferSize: sumInt(s => s.targetBufferSize),
46
+ isInReorgThreshold: snapshots.some(s => s.isInReorgThreshold),
47
+ hasArrivedAtHead: Utils.$$Array.notEmpty(snapshots) && snapshots.every(s => s.hasArrivedAtHead),
48
+ rollbackEnabled: snapshots.some(s => s.rollbackEnabled),
49
+ maxBatchSize: Stdlib_Array.reduce(snapshots, 0, (acc, s) => Primitive_int.max(acc, s.maxBatchSize)),
50
+ preloadSeconds: sumFloat(s => s.preloadSeconds),
51
+ processingSeconds: sumFloat(s => s.processingSeconds),
52
+ processingStalledOnFetchSeconds: sumFloat(s => s.processingStalledOnFetchSeconds),
53
+ processingStalledOnStorageWriteSeconds: sumFloat(s => s.processingStalledOnStorageWriteSeconds),
54
+ rollbackSeconds: sumFloat(s => s.rollbackSeconds),
55
+ rollbackCount: sumInt(s => s.rollbackCount),
56
+ rollbackEventsCount: sumFloat(s => s.rollbackEventsCount),
57
+ chains: snapshots.flatMap(s => s.chains),
58
+ handlers: sumByKey(snapshots.flatMap(s => s.handlers), h => h.contract + `.` + h.event, (a, b) => ({
59
+ contract: a.contract,
60
+ event: a.event,
61
+ processingSeconds: a.processingSeconds + b.processingSeconds,
62
+ processingCount: a.processingCount + b.processingCount,
63
+ preloadSeconds: a.preloadSeconds + b.preloadSeconds,
64
+ preloadCount: a.preloadCount + b.preloadCount,
65
+ preloadSecondsTotal: a.preloadSecondsTotal + b.preloadSecondsTotal
66
+ })),
67
+ effects: sumByKey(snapshots.flatMap(s => s.effects), e => e.effect + `.` + e.scope, (a, b) => {
68
+ let match = a.cacheCount;
69
+ let match$1 = b.cacheCount;
70
+ return {
71
+ effect: a.effect,
72
+ scope: a.scope,
73
+ callSeconds: a.callSeconds + b.callSeconds,
74
+ callSecondsTotal: a.callSecondsTotal + b.callSecondsTotal,
75
+ callCount: a.callCount + b.callCount,
76
+ activeCallsCount: a.activeCallsCount + b.activeCallsCount | 0,
77
+ queueCount: a.queueCount + b.queueCount | 0,
78
+ queueWaitSeconds: a.queueWaitSeconds + b.queueWaitSeconds,
79
+ invalidationsCount: a.invalidationsCount + b.invalidationsCount,
80
+ cacheCount: match !== undefined ? (
81
+ match$1 !== undefined ? match + match$1 | 0 : match
82
+ ) : match$1
83
+ };
84
+ }),
85
+ storageLoads: sumByKey(snapshots.flatMap(s => s.storageLoads), l => l.storage + `.` + l.operation, (a, b) => ({
86
+ operation: a.operation,
87
+ storage: a.storage,
88
+ seconds: a.seconds + b.seconds,
89
+ secondsTotal: a.secondsTotal + b.secondsTotal,
90
+ count: a.count + b.count,
91
+ whereSize: a.whereSize + b.whereSize,
92
+ size: a.size + b.size
93
+ })),
94
+ storageWrites: sumByKey(snapshots.flatMap(s => s.storageWrites), w => w.storage, (a, b) => ({
95
+ storage: a.storage,
96
+ seconds: a.seconds + b.seconds,
97
+ count: a.count + b.count | 0
98
+ })),
99
+ historyPrunes: sumByKey(snapshots.flatMap(s => s.historyPrunes), p => p.entity, (a, b) => ({
100
+ entity: a.entity,
101
+ seconds: a.seconds + b.seconds,
102
+ count: a.count + b.count | 0
103
+ })),
104
+ sourceRequests: snapshots.flatMap(s => s.sourceRequests),
105
+ sourceHeights: snapshots.flatMap(s => s.sourceHeights),
106
+ sourceHeightStreams: snapshots.flatMap(s => s.sourceHeightStreams)
107
+ };
108
+ }
109
+
21
110
  function formatValue(value) {
22
111
  return (Math.round(value * 1000) / 1000).toString();
23
112
  }
@@ -365,24 +454,12 @@ function startRuntimeCollectors() {
365
454
  getRuntimeCollectors();
366
455
  }
367
456
 
368
- function collectRuntime() {
369
- let b = {
370
- out: ""
371
- };
457
+ function sampleRuntime() {
372
458
  let memory = Process.memoryUsage();
373
459
  let cpu = Process.cpuUsage();
374
460
  let elu = Perf_hooks.performance.eventLoopUtilization();
375
461
  let match = getRuntimeCollectors();
376
462
  let eventLoopDelay = match.eventLoopDelay;
377
- single(b, "process_cpu_user_seconds_total", "Total user CPU time spent in seconds.", "counter", cpu.user / 1000000);
378
- single(b, "process_cpu_system_seconds_total", "Total system CPU time spent in seconds.", "counter", cpu.system / 1000000);
379
- single(b, "process_cpu_seconds_total", "Total user and system CPU time spent in seconds.", "counter", (cpu.user + cpu.system) / 1000000);
380
- single(b, "process_start_time_seconds", "Start time of the process since unix epoch in seconds.", "gauge", match.processStartTimeSeconds);
381
- single(b, "process_resident_memory_bytes", "Resident memory size in bytes.", "gauge", memory.rss);
382
- single(b, "nodejs_heap_size_total_bytes", "Process heap size from Node.js in bytes.", "gauge", memory.heapTotal);
383
- single(b, "nodejs_heap_size_used_bytes", "Process heap size used from Node.js in bytes.", "gauge", memory.heapUsed);
384
- single(b, "nodejs_external_memory_bytes", "Node.js external memory size in bytes.", "gauge", memory.external);
385
- single(b, "nodejs_eventloop_utilization", "Ratio of time the event loop is active, since process start.", "gauge", elu.utilization);
386
463
  let hasLagSamples = eventLoopDelay.max > 0;
387
464
  let nsToSeconds = ns => {
388
465
  if (hasLagSamples && !Number.isNaN(ns)) {
@@ -391,52 +468,152 @@ function collectRuntime() {
391
468
  return 0;
392
469
  }
393
470
  };
394
- single(b, "nodejs_eventloop_lag_mean_seconds", "The mean of the recorded event loop delays.", "gauge", nsToSeconds(eventLoopDelay.mean));
395
- single(b, "nodejs_eventloop_lag_min_seconds", "The minimum recorded event loop delay.", "gauge", nsToSeconds(eventLoopDelay.min));
396
- single(b, "nodejs_eventloop_lag_max_seconds", "The maximum recorded event loop delay.", "gauge", nsToSeconds(eventLoopDelay.max));
397
- single(b, "nodejs_eventloop_lag_stddev_seconds", "The standard deviation of the recorded event loop delays.", "gauge", nsToSeconds(eventLoopDelay.stddev));
398
- single(b, "nodejs_eventloop_lag_p50_seconds", "The 50th percentile of the recorded event loop delays.", "gauge", nsToSeconds(eventLoopDelay.percentile(50)));
399
- single(b, "nodejs_eventloop_lag_p90_seconds", "The 90th percentile of the recorded event loop delays.", "gauge", nsToSeconds(eventLoopDelay.percentile(90)));
400
- single(b, "nodejs_eventloop_lag_p99_seconds", "The 99th percentile of the recorded event loop delays.", "gauge", nsToSeconds(eventLoopDelay.percentile(99)));
401
- eventLoopDelay.reset();
402
- let heapSpaces = V8.getHeapSpaceStatistics().map(s => [
403
- `{space="` + s.space_name.replace("_space", "") + `"}`,
404
- s
405
- ]);
406
- series(b, "nodejs_heap_space_size_total_bytes", "Process heap space size total from Node.js in bytes.", "gauge", heapSpaces, s => s.space_size);
407
- series(b, "nodejs_heap_space_size_used_bytes", "Process heap space size used from Node.js in bytes.", "gauge", heapSpaces, s => s.space_used_size);
408
- series(b, "nodejs_heap_space_size_available_bytes", "Process heap space size available from Node.js in bytes.", "gauge", heapSpaces, s => s.space_available_size);
409
471
  let byType = {};
410
- Process.getActiveResourcesInfo().forEach(resource => {
411
- let label = `{type="` + escapeLabelValue(resource) + `"}`;
412
- byType[label] = Stdlib_Option.getOr(byType[label], 0) + 1;
472
+ let sample_cpuUserSeconds = cpu.user / 1000000;
473
+ let sample_cpuSystemSeconds = cpu.system / 1000000;
474
+ let sample_processStartTimeSeconds = match.processStartTimeSeconds;
475
+ let sample_residentMemoryBytes = memory.rss;
476
+ let sample_heapTotalBytes = memory.heapTotal;
477
+ let sample_heapUsedBytes = memory.heapUsed;
478
+ let sample_externalMemoryBytes = memory.external;
479
+ let sample_eventLoopUtilization = elu.utilization;
480
+ let sample_eventLoopLagMeanSeconds = nsToSeconds(eventLoopDelay.mean);
481
+ let sample_eventLoopLagMinSeconds = nsToSeconds(eventLoopDelay.min);
482
+ let sample_eventLoopLagMaxSeconds = nsToSeconds(eventLoopDelay.max);
483
+ let sample_eventLoopLagStddevSeconds = nsToSeconds(eventLoopDelay.stddev);
484
+ let sample_eventLoopLagP50Seconds = nsToSeconds(eventLoopDelay.percentile(50));
485
+ let sample_eventLoopLagP90Seconds = nsToSeconds(eventLoopDelay.percentile(90));
486
+ let sample_eventLoopLagP99Seconds = nsToSeconds(eventLoopDelay.percentile(99));
487
+ let sample_heapSpaces = V8.getHeapSpaceStatistics().map(s => ({
488
+ space: s.space_name.replace("_space", ""),
489
+ size: s.space_size,
490
+ used: s.space_used_size,
491
+ available: s.space_available_size
492
+ }));
493
+ let sample_activeResources = (Process.getActiveResourcesInfo().forEach(resource => {
494
+ byType[resource] = Stdlib_Option.getOr(byType[resource], 0) + 1;
495
+ }), Object.entries(byType));
496
+ let sample_gc = Object.entries(match.gcStats).map(param => {
497
+ let stat = param[1];
498
+ return {
499
+ kind: param[0],
500
+ count: stat.count,
501
+ seconds: stat.seconds
502
+ };
413
503
  });
414
- let activeResources = Object.entries(byType);
415
- series(b, "nodejs_active_resources", "Number of active resources that are currently keeping the event loop alive, grouped by async resource type.", "gauge", activeResources, count => count);
416
- single(b, "nodejs_active_resources_total", "Total number of active resources.", "gauge", Stdlib_Array.reduce(activeResources, 0, (acc, param) => acc + param[1]));
417
- let gcEntries = [];
418
- Utils.Dict.forEachWithKey(match.gcStats, (stat, kind) => {
419
- gcEntries.push([
420
- `{kind="` + kind + `"}`,
421
- stat
504
+ let sample = {
505
+ cpuUserSeconds: sample_cpuUserSeconds,
506
+ cpuSystemSeconds: sample_cpuSystemSeconds,
507
+ processStartTimeSeconds: sample_processStartTimeSeconds,
508
+ residentMemoryBytes: sample_residentMemoryBytes,
509
+ heapTotalBytes: sample_heapTotalBytes,
510
+ heapUsedBytes: sample_heapUsedBytes,
511
+ externalMemoryBytes: sample_externalMemoryBytes,
512
+ eventLoopUtilization: sample_eventLoopUtilization,
513
+ eventLoopLagMeanSeconds: sample_eventLoopLagMeanSeconds,
514
+ eventLoopLagMinSeconds: sample_eventLoopLagMinSeconds,
515
+ eventLoopLagMaxSeconds: sample_eventLoopLagMaxSeconds,
516
+ eventLoopLagStddevSeconds: sample_eventLoopLagStddevSeconds,
517
+ eventLoopLagP50Seconds: sample_eventLoopLagP50Seconds,
518
+ eventLoopLagP90Seconds: sample_eventLoopLagP90Seconds,
519
+ eventLoopLagP99Seconds: sample_eventLoopLagP99Seconds,
520
+ heapSpaces: sample_heapSpaces,
521
+ activeResources: sample_activeResources,
522
+ gc: sample_gc,
523
+ nodeVersion: Process.version
524
+ };
525
+ eventLoopDelay.reset();
526
+ return sample;
527
+ }
528
+
529
+ function renderRuntime(samples) {
530
+ let b = {
531
+ out: ""
532
+ };
533
+ let labels = (scope, own) => {
534
+ if (scope === "") {
535
+ if (own === "") {
536
+ return "";
537
+ } else {
538
+ return `{` + own + `}`;
539
+ }
540
+ } else if (own === "") {
541
+ return `{` + scope + `}`;
542
+ } else {
543
+ return `{` + scope + `,` + own + `}`;
544
+ }
545
+ };
546
+ let scoped = samples.map(param => [
547
+ labels(param[0], ""),
548
+ param[1]
549
+ ]);
550
+ let each = select => samples.flatMap(param => {
551
+ let scope = param[0];
552
+ return select(param[1]).map(param => [
553
+ labels(scope, param[0]),
554
+ param[1]
422
555
  ]);
423
556
  });
424
- series(b, "nodejs_gc_duration_seconds_sum", "Cumulative garbage collection pause time by kind, one of major, minor, incremental or weakcb.", "counter", gcEntries, s => s.seconds);
425
- series(b, "nodejs_gc_duration_seconds_count", "Number of garbage collection pauses by kind, one of major, minor, incremental or weakcb.", "counter", gcEntries, s => s.count);
426
- let version = Process.version;
427
- let versionParts = version.replace("v", "").split(".");
428
- let versionPart = i => Stdlib_Option.getOr(versionParts[i], "0");
429
- series(b, "nodejs_version_info", "Node.js version info.", "gauge", [[
430
- `{version="` + version + `",major="` + versionPart(0) + `",minor="` + versionPart(1) + `",patch="` + versionPart(2) + `"}`,
431
- undefined
432
- ]], () => 1);
557
+ let gauge = (name, help, value) => series(b, name, help, "gauge", scoped, value);
558
+ let counter = (name, help, value) => series(b, name, help, "counter", scoped, value);
559
+ counter("process_cpu_user_seconds_total", "Total user CPU time spent in seconds.", s => s.cpuUserSeconds);
560
+ counter("process_cpu_system_seconds_total", "Total system CPU time spent in seconds.", s => s.cpuSystemSeconds);
561
+ counter("process_cpu_seconds_total", "Total user and system CPU time spent in seconds.", s => s.cpuUserSeconds + s.cpuSystemSeconds);
562
+ gauge("process_start_time_seconds", "Start time of the process since unix epoch in seconds.", s => s.processStartTimeSeconds);
563
+ gauge("process_resident_memory_bytes", "Resident memory size in bytes.", s => s.residentMemoryBytes);
564
+ gauge("nodejs_heap_size_total_bytes", "Process heap size from Node.js in bytes.", s => s.heapTotalBytes);
565
+ gauge("nodejs_heap_size_used_bytes", "Process heap size used from Node.js in bytes.", s => s.heapUsedBytes);
566
+ gauge("nodejs_external_memory_bytes", "Node.js external memory size in bytes.", s => s.externalMemoryBytes);
567
+ gauge("nodejs_eventloop_utilization", "Ratio of time the event loop is active, since process start.", s => s.eventLoopUtilization);
568
+ gauge("nodejs_eventloop_lag_mean_seconds", "The mean of the recorded event loop delays.", s => s.eventLoopLagMeanSeconds);
569
+ gauge("nodejs_eventloop_lag_min_seconds", "The minimum recorded event loop delay.", s => s.eventLoopLagMinSeconds);
570
+ gauge("nodejs_eventloop_lag_max_seconds", "The maximum recorded event loop delay.", s => s.eventLoopLagMaxSeconds);
571
+ gauge("nodejs_eventloop_lag_stddev_seconds", "The standard deviation of the recorded event loop delays.", s => s.eventLoopLagStddevSeconds);
572
+ gauge("nodejs_eventloop_lag_p50_seconds", "The 50th percentile of the recorded event loop delays.", s => s.eventLoopLagP50Seconds);
573
+ gauge("nodejs_eventloop_lag_p90_seconds", "The 90th percentile of the recorded event loop delays.", s => s.eventLoopLagP90Seconds);
574
+ gauge("nodejs_eventloop_lag_p99_seconds", "The 99th percentile of the recorded event loop delays.", s => s.eventLoopLagP99Seconds);
575
+ let heapSpaces = each(s => s.heapSpaces.map(h => [
576
+ `space="` + h.space + `"`,
577
+ h
578
+ ]));
579
+ series(b, "nodejs_heap_space_size_total_bytes", "Process heap space size total from Node.js in bytes.", "gauge", heapSpaces, h => h.size);
580
+ series(b, "nodejs_heap_space_size_used_bytes", "Process heap space size used from Node.js in bytes.", "gauge", heapSpaces, h => h.used);
581
+ series(b, "nodejs_heap_space_size_available_bytes", "Process heap space size available from Node.js in bytes.", "gauge", heapSpaces, h => h.available);
582
+ series(b, "nodejs_active_resources", "Number of active resources that are currently keeping the event loop alive, grouped by async resource type.", "gauge", each(s => s.activeResources.map(param => [
583
+ `type="` + escapeLabelValue(param[0]) + `"`,
584
+ param[1]
585
+ ])), count => count);
586
+ gauge("nodejs_active_resources_total", "Total number of active resources.", s => Stdlib_Array.reduce(s.activeResources, 0, (acc, param) => acc + param[1]));
587
+ let gc = each(s => s.gc.map(g => [
588
+ `kind="` + g.kind + `"`,
589
+ g
590
+ ]));
591
+ series(b, "nodejs_gc_duration_seconds_sum", "Cumulative garbage collection pause time by kind, one of major, minor, incremental or weakcb.", "counter", gc, g => g.seconds);
592
+ series(b, "nodejs_gc_duration_seconds_count", "Number of garbage collection pauses by kind, one of major, minor, incremental or weakcb.", "counter", gc, g => g.count);
593
+ series(b, "nodejs_version_info", "Node.js version info.", "gauge", each(s => {
594
+ let parts = s.nodeVersion.replace("v", "").split(".");
595
+ let part = i => Stdlib_Option.getOr(parts[i], "0");
596
+ return [[
597
+ `version="` + s.nodeVersion + `",major="` + part(0) + `",minor="` + part(1) + `",patch="` + part(2) + `"`,
598
+ undefined
599
+ ]];
600
+ }), () => 1);
433
601
  return b.out + "\n";
434
602
  }
435
603
 
604
+ function collectRuntime() {
605
+ return renderRuntime([[
606
+ "",
607
+ sampleRuntime()
608
+ ]]);
609
+ }
610
+
436
611
  let contentType = "text/plain; version=0.0.4; charset=utf-8";
437
612
 
438
613
  export {
439
614
  hasProcessedToEndblock,
615
+ sumByKey,
616
+ merge,
440
617
  formatValue,
441
618
  escapeLabelValue,
442
619
  block,
@@ -451,6 +628,8 @@ export {
451
628
  runtimeCollectors,
452
629
  getRuntimeCollectors,
453
630
  startRuntimeCollectors,
631
+ sampleRuntime,
632
+ renderRuntime,
454
633
  collectRuntime,
455
634
  }
456
635
  /* v8 Not a pure module */
@@ -324,7 +324,10 @@ let init = {
324
324
  | _ => false
325
325
  }
326
326
  ) {
327
- Logging.info(`Found existing indexer storage. Resuming indexing state...`)
327
+ // An isolated process resumes state its supervisor already announced
328
+ // for the whole run, so it says so only to its own log file.
329
+ let logResume = requireInitialized ? Logging.debug : Logging.info
330
+ logResume(`Found existing indexer storage. Resuming indexing state...`)
328
331
  let initialState = await persistence.storage.resumeInitialState(
329
332
  ~entities=persistence.allEntities,
330
333
  ~chainIds=chainConfigs->Array.map(chain => chain.id),
@@ -343,7 +346,7 @@ let init = {
343
346
  initialState.chains->Array.forEach(c => {
344
347
  progress->ChainId.Dict.set(c.id, c.progressBlockNumber)
345
348
  })
346
- Logging.info({
349
+ logResume({
347
350
  "msg": `Successfully resumed indexing state! Continuing from the last checkpoint.`,
348
351
  "progress": progress,
349
352
  })
@@ -356,6 +359,28 @@ let init = {
356
359
  }
357
360
  }
358
361
 
362
+ // Brings the schema up to date for a run that is about to start, as opposed to
363
+ // a migration command: what a config change prints names the command the
364
+ // operator ran, and an unreachable chain is waited on rather than reported,
365
+ // since somebody is watching the run come up.
366
+ let initForRun = (
367
+ persistence,
368
+ ~config: Config.t,
369
+ ~reset,
370
+ ~isDevelopmentMode,
371
+ ~requireInitialized,
372
+ ) =>
373
+ persistence->init(
374
+ ~reset,
375
+ ~chainConfigs=config.chainMap->ChainMap.values,
376
+ ~contractMapping=config.contractMapping,
377
+ ~envioInfo=Config.envioInfo(),
378
+ ~resetCommand=isDevelopmentMode ? "envio dev -r" : "envio start -r",
379
+ ~runCommand=Some(isDevelopmentMode ? "envio dev" : "envio start"),
380
+ ~lowercaseAddresses=config.lowercaseAddresses,
381
+ ~requireInitialized,
382
+ )
383
+
359
384
  let getInitializedStorageOrThrow = persistence => {
360
385
  switch persistence.storageStatus {
361
386
  | Unknown
@@ -3,6 +3,7 @@
3
3
  import * as Batch from "./Batch.res.mjs";
4
4
  import * as Config from "./Config.res.mjs";
5
5
  import * as Logging from "./Logging.res.mjs";
6
+ import * as ChainMap from "./ChainMap.res.mjs";
6
7
  import * as Frontier from "./db/Frontier.res.mjs";
7
8
  import * as EntityHistory from "./db/EntityHistory.res.mjs";
8
9
  import * as ErrorHandling from "./ErrorHandling.res.mjs";
@@ -77,7 +78,8 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
77
78
  let tmp;
78
79
  tmp = typeof match !== "object" ? false : match.TAG === "Initializing";
79
80
  if (tmp) {
80
- Logging.info(`Found existing indexer storage. Resuming indexing state...`);
81
+ let logResume = requireInitialized ? Logging.debug : Logging.info;
82
+ logResume(`Found existing indexer storage. Resuming indexing state...`);
81
83
  let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, chainConfigs.map(chain => chain.id), (storedEnvioInfo, storedContractMapping) => Config.throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand));
82
84
  persistence.storageStatus = {
83
85
  TAG: "Ready",
@@ -87,7 +89,7 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
87
89
  initialState$1.chains.forEach(c => {
88
90
  progress[c.id] = c.progressBlockNumber;
89
91
  });
90
- Logging.info({
92
+ logResume({
91
93
  msg: `Successfully resumed indexing state! Continuing from the last checkpoint.`,
92
94
  progress: progress
93
95
  });
@@ -100,6 +102,10 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
100
102
  }
101
103
  }
102
104
 
105
+ function initForRun(persistence, config, reset, isDevelopmentMode, requireInitialized) {
106
+ return init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.envioInfo(), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset, config.lowercaseAddresses, requireInitialized, undefined);
107
+ }
108
+
103
109
  function getInitializedStorageOrThrow(persistence) {
104
110
  let match = persistence.storageStatus;
105
111
  if (typeof match !== "object" || match.TAG === "Initializing") {
@@ -123,6 +129,7 @@ export {
123
129
  StorageError,
124
130
  make,
125
131
  init,
132
+ initForRun,
126
133
  getInitializedStorageOrThrow,
127
134
  getInitializedState,
128
135
  }
package/src/PgStorage.res CHANGED
@@ -1,4 +1,4 @@
1
- let makeClient = () => {
1
+ let makeClient = (~maxConnections=Env.Db.maxConnections) => {
2
2
  Postgres.makeSql(
3
3
  ~config={
4
4
  host: Env.Db.host,
@@ -14,7 +14,7 @@ let makeClient = () => {
14
14
  : Some(_str => ())
15
15
  ),
16
16
  transform: {undefined: Null},
17
- max: Env.Db.maxConnections,
17
+ max: maxConnections,
18
18
  // debug: (~connection, ~query, ~params as _, ~types as _) => Js.log2(connection, query),
19
19
  },
20
20
  )
@@ -2264,13 +2264,17 @@ let make = (
2264
2264
  }
2265
2265
 
2266
2266
  switch missing {
2267
- | [] =>
2267
+ // A schema that declares no indexes has nothing to say about them, and one
2268
+ // whose indexes are all in place says it once. Either way the line that
2269
+ // matters is the indexer reporting itself ready, which finalization logs.
2270
+ | [] if schemaIndexes->Utils.Array.notEmpty =>
2268
2271
  Logging.info({
2269
2272
  "storage": storageName,
2270
2273
  "msg": `All ${schemaIndexes
2271
2274
  ->Array.length
2272
2275
  ->Int.toString} schema indexes are already in place. Marking the indexer ready.`,
2273
2276
  })
2277
+ | [] => ()
2274
2278
  | _ =>
2275
2279
  Logging.info({
2276
2280
  "storage": storageName,
@@ -2318,12 +2322,16 @@ let make = (
2318
2322
  }
2319
2323
  })
2320
2324
 
2321
- Logging.info({
2322
- "storage": storageName,
2323
- "msg": `Committed ${missing
2324
- ->Array.length
2325
- ->Int.toString} schema indexes and the ready timestamp in ${timeRef->formatSeconds}s.`,
2326
- })
2325
+ // Only when something was built: the wait this closes is the index build,
2326
+ // and the stamp on its own is not one anybody waited through.
2327
+ if missing->Utils.Array.notEmpty {
2328
+ Logging.info({
2329
+ "storage": storageName,
2330
+ "msg": `Committed ${missing
2331
+ ->Array.length
2332
+ ->Int.toString} schema indexes and the ready timestamp in ${timeRef->formatSeconds}s.`,
2333
+ })
2334
+ }
2327
2335
  }
2328
2336
 
2329
2337
  let setOrThrow = (
@@ -38,7 +38,8 @@ import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
38
38
  import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
39
39
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
40
40
 
41
- function makeClient() {
41
+ function makeClient(maxConnectionsOpt) {
42
+ let maxConnections = maxConnectionsOpt !== undefined ? maxConnectionsOpt : Env.Db.maxConnections;
42
43
  return Postgres({
43
44
  host: Env.Db.host,
44
45
  port: Env.Db.port,
@@ -46,7 +47,7 @@ function makeClient() {
46
47
  username: Env.Db.user,
47
48
  password: Env.Db.password,
48
49
  ssl: Env.Db.ssl,
49
- max: Env.Db.maxConnections,
50
+ max: maxConnections,
50
51
  onnotice: Primitive_object.equal(Env.userLogLevel, "warn") || Primitive_object.equal(Env.userLogLevel, "error") ? undefined : _str => {},
51
52
  transform: {
52
53
  undefined: null
@@ -1407,7 +1408,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1407
1408
  msg: `Creating the ` + missing.length.toString() + ` remaining schema indexes before the indexer reports ready. Writes are paused until they are committed. ` + slowOnLargeDatabaseNotice,
1408
1409
  indexes: missing.map(prepared => prepared.name)
1409
1410
  });
1410
- } else {
1411
+ } else if (Utils.$$Array.notEmpty(schemaIndexes)) {
1411
1412
  Logging.info({
1412
1413
  storage: storageName,
1413
1414
  msg: `All ` + schemaIndexes.length.toString() + ` schema indexes are already in place. Marking the indexer ready.`
@@ -1438,10 +1439,12 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1438
1439
  ], {prepare: true});
1439
1440
  }
1440
1441
  });
1441
- return Logging.info({
1442
- storage: storageName,
1443
- msg: `Committed ` + missing.length.toString() + ` schema indexes and the ready timestamp in ` + formatSeconds(timeRef) + `s.`
1444
- });
1442
+ if (Utils.$$Array.notEmpty(missing)) {
1443
+ return Logging.info({
1444
+ storage: storageName,
1445
+ msg: `Committed ` + missing.length.toString() + ` schema indexes and the ready timestamp in ` + formatSeconds(timeRef) + `s.`
1446
+ });
1447
+ }
1445
1448
  };
1446
1449
  let setOrThrow$1 = (items, table, itemSchema) => setOrThrow(sql, items, table, itemSchema, pgSchema, setQueryCache, chainIdMode);
1447
1450
  let setEffectCacheOrThrow = async (table, itemSchema, items, initialize) => {
@@ -1663,7 +1666,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1663
1666
  }
1664
1667
 
1665
1668
  function makeStorageFromEnv(config, sqlOpt, pgSchemaOpt, isHasuraEnabledOpt) {
1666
- let sql = sqlOpt !== undefined ? Primitive_option.valFromOption(sqlOpt) : makeClient();
1669
+ let sql = sqlOpt !== undefined ? Primitive_option.valFromOption(sqlOpt) : makeClient(undefined);
1667
1670
  let pgSchema = pgSchemaOpt !== undefined ? pgSchemaOpt : Env.Db.publicSchema;
1668
1671
  let isHasuraEnabled = isHasuraEnabledOpt !== undefined ? isHasuraEnabledOpt : Env.Hasura.enabled;
1669
1672
  let tmp;