envio 3.11.0 → 3.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/index.d.ts +3 -0
  2. package/package.json +6 -6
  3. package/src/Batch.res +111 -78
  4. package/src/Batch.res.mjs +53 -46
  5. package/src/BatchProcessing.res +1 -11
  6. package/src/BatchProcessing.res.mjs +1 -4
  7. package/src/ChainFetching.res +7 -3
  8. package/src/ChainFetching.res.mjs +1 -1
  9. package/src/ChainState.res +82 -20
  10. package/src/ChainState.res.mjs +51 -13
  11. package/src/ChainState.resi +6 -1
  12. package/src/Config.res +58 -37
  13. package/src/Config.res.mjs +63 -53
  14. package/src/CrossChainState.res +50 -53
  15. package/src/CrossChainState.res.mjs +25 -23
  16. package/src/CrossChainState.resi +6 -13
  17. package/src/EffectState.res +13 -4
  18. package/src/EffectState.resi +7 -1
  19. package/src/EventProcessing.res +6 -6
  20. package/src/EventProcessing.res.mjs +6 -6
  21. package/src/FetchState.res +65 -46
  22. package/src/FetchState.res.mjs +68 -36
  23. package/src/FinalizeBackfill.res +16 -12
  24. package/src/FinalizeBackfill.res.mjs +3 -3
  25. package/src/HistoryPolicy.res +46 -0
  26. package/src/HistoryPolicy.res.mjs +50 -0
  27. package/src/InMemoryStore.res +30 -32
  28. package/src/InMemoryStore.res.mjs +33 -28
  29. package/src/InMemoryTable.res +155 -67
  30. package/src/InMemoryTable.res.mjs +151 -60
  31. package/src/IndexerState.res +66 -45
  32. package/src/IndexerState.res.mjs +57 -29
  33. package/src/IndexerState.resi +8 -13
  34. package/src/Internal.res +4 -2
  35. package/src/LoadLayer.res +68 -38
  36. package/src/LoadLayer.res.mjs +50 -64
  37. package/src/LoadLayer.resi +1 -1
  38. package/src/Main.res +25 -24
  39. package/src/Main.res.mjs +24 -25
  40. package/src/Metrics.res +49 -1
  41. package/src/Metrics.res.mjs +26 -3
  42. package/src/Persistence.res +55 -16
  43. package/src/Persistence.res.mjs +18 -3
  44. package/src/PgStorage.res +286 -180
  45. package/src/PgStorage.res.mjs +209 -122
  46. package/src/PruneStaleHistory.res +9 -15
  47. package/src/PruneStaleHistory.res.mjs +11 -20
  48. package/src/Rollback.res +9 -14
  49. package/src/Rollback.res.mjs +5 -3
  50. package/src/Sink.res +7 -5
  51. package/src/Sink.res.mjs +4 -4
  52. package/src/Staging.res +298 -0
  53. package/src/Staging.res.mjs +257 -0
  54. package/src/TestIndexer.res +14 -30
  55. package/src/TestIndexer.res.mjs +11 -9
  56. package/src/UserContext.res +20 -36
  57. package/src/UserContext.res.mjs +6 -9
  58. package/src/Utils.res +10 -4
  59. package/src/Utils.res.mjs +16 -16
  60. package/src/Writing.res +34 -20
  61. package/src/Writing.res.mjs +15 -20
  62. package/src/bindings/ClickHouse.res +41 -34
  63. package/src/bindings/ClickHouse.res.mjs +36 -30
  64. package/src/bindings/ClickHouseSink.res +45 -186
  65. package/src/bindings/ClickHouseSink.res.mjs +21 -174
  66. package/src/db/CheckpointSequence.res +109 -0
  67. package/src/db/CheckpointSequence.res.mjs +99 -0
  68. package/src/db/EntityFilter.res +487 -275
  69. package/src/db/EntityFilter.res.mjs +557 -309
  70. package/src/db/EntityHistory.res +6 -5
  71. package/src/db/EntityHistory.res.mjs +5 -5
  72. package/src/db/Frontier.res +86 -0
  73. package/src/db/Frontier.res.mjs +126 -0
  74. package/src/db/InternalTable.res +145 -27
  75. package/src/db/InternalTable.res.mjs +115 -29
  76. package/src/db/RollbackFloors.res +34 -41
  77. package/src/db/RollbackFloors.res.mjs +27 -71
  78. package/src/db/Table.res +21 -6
  79. package/src/db/Table.res.mjs +13 -4
  80. package/src/sources/BlockStore.res +8 -0
  81. package/src/sources/EvmHyperSyncSource.res +5 -1
  82. package/src/sources/EvmHyperSyncSource.res.mjs +4 -3
  83. package/src/sources/FuelHyperSyncSource.res +1 -0
  84. package/src/sources/FuelHyperSyncSource.res.mjs +1 -1
  85. package/src/sources/HyperSync.res +6 -0
  86. package/src/sources/HyperSync.res.mjs +5 -2
  87. package/src/sources/HyperSync.resi +2 -0
  88. package/src/sources/HyperSyncClient.res +6 -0
  89. package/src/sources/HyperSyncSSE.res +1 -1
  90. package/src/sources/HyperSyncSSE.res.mjs +4 -10
  91. package/src/sources/RequestStat.res +8 -1
  92. package/src/sources/RpcSource.res +1 -0
  93. package/src/sources/RpcSource.res.mjs +1 -1
  94. package/src/sources/SimulateSource.res +1 -0
  95. package/src/sources/SimulateSource.res.mjs +1 -1
  96. package/src/sources/Source.res +11 -3
  97. package/src/sources/SourceManager.res +34 -13
  98. package/src/sources/SourceManager.res.mjs +26 -9
  99. package/src/sources/SourceManager.resi +2 -0
  100. package/src/sources/SvmHyperSyncClient.res +5 -0
  101. package/src/sources/SvmHyperSyncSource.res +3 -0
  102. package/src/sources/SvmHyperSyncSource.res.mjs +4 -2
  103. package/src/tui/Tui.res +56 -143
  104. package/src/tui/Tui.res.mjs +70 -166
  105. package/src/tui/components/TuiData.res +61 -16
  106. package/src/tui/components/TuiData.res.mjs +53 -15
  107. package/svm.schema.json +0 -7
  108. package/src/db/CheckpointBounds.res +0 -53
  109. package/src/db/CheckpointBounds.res.mjs +0 -44
@@ -10,22 +10,17 @@ type synced = {
10
10
 
11
11
  type progress = SearchingForEvents | Syncing(syncing) | Synced(synced)
12
12
 
13
- let getNumberOfEventsProccessed = (progress: progress) => {
14
- switch progress {
15
- | SearchingForEvents => 0.
16
- | Syncing(syncing) => syncing.numEventsProcessed
17
- | Synced(synced) => synced.numEventsProcessed
18
- }
19
- }
20
13
  type chain = {
21
14
  chainId: string,
22
15
  eventsProcessed: float,
23
- progressBlock: option<int>,
24
- bufferBlock: option<int>,
25
- sourceBlock: option<int>,
16
+ // Clamped into [startBlock, toBlock]: the source height is 0 until the first
17
+ // height fetch lands, and the buffer starts one block below startBlock, so raw
18
+ // values can fall outside the range the progress bar counts up to.
19
+ progressBlock: int,
20
+ bufferBlock: int,
21
+ toBlock: int,
26
22
  startBlock: int,
27
23
  endBlock: option<int>,
28
- firstEventBlockNumber: option<int>,
29
24
  poweredByHyperSync: bool,
30
25
  progress: progress,
31
26
  latestFetchedBlockNumber: int,
@@ -34,14 +29,64 @@ type chain = {
34
29
  `"Blocks"` everywhere else. Drives the per-chain progress label. */
35
30
  blockUnit: string,
36
31
  rateLimitTimeMs: float,
37
- isRateLimited: bool,
38
32
  rateLimitResetInMs: option<float>,
39
33
  }
40
34
 
41
- let minOfOption: (int, option<int>) => int = (a: int, b: option<int>) => {
42
- switch (a, b) {
43
- | (a, Some(b)) => min(a, b)
44
- | (a, None) => a
35
+ let fromChainMetrics = (m: Metrics.chainMetrics, ~blockUnit): chain => {
36
+ // A chain can reach its end block without ever matching an event, so it
37
+ // still has to render as synced with no first event block to report.
38
+ let firstEventBlockNumber = m.firstEventBlockNumber->Option.getOr(0)
39
+ let syncing: syncing = {
40
+ firstEventBlockNumber,
41
+ latestProcessedBlock: m.progressBlockNumber,
42
+ numEventsProcessed: m.numEventsProcessed,
43
+ }
44
+ let synced = (~timestampCaughtUpToHeadOrEndblock): progress => Synced({
45
+ firstEventBlockNumber,
46
+ latestProcessedBlock: m.progressBlockNumber,
47
+ numEventsProcessed: m.numEventsProcessed,
48
+ timestampCaughtUpToHeadOrEndblock,
49
+ })
50
+ let progress = if m->Metrics.hasProcessedToEndblock {
51
+ synced(
52
+ ~timestampCaughtUpToHeadOrEndblock=m.timestampCaughtUpToHeadOrEndblock->Option.getOr(
53
+ Date.now()->Date.fromTime,
54
+ ),
55
+ )
56
+ } else {
57
+ switch (m.firstEventBlockNumber, m.timestampCaughtUpToHeadOrEndblock) {
58
+ | (Some(_), Some(timestampCaughtUpToHeadOrEndblock)) =>
59
+ synced(~timestampCaughtUpToHeadOrEndblock)
60
+ | (Some(_), None) => Syncing(syncing)
61
+ | (None, _) => SearchingForEvents
62
+ }
63
+ }
64
+
65
+ let toBlock = Pervasives.max(
66
+ switch m.endBlock {
67
+ | Some(endBlock) => Pervasives.min(m.sourceBlockNumber, endBlock)
68
+ | None => m.sourceBlockNumber
69
+ },
70
+ m.startBlock,
71
+ )
72
+ let clampToRange = blockNumber =>
73
+ blockNumber->Pervasives.max(m.startBlock)->Pervasives.min(toBlock)
74
+
75
+ {
76
+ progress,
77
+ chainId: m.chainId->ChainId.toString,
78
+ eventsProcessed: m.numEventsProcessed,
79
+ progressBlock: m.progressBlockNumber->clampToRange,
80
+ bufferBlock: m.latestFetchedBlockNumber->clampToRange,
81
+ toBlock,
82
+ startBlock: m.startBlock,
83
+ endBlock: m.endBlock,
84
+ poweredByHyperSync: m.poweredByHyperSync,
85
+ latestFetchedBlockNumber: m.latestFetchedBlockNumber,
86
+ knownHeight: m.knownHeight,
87
+ blockUnit,
88
+ rateLimitTimeMs: m.rateLimitTimeMs,
89
+ rateLimitResetInMs: m.rateLimitResetInMs,
45
90
  }
46
91
  }
47
92
 
@@ -1,21 +1,60 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
+ import * as ChainId from "../../ChainId.res.mjs";
4
+ import * as Metrics from "../../Metrics.res.mjs";
3
5
  import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
6
+ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
4
7
 
5
- function getNumberOfEventsProccessed(progress) {
6
- if (typeof progress !== "object") {
7
- return 0;
8
+ function fromChainMetrics(m, blockUnit) {
9
+ let firstEventBlockNumber = Stdlib_Option.getOr(m.firstEventBlockNumber, 0);
10
+ let syncing_latestProcessedBlock = m.progressBlockNumber;
11
+ let syncing_numEventsProcessed = m.numEventsProcessed;
12
+ let syncing = {
13
+ firstEventBlockNumber: firstEventBlockNumber,
14
+ latestProcessedBlock: syncing_latestProcessedBlock,
15
+ numEventsProcessed: syncing_numEventsProcessed
16
+ };
17
+ let synced = timestampCaughtUpToHeadOrEndblock => ({
18
+ TAG: "Synced",
19
+ _0: {
20
+ firstEventBlockNumber: firstEventBlockNumber,
21
+ latestProcessedBlock: m.progressBlockNumber,
22
+ numEventsProcessed: m.numEventsProcessed,
23
+ timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock
24
+ }
25
+ });
26
+ let progress;
27
+ if (Metrics.hasProcessedToEndblock(m)) {
28
+ progress = synced(Stdlib_Option.getOr(m.timestampCaughtUpToHeadOrEndblock, new Date(Date.now())));
8
29
  } else {
9
- return progress._0.numEventsProcessed;
10
- }
11
- }
12
-
13
- function minOfOption(a, b) {
14
- if (b !== undefined) {
15
- return Primitive_int.min(a, b);
16
- } else {
17
- return a;
30
+ let match = m.firstEventBlockNumber;
31
+ let match$1 = m.timestampCaughtUpToHeadOrEndblock;
32
+ progress = match !== undefined ? (
33
+ match$1 !== undefined ? synced(match$1) : ({
34
+ TAG: "Syncing",
35
+ _0: syncing
36
+ })
37
+ ) : "SearchingForEvents";
18
38
  }
39
+ let endBlock = m.endBlock;
40
+ let toBlock = Primitive_int.max(endBlock !== undefined ? Primitive_int.min(m.sourceBlockNumber, endBlock) : m.sourceBlockNumber, m.startBlock);
41
+ let clampToRange = blockNumber => Primitive_int.min(Primitive_int.max(blockNumber, m.startBlock), toBlock);
42
+ return {
43
+ chainId: ChainId.toString(m.chainId),
44
+ eventsProcessed: m.numEventsProcessed,
45
+ progressBlock: clampToRange(m.progressBlockNumber),
46
+ bufferBlock: clampToRange(m.latestFetchedBlockNumber),
47
+ toBlock: toBlock,
48
+ startBlock: m.startBlock,
49
+ endBlock: m.endBlock,
50
+ poweredByHyperSync: m.poweredByHyperSync,
51
+ progress: progress,
52
+ latestFetchedBlockNumber: m.latestFetchedBlockNumber,
53
+ knownHeight: m.knownHeight,
54
+ blockUnit: blockUnit,
55
+ rateLimitTimeMs: m.rateLimitTimeMs,
56
+ rateLimitResetInMs: m.rateLimitResetInMs
57
+ };
19
58
  }
20
59
 
21
60
  function formatLocaleString(n) {
@@ -27,9 +66,8 @@ function formatFloatLocaleString(n) {
27
66
  }
28
67
 
29
68
  export {
30
- getNumberOfEventsProccessed,
31
- minOfOption,
69
+ fromChainMetrics,
32
70
  formatLocaleString,
33
71
  formatFloatLocaleString,
34
72
  }
35
- /* No side effect */
73
+ /* ChainId Not a pure module */
package/svm.schema.json CHANGED
@@ -341,13 +341,6 @@
341
341
  "description": "Base58-encoded program id (32 bytes). A single value is allowed only when the config defines one chain; with several, give a mapping keyed by chain id that names every one of them, writing `_` for a chain the program is not deployed on.",
342
342
  "$ref": "#/$defs/SvmProgramId"
343
343
  },
344
- "handler": {
345
- "description": "Optional relative path to a file where handlers are registered for the given program. If not provided, handlers can be auto-loaded from the src directory.",
346
- "type": [
347
- "string",
348
- "null"
349
- ]
350
- },
351
344
  "idl": {
352
345
  "description": "Optional path (relative to config.yaml) to an IDL JSON file (Anchor 0.30+, legacy Anchor, Shank, or Codama). When present, every usable instruction is in the catalog and `onInstruction` selects by name. Omit YAML `instructions` to take that catalog as-is. A YAML row overwrites the IDL instruction of the same name, or adds a name the IDL did not declare.",
353
346
  "type": [
@@ -1,53 +0,0 @@
1
- // A checkpoint id per chain that a query compares each chain's rows against. A
2
- // schema with a cross-chain entity gets one bound for every chain, since a reorg
3
- // on any chain reaches every chain's rows; otherwise each chain has its own, and
4
- // a chain absent from them is left alone.
5
- type t =
6
- | EveryChain(Internal.checkpointId)
7
- | PerChain(array<(ChainId.t, Internal.checkpointId)>)
8
-
9
- // The pieces a query splices in to read its bound as `checkpointId`: a SELECT
10
- // joins the bounds next to its table with `join`, a DELETE names them with
11
- // `using` and matches the chain with `usingMatch` in its WHERE.
12
- type sql = {
13
- join: string,
14
- using: string,
15
- usingMatch: string,
16
- checkpointId: string,
17
- }
18
-
19
- // The chain ids are cast to the widest integer type rather than the column's
20
- // own: nothing indexes the join, so the values only have to compare.
21
- let relation = `unnest($1::${(Postgres.BigInt :> string)}[],$2::${(Postgres.BigInt :> string)}[]) AS envio_bounds(chain_id, checkpoint_id)`
22
-
23
- let sql = (bounds: t, ~chainIdColumn: option<string>, ~tableRef: string): sql =>
24
- switch (bounds, chainIdColumn) {
25
- | (EveryChain(_), _) => {join: "", using: "", usingMatch: "", checkpointId: "$1"}
26
- | (PerChain(_), Some(column)) =>
27
- let chainMatch = `envio_bounds.chain_id = ${tableRef}."${column}"`
28
- {
29
- join: ` JOIN ${relation} ON ${chainMatch}`,
30
- using: ` USING ${relation}`,
31
- usingMatch: ` AND ${chainMatch}`,
32
- checkpointId: "envio_bounds.checkpoint_id",
33
- }
34
- | (PerChain(_), None) =>
35
- JsError.throwWithMessage(
36
- "Internal error: per-chain checkpoint bounds can't bound a table with no chain-id column. Only a schema whose entities are all per-chain has them.",
37
- )
38
- }
39
-
40
- // Bound in the order `sql` reads them: the one bound as `$1`, or the chains and
41
- // their bounds as the two parallel arrays the relation unnests.
42
- let params = (bounds: t): unknown =>
43
- switch bounds {
44
- | EveryChain(checkpointId) =>
45
- [checkpointId->BigInt.toString]->(Utils.magic: array<string> => unknown)
46
- | PerChain(byChain) =>
47
- [
48
- byChain->Array.map(((chainId, _)) => chainId)->(Utils.magic: array<ChainId.t> => unknown),
49
- byChain
50
- ->Array.map(((_, checkpointId)) => checkpointId->BigInt.toString)
51
- ->(Utils.magic: array<string> => unknown),
52
- ]->(Utils.magic: array<unknown> => unknown)
53
- }
@@ -1,44 +0,0 @@
1
- // Generated by ReScript, PLEASE EDIT WITH CARE
2
-
3
- import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
4
-
5
- let relation = `unnest($1::` + "BIGINT" + `[],$2::` + "BIGINT" + `[]) AS envio_bounds(chain_id, checkpoint_id)`;
6
-
7
- function sql(bounds, chainIdColumn, tableRef) {
8
- if (bounds.TAG === "EveryChain") {
9
- return {
10
- join: "",
11
- using: "",
12
- usingMatch: "",
13
- checkpointId: "$1"
14
- };
15
- }
16
- if (chainIdColumn === undefined) {
17
- return Stdlib_JsError.throwWithMessage("Internal error: per-chain checkpoint bounds can't bound a table with no chain-id column. Only a schema whose entities are all per-chain has them.");
18
- }
19
- let chainMatch = `envio_bounds.chain_id = ` + tableRef + `."` + chainIdColumn + `"`;
20
- return {
21
- join: ` JOIN ` + relation + ` ON ` + chainMatch,
22
- using: ` USING ` + relation,
23
- usingMatch: ` AND ` + chainMatch,
24
- checkpointId: "envio_bounds.checkpoint_id"
25
- };
26
- }
27
-
28
- function params(bounds) {
29
- if (bounds.TAG === "EveryChain") {
30
- return [bounds._0.toString()];
31
- }
32
- let byChain = bounds._0;
33
- return [
34
- byChain.map(param => param[0]),
35
- byChain.map(param => param[1].toString())
36
- ];
37
- }
38
-
39
- export {
40
- relation,
41
- sql,
42
- params,
43
- }
44
- /* No side effect */