envio 3.9.0 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/evm.schema.json +26 -0
  2. package/fuel.schema.json +26 -0
  3. package/index.d.ts +35 -3
  4. package/package.json +6 -7
  5. package/src/Batch.res +9 -0
  6. package/src/ChainState.res +35 -39
  7. package/src/ChainState.res.mjs +14 -11
  8. package/src/ChainState.resi +15 -7
  9. package/src/Config.res +52 -44
  10. package/src/Config.res.mjs +51 -17
  11. package/src/Core.res +4 -0
  12. package/src/CrossChainState.res +15 -23
  13. package/src/CrossChainState.res.mjs +10 -18
  14. package/src/CrossChainState.resi +4 -1
  15. package/src/Ecosystem.res +7 -2
  16. package/src/Envio.res +6 -4
  17. package/src/EventConfigBuilder.res +169 -26
  18. package/src/EventConfigBuilder.res.mjs +124 -15
  19. package/src/HandlerRegister.res +21 -4
  20. package/src/HandlerRegister.res.mjs +24 -5
  21. package/src/InMemoryStore.res +9 -7
  22. package/src/InMemoryStore.res.mjs +3 -3
  23. package/src/IndexerState.res +43 -6
  24. package/src/IndexerState.res.mjs +36 -7
  25. package/src/IndexerState.resi +4 -4
  26. package/src/Internal.res +16 -17
  27. package/src/Metrics.res +72 -6
  28. package/src/Metrics.res.mjs +30 -2
  29. package/src/Persistence.res +36 -30
  30. package/src/Persistence.res.mjs +1 -20
  31. package/src/PgStorage.res +104 -56
  32. package/src/PgStorage.res.mjs +57 -35
  33. package/src/PruneStaleHistory.res +85 -52
  34. package/src/PruneStaleHistory.res.mjs +63 -37
  35. package/src/Rollback.res +68 -45
  36. package/src/Rollback.res.mjs +38 -19
  37. package/src/RollbackCommit.res +7 -9
  38. package/src/RollbackCommit.res.mjs +5 -6
  39. package/src/SafeCheckpointTracking.res +11 -11
  40. package/src/SafeCheckpointTracking.res.mjs +7 -6
  41. package/src/SimulateItems.res +33 -18
  42. package/src/SimulateItems.res.mjs +26 -22
  43. package/src/Sink.res +38 -26
  44. package/src/Sink.res.mjs +21 -16
  45. package/src/TestIndexer.res +5 -5
  46. package/src/TestIndexer.res.mjs +1 -1
  47. package/src/Utils.res +170 -0
  48. package/src/Utils.res.mjs +153 -0
  49. package/src/Writing.res +2 -2
  50. package/src/Writing.res.mjs +1 -1
  51. package/src/bindings/ClickHouse.res +366 -698
  52. package/src/bindings/ClickHouse.res.mjs +371 -421
  53. package/src/bindings/ClickHouseSink.res +337 -0
  54. package/src/bindings/ClickHouseSink.res.mjs +246 -0
  55. package/src/bindings/EventSource.res +8 -2
  56. package/src/bindings/NodeJs.res +6 -0
  57. package/src/bindings/Vitest.res +23 -0
  58. package/src/bindings/Vitest.res.mjs +3 -0
  59. package/src/bindings/WebSocket.res +9 -10
  60. package/src/db/CheckpointBounds.res +53 -0
  61. package/src/db/CheckpointBounds.res.mjs +44 -0
  62. package/src/db/EntityFilter.res +36 -17
  63. package/src/db/EntityFilter.res.mjs +25 -14
  64. package/src/db/EntityHistory.res +47 -22
  65. package/src/db/EntityHistory.res.mjs +19 -12
  66. package/src/db/InternalTable.res +84 -35
  67. package/src/db/InternalTable.res.mjs +56 -23
  68. package/src/db/RollbackFloors.res +55 -0
  69. package/src/db/RollbackFloors.res.mjs +92 -0
  70. package/src/db/Table.res +22 -4
  71. package/src/db/Table.res.mjs +30 -9
  72. package/src/sources/Evm.res +2 -0
  73. package/src/sources/Evm.res.mjs +2 -0
  74. package/src/sources/EvmHyperSyncSource.res +3 -3
  75. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  76. package/src/sources/EvmRpcWs.res +124 -0
  77. package/src/sources/EvmRpcWs.res.mjs +117 -0
  78. package/src/sources/Fuel.res +2 -0
  79. package/src/sources/Fuel.res.mjs +2 -0
  80. package/src/sources/FuelHyperSyncSource.res +1 -1
  81. package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
  82. package/src/sources/HeightFeed.res +568 -0
  83. package/src/sources/HeightFeed.res.mjs +452 -0
  84. package/src/sources/HeightStream.res +257 -0
  85. package/src/sources/HeightStream.res.mjs +194 -0
  86. package/src/sources/HyperSync.res +5 -0
  87. package/src/sources/HyperSync.res.mjs +3 -0
  88. package/src/sources/HyperSync.resi +4 -0
  89. package/src/sources/HyperSyncSSE.res +49 -0
  90. package/src/sources/HyperSyncSSE.res.mjs +79 -0
  91. package/src/sources/RpcSource.res +1 -1
  92. package/src/sources/RpcSource.res.mjs +2 -2
  93. package/src/sources/Source.res +54 -1
  94. package/src/sources/Source.res.mjs +25 -0
  95. package/src/sources/SourceManager.res +220 -227
  96. package/src/sources/SourceManager.res.mjs +139 -164
  97. package/src/sources/SourceManager.resi +8 -0
  98. package/src/sources/Svm.res +2 -0
  99. package/src/sources/Svm.res.mjs +2 -0
  100. package/src/sources/SvmHyperSyncClient.res +70 -127
  101. package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
  102. package/src/sources/SvmHyperSyncSource.res +18 -15
  103. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  104. package/svm.schema.json +1 -75
  105. package/src/MemoryStorage.res +0 -729
  106. package/src/MemoryStorage.res.mjs +0 -586
  107. package/src/sources/HyperSyncHeightStream.res +0 -129
  108. package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
  109. package/src/sources/RpcWebSocketHeightStream.res +0 -175
  110. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
package/src/Sink.res CHANGED
@@ -1,44 +1,56 @@
1
1
  type t = {
2
2
  name: string,
3
- initialize: (
4
- ~chainConfigs: array<Config.chain>=?,
5
- ~entities: array<Internal.entityConfig>=?,
6
- ~enums: array<Table.enumConfig<Table.enum>>=?,
3
+ initialize: (~entities: array<Internal.entityConfig>) => promise<unit>,
4
+ resume: (
5
+ ~checkpointId: Internal.checkpointId,
6
+ ~chains: array<Persistence.initialChainState>,
7
+ ~entities: array<Internal.entityConfig>,
7
8
  ) => promise<unit>,
8
- resume: (~checkpointId: Internal.checkpointId) => promise<unit>,
9
9
  writeBatch: (
10
10
  ~batch: Batch.t,
11
11
  ~updatedEntities: array<Persistence.updatedEntity>,
12
12
  ) => promise<unit>,
13
13
  }
14
14
 
15
- let makeClickHouse = (~host, ~database, ~username, ~password, ~chainIdMode: ChainId.mode=Int32): t => {
16
- let client = ClickHouse.createClient({
17
- url: host,
18
- username,
19
- password,
20
- })
21
-
22
- // Don't pass database to the client; it would fail if the database doesn't
23
- // exist yet. Each query qualifies the name explicitly or runs USE first.
24
-
25
- let cache = Dict.make()
15
+ let makeClickHouse = (
16
+ ~host,
17
+ ~database,
18
+ ~username,
19
+ ~password,
20
+ ~chainIdMode: ChainId.mode=Int32,
21
+ ): t => {
22
+ let sink = ClickHouse.makeSink(~host, ~username, ~password, ~database, ~chainIdMode)
23
+ let registry = ClickHouse.makeRegistry()
24
+ let mirrored = (entities: array<Internal.entityConfig>) =>
25
+ entities->Array.filter(e => e.storage.clickhouse)
26
26
 
27
27
  {
28
28
  name: "clickhouse",
29
- initialize: (~chainConfigs as _=[], ~entities=[], ~enums=[]) => {
30
- ClickHouse.initialize(client, ~database, ~entities, ~enums, ~chainIdMode)
29
+ initialize: (~entities) => {
30
+ ClickHouse.initialize(sink, ~entities=mirrored(entities))
31
31
  },
32
- resume: (~checkpointId) => {
33
- ClickHouse.resume(client, ~database, ~checkpointId)
32
+ resume: (~checkpointId, ~chains, ~entities) => {
33
+ ClickHouse.resume(sink, ~checkpointId, ~chains, ~entities=mirrored(entities))
34
34
  },
35
35
  writeBatch: async (~batch, ~updatedEntities) => {
36
- await Promise.all(
37
- updatedEntities->Array.map(({entityConfig, scope, changes}) => {
38
- ClickHouse.setUpdatesOrThrow(client, ~cache, ~changes, ~entityConfig, ~scope, ~database)
39
- }),
40
- )->Utils.Promise.ignoreValue
41
- await ClickHouse.setCheckpointsOrThrow(client, ~batch, ~database)
36
+ // Staging reads JS values, so it holds the isolate and runs here. The
37
+ // encode and the round trips happen in Rust, which also keeps the
38
+ // checkpoints behind the rows they cover.
39
+ let entities = []
40
+ let checkpoints = try {
41
+ updatedEntities->Array.forEach(({entityConfig, scope, changes}) =>
42
+ switch ClickHouse.stageUpdatesOrThrow(sink, ~registry, ~changes, ~entityConfig, ~scope) {
43
+ | Some(handle) => entities->Array.push(handle)
44
+ | None => ()
45
+ }
46
+ )
47
+ ClickHouse.stageCheckpointsOrThrow(sink, ~registry, ~batch)
48
+ } catch {
49
+ | exn =>
50
+ sink->ClickHouseSink.discard(entities)
51
+ throw(exn)
52
+ }
53
+ await ClickHouse.writeStagedOrThrow(sink, ~entities, ~checkpoints)
42
54
  },
43
55
  }
44
56
  }
package/src/Sink.res.mjs CHANGED
@@ -1,28 +1,33 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
3
  import * as ClickHouse from "./bindings/ClickHouse.res.mjs";
4
- import * as Client from "@clickhouse/client";
5
4
 
6
5
  function makeClickHouse(host, database, username, password, chainIdModeOpt) {
7
6
  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
8
- let client = Client.createClient({
9
- url: host,
10
- username: username,
11
- password: password
12
- });
13
- let cache = {};
7
+ let sink = ClickHouse.makeSink(host, username, password, database, chainIdMode);
8
+ let registry = ClickHouse.makeRegistry();
9
+ let mirrored = entities => entities.filter(e => e.storage.clickhouse);
14
10
  return {
15
11
  name: "clickhouse",
16
- initialize: ($staropt$star, $staropt$star$1, $staropt$star$2) => {
17
- $staropt$star !== undefined;
18
- let entities = $staropt$star$1 !== undefined ? $staropt$star$1 : [];
19
- let enums = $staropt$star$2 !== undefined ? $staropt$star$2 : [];
20
- return ClickHouse.initialize(client, database, entities, enums, chainIdMode);
21
- },
22
- resume: checkpointId => ClickHouse.resume(client, database, checkpointId),
12
+ initialize: entities => ClickHouse.initialize(sink, mirrored(entities)),
13
+ resume: (checkpointId, chains, entities) => ClickHouse.resume(sink, checkpointId, chains, mirrored(entities)),
23
14
  writeBatch: async (batch, updatedEntities) => {
24
- await Promise.all(updatedEntities.map(param => ClickHouse.setUpdatesOrThrow(client, cache, param.changes, param.entityConfig, param.scope, database)));
25
- return await ClickHouse.setCheckpointsOrThrow(client, batch, database);
15
+ let entities = [];
16
+ let checkpoints;
17
+ try {
18
+ updatedEntities.forEach(param => {
19
+ let handle = ClickHouse.stageUpdatesOrThrow(sink, registry, param.changes, param.entityConfig, param.scope);
20
+ if (handle !== undefined) {
21
+ entities.push(handle);
22
+ return;
23
+ }
24
+ });
25
+ checkpoints = ClickHouse.stageCheckpointsOrThrow(sink, registry, batch);
26
+ } catch (exn) {
27
+ sink.discard(entities);
28
+ throw exn;
29
+ }
30
+ return await ClickHouse.writeStagedOrThrow(sink, entities, checkpoints);
26
31
  }
27
32
  };
28
33
  }
@@ -589,7 +589,7 @@ let makeInMemoryStorage = (~state: testIndexerState): Persistence.storage => {
589
589
  JsError.throwWithMessage(
590
590
  "TestIndexer: initialize should not be called; the initial state is derived from config.",
591
591
  ),
592
- resumeInitialState: async () =>
592
+ resumeInitialState: async (~entities as _, ~throwIfIncompatible as _) =>
593
593
  JsError.throwWithMessage(
594
594
  "TestIndexer: resumeInitialState should not be called; the initial state is derived from config.",
595
595
  ),
@@ -625,22 +625,22 @@ let makeInMemoryStorage = (~state: testIndexerState): Persistence.storage => {
625
625
  dumpEffectCache: async () => (),
626
626
  reset: async () => (),
627
627
  setChainMeta: async _ => Obj.magic(),
628
- pruneStaleCheckpoints: async (~safeCheckpointId as _) => (),
628
+ pruneStaleCheckpoints: async (~safeCheckpoints as _) => (),
629
629
  pruneStaleEntityHistory: async (
630
630
  ~entityName as _,
631
631
  ~entityIndex as _,
632
632
  ~chainIdColumn as _,
633
- ~safeCheckpointId as _,
633
+ ~safeCheckpoints as _,
634
634
  ) => (),
635
635
  getRollbackTargetCheckpoint: async (~reorgChainId as _, ~lastKnownValidBlockNumber as _) =>
636
636
  JsError.throwWithMessage(
637
637
  "TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable.",
638
638
  ),
639
- getRollbackProgressDiff: async (~rollbackTargetCheckpointId as _) =>
639
+ getRollbackProgressDiff: async (~floors as _) =>
640
640
  JsError.throwWithMessage(
641
641
  "TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable.",
642
642
  ),
643
- getRollbackData: async (~entityConfig as _, ~rollbackTargetCheckpointId as _) =>
643
+ getRollbackData: async (~entityConfig as _, ~floors as _) =>
644
644
  JsError.throwWithMessage(
645
645
  "TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable.",
646
646
  ),
@@ -385,7 +385,7 @@ function makeInMemoryStorage(state) {
385
385
  name: "test-inmemory",
386
386
  isInitialized: async () => true,
387
387
  initialize: async (param, param$1, param$2, param$3, param$4) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
388
- resumeInitialState: async () => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
388
+ resumeInitialState: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
389
389
  loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
390
390
  ensureQueryIndexes: async (param, param$1) => {},
391
391
  ensureSchemaIndexes: async param => {},
package/src/Utils.res CHANGED
@@ -18,6 +18,33 @@ external importPathWithJson: (
18
18
  "default": JSON.t,
19
19
  }> = "import"
20
20
 
21
+ // Half the delay fixed, half spread across it. Every indexer pointed at one
22
+ // provider loses its stream, or gives up on a quiet chain, in the same instant
23
+ // that provider blinks, and putting them all back on the same schedule is how a
24
+ // blip becomes a stampede.
25
+ let jitter = delay => delay / 2 + (Math.random() *. (delay / 2)->Int.toFloat)->Float.toInt
26
+
27
+ // Clearing a pending timer and forgetting it are one action. A ref left holding
28
+ // a spent id reads as a timer still pending, and the next clear looks like it
29
+ // did something.
30
+ let clearTimeoutRef = timeoutId => {
31
+ switch timeoutId.contents {
32
+ | Some(id) => clearTimeout(id)
33
+ | None => ()
34
+ }
35
+ timeoutId := None
36
+ }
37
+
38
+ // Keeps the doubling from overflowing on something that has been failing for
39
+ // days. Every caller's delay reaches its cap long before this.
40
+ let maxBackoffExponent = 20
41
+
42
+ let expBackoff = (~base, ~exp, ~maxMillis) =>
43
+ Pervasives.min(
44
+ base * Math.pow(2.0, ~exp=Pervasives.min(exp, maxBackoffExponent)->Int.toFloat)->Float.toInt,
45
+ maxMillis,
46
+ )
47
+
21
48
  let delay = milliseconds =>
22
49
  Promise.make((resolve, _) => {
23
50
  let _interval = setTimeout(_ => {
@@ -97,6 +124,15 @@ module Dict = {
97
124
  @get_index
98
125
  external dangerouslyGetNonOption: (dict<'a>, string) => option<'a> = ""
99
126
 
127
+ let incrementBy = (dict, key, count) =>
128
+ dict->Dict.set(
129
+ key,
130
+ switch dict->dangerouslyGetNonOption(key) {
131
+ | Some(current) => current + count
132
+ | None => count
133
+ },
134
+ )
135
+
100
136
  let getOrInsertEmptyDict = (dict, key) => {
101
137
  switch dict->dangerouslyGetNonOption(key) {
102
138
  | Some(d) => d
@@ -253,6 +289,10 @@ module Array = {
253
289
 
254
290
  let immutableEmpty: array<unknown> = []
255
291
 
292
+ /** Render names for an error message: `"a", "b"`. */
293
+ let quotedJoin = (names: array<string>) =>
294
+ names->Array.map(name => `"${name}"`)->Array.joinUnsafe(", ")
295
+
256
296
  @send
257
297
  external forEachAsync: (array<'a>, 'a => promise<unit>) => unit = "forEach"
258
298
 
@@ -491,9 +531,120 @@ let unwrapResultExn = res =>
491
531
 
492
532
  external queueMicrotask: (unit => unit) => unit = "queueMicrotask"
493
533
 
534
+ module Bytes = {
535
+ let asUint8Array: unknown => option<Uint8Array.t> = %raw(`(value) =>
536
+ value instanceof Uint8Array
537
+ ? value.constructor === Uint8Array
538
+ ? value
539
+ : new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
540
+ : undefined`)
541
+
542
+ let toHex = (bytes: Uint8Array.t) =>
543
+ NodeJs.Buffer.fromArrayBuffer(
544
+ bytes->TypedArray.buffer,
545
+ ~byteOffset=bytes->TypedArray.byteOffset,
546
+ ~length=bytes->TypedArray.byteLength,
547
+ )->NodeJs.Buffer.toHex
548
+
549
+ @get_index external byteAt: (Uint8Array.t, int) => int = ""
550
+
551
+ // A bytea[] parameter as the array literal Postgres parses itself. postgres.js
552
+ // serializes an array parameter with the element serializer of the type the
553
+ // server describes, and the bytea one expects a Uint8Array — yet it types the
554
+ // array after its first element, so an array of Uint8Arrays binds as a single
555
+ // bytea. Text binds untyped and reaches the server's array parser as it is.
556
+ //
557
+ // Nests, since an `in` filter over a list column carries one array per
558
+ // candidate value. A sub-array is written unquoted, which is how Postgres
559
+ // spells a dimension rather than an element.
560
+ let rec toPgArrayLiteral = (values: array<unknown>) =>
561
+ "{" ++
562
+ values
563
+ ->Array.map(value =>
564
+ switch value->(magic: unknown => Nullable.t<unknown>)->Nullable.toOption {
565
+ | None => "NULL"
566
+ | Some(value) =>
567
+ switch value->asUint8Array {
568
+ | Some(bytes) => `"\\\\x${bytes->toHex}"`
569
+ | None =>
570
+ if value->Array.isArray {
571
+ value->(magic: unknown => array<unknown>)->toPgArrayLiteral
572
+ } else {
573
+ JsError.throwWithMessage("Expected Uint8Array")
574
+ }
575
+ }
576
+ }
577
+ )
578
+ ->Array.join(",") ++ "}"
579
+
580
+ // Bytewise, the order Postgres sorts a bytea in.
581
+ let compare = (a: Uint8Array.t, b: Uint8Array.t) => {
582
+ let aLength = a->TypedArray.length
583
+ let bLength = b->TypedArray.length
584
+ let rec loop = index =>
585
+ if index === aLength || index === bLength {
586
+ Int.compare(aLength, bLength)
587
+ } else {
588
+ switch Int.compare(a->byteAt(index), b->byteAt(index)) {
589
+ | 0. => loop(index + 1)
590
+ | order => order
591
+ }
592
+ }
593
+ loop(0)
594
+ }
595
+ }
596
+
494
597
  module Schema = {
495
598
  let variantTag = S.union([S.string, S.object(s => s.field("TAG", S.string))])
496
599
 
600
+ // A ReScript `option` is `undefined` at runtime, so `S.null`'s serializer
601
+ // treats only `undefined` as the empty case — a genuine `null`, which a JS
602
+ // handler writes for a field it doesn't set, reaches the value serializer
603
+ // and crashes it (`null.toString()` for a BigInt column). Collapse both to
604
+ // `None` before the value serializer sees them.
605
+ //
606
+ // Revisit on the Sury v11 migration: if its nullable handles a `null` value
607
+ // on the serialize side, this wrapper goes away.
608
+ let nullTolerant = schema =>
609
+ S.null(schema)->S.transform(_ => {
610
+ parser: value => value,
611
+ serializer: value => value->(magic: option<'a> => Nullable.t<'a>)->Nullable.toOption,
612
+ })
613
+
614
+ // Postgres hands a bytea back as a Buffer, while handlers are promised the
615
+ // plain Uint8Array of the entity type — Buffer's `slice` and `toString` behave
616
+ // differently.
617
+ let bytes = S.custom("Bytes", s => {
618
+ parser: unknown =>
619
+ switch unknown->Bytes.asUint8Array {
620
+ | Some(bytes) => bytes
621
+ | None => s.fail("Expected Uint8Array")
622
+ },
623
+ serializer: (bytes: Uint8Array.t) => bytes,
624
+ })
625
+
626
+ // A bytea[] value: a list column, or the values an `in` filter compares
627
+ // against — one array per candidate when the column is itself a list, which
628
+ // the literal writer nests. Only a column is ever read back, so the parser
629
+ // takes the one level a bytea[] column returns.
630
+ let bytesArray = S.custom("BytesArray", s => {
631
+ parser: unknown =>
632
+ if unknown->Array.isArray {
633
+ unknown
634
+ ->(magic: unknown => array<unknown>)
635
+ ->Array.map(item =>
636
+ switch item->Bytes.asUint8Array {
637
+ | Some(bytes) => bytes
638
+ | None => s.fail("Expected Uint8Array")
639
+ }
640
+ )
641
+ } else {
642
+ s.fail("Expected array of Uint8Array")
643
+ },
644
+ serializer: (values: array<Uint8Array.t>) =>
645
+ values->(magic: array<Uint8Array.t> => array<unknown>)->Bytes.toPgArrayLiteral,
646
+ })
647
+
497
648
  // Don't use S.unknown, since it's not serializable to json
498
649
  // In a nutshell, this is completely unsafe.
499
650
  let dbDate =
@@ -501,6 +652,19 @@ module Schema = {
501
652
  ->(magic: S.t<JSON.t> => S.t<Date.t>)
502
653
  ->S.preprocess(_ => {serializer: date => date->magic->Date.toISOString})
503
654
 
655
+ // JSON `null` is a document, and Postgres stores it as one. Reaching the
656
+ // ClickHouse sink as a JS `null` it would instead read as a field the handler
657
+ // never set, which a String column has no way to hold — so it travels as the
658
+ // text it would have been serialized to. Every other document is left for the
659
+ // sink to serialize.
660
+ let clickHouseJson = S.json(~validate=false)->S.preprocess(_ => {
661
+ serializer: value =>
662
+ switch value->(magic: unknown => Nullable.t<unknown>)->Nullable.toOption {
663
+ | None => "null"->magic
664
+ | Some(json) => json
665
+ },
666
+ })
667
+
504
668
  // ClickHouse expects timestamps as numbers (milliseconds), not ISO strings
505
669
  let clickHouseDate =
506
670
  S.json(~validate=false)
@@ -761,6 +925,12 @@ let prettifyExn = exn => {
761
925
  }
762
926
  }
763
927
 
928
+ let exnMessage = exn =>
929
+ switch exn->JsExn.anyToExnInternal {
930
+ | JsExn(jsExn) => jsExn->JsExn.message
931
+ | _ => None
932
+ }
933
+
764
934
  module EnvioPackage = {
765
935
  type t = {version: string}
766
936
 
package/src/Utils.res.mjs CHANGED
@@ -3,6 +3,8 @@ import packageJson from '../package.json' with { type: 'json' }
3
3
 
4
4
  import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
5
5
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
6
+ import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
7
+ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
6
8
  import * as Stdlib_BigInt from "@rescript/runtime/lib/es6/Stdlib_BigInt.js";
7
9
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
8
10
  import * as Stdlib_String from "@rescript/runtime/lib/es6/Stdlib_String.js";
@@ -14,6 +16,22 @@ import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js
14
16
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
15
17
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
16
18
 
19
+ function jitter(delay) {
20
+ return (delay / 2 | 0) + (Math.random() * (delay / 2 | 0) | 0) | 0;
21
+ }
22
+
23
+ function clearTimeoutRef(timeoutId) {
24
+ let id = timeoutId.contents;
25
+ if (id !== undefined) {
26
+ clearTimeout(Primitive_option.valFromOption(id));
27
+ }
28
+ timeoutId.contents = undefined;
29
+ }
30
+
31
+ function expBackoff(base, exp, maxMillis) {
32
+ return Primitive_int.min(base * (Math.pow(2.0, Primitive_int.min(exp, 20)) | 0) | 0, maxMillis);
33
+ }
34
+
17
35
  function delay(milliseconds) {
18
36
  return new Promise((resolve, param) => {
19
37
  setTimeout(() => resolve(), milliseconds);
@@ -61,6 +79,11 @@ let Option = {
61
79
  getExn: getExn
62
80
  };
63
81
 
82
+ function incrementBy(dict, key, count) {
83
+ let current = dict[key];
84
+ dict[key] = current !== undefined ? current + count | 0 : count;
85
+ }
86
+
64
87
  function getOrInsertEmptyDict(dict, key) {
65
88
  let d = dict[key];
66
89
  if (d !== undefined) {
@@ -169,6 +192,7 @@ let shallowCopy = ((dict) => ({...dict}));
169
192
 
170
193
  let Dict = {
171
194
  $$delete: Stdlib_Dict.$$delete,
195
+ incrementBy: incrementBy,
172
196
  getOrInsertEmptyDict: getOrInsertEmptyDict,
173
197
  push: push,
174
198
  pushMany: pushMany,
@@ -211,6 +235,10 @@ let UnsafeIntOperators = {};
211
235
 
212
236
  let immutableEmpty = [];
213
237
 
238
+ function quotedJoin(names) {
239
+ return names.map(name => `"` + name + `"`).join(", ");
240
+ }
241
+
214
242
  function mergeSorted(f, xs, ys) {
215
243
  if (xs.length === 0) {
216
244
  return ys;
@@ -393,6 +421,7 @@ let $$Array$1 = {
393
421
  partition: Stdlib_Array.partition,
394
422
  unzip: Stdlib_Array.unzip,
395
423
  immutableEmpty: immutableEmpty,
424
+ quotedJoin: quotedJoin,
396
425
  mergeSorted: mergeSorted,
397
426
  clearInPlace: clearInPlace$1,
398
427
  setIndexImmutable: setIndexImmutable,
@@ -462,15 +491,120 @@ function unwrapResultExn(res) {
462
491
  throw res._0;
463
492
  }
464
493
 
494
+ let asUint8Array = ((value) =>
495
+ value instanceof Uint8Array
496
+ ? value.constructor === Uint8Array
497
+ ? value
498
+ : new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
499
+ : undefined);
500
+
501
+ function toHex(bytes) {
502
+ return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("hex");
503
+ }
504
+
505
+ function toPgArrayLiteral(values) {
506
+ return "{" + values.map(value => {
507
+ if (value == null) {
508
+ return "NULL";
509
+ }
510
+ let bytes = asUint8Array(value);
511
+ if (bytes !== undefined) {
512
+ return `"\\\\x` + toHex(bytes) + `"`;
513
+ } else if (Array.isArray(value)) {
514
+ return toPgArrayLiteral(value);
515
+ } else {
516
+ return Stdlib_JsError.throwWithMessage("Expected Uint8Array");
517
+ }
518
+ }).join(",") + "}";
519
+ }
520
+
521
+ function compare(a, b) {
522
+ let aLength = a.length;
523
+ let bLength = b.length;
524
+ let _index = 0;
525
+ while (true) {
526
+ let index = _index;
527
+ if (index === aLength || index === bLength) {
528
+ return Primitive_int.compare(aLength, bLength);
529
+ }
530
+ let order = Primitive_int.compare(a[index], b[index]);
531
+ if (order !== 0) {
532
+ return order;
533
+ }
534
+ _index = index + 1 | 0;
535
+ continue;
536
+ };
537
+ }
538
+
539
+ let Bytes = {
540
+ asUint8Array: asUint8Array,
541
+ toHex: toHex,
542
+ toPgArrayLiteral: toPgArrayLiteral,
543
+ compare: compare
544
+ };
545
+
465
546
  let variantTag = S$RescriptSchema.union([
466
547
  S$RescriptSchema.string,
467
548
  S$RescriptSchema.object(s => s.f("TAG", S$RescriptSchema.string))
468
549
  ]);
469
550
 
551
+ function nullTolerant(schema) {
552
+ return S$RescriptSchema.transform(S$RescriptSchema.$$null(schema), param => ({
553
+ p: value => value,
554
+ s: value => {
555
+ if (value == null) {
556
+ return;
557
+ } else {
558
+ return Primitive_option.some(value);
559
+ }
560
+ }
561
+ }));
562
+ }
563
+
564
+ let bytes = S$RescriptSchema.custom("Bytes", s => ({
565
+ p: unknown => {
566
+ let bytes = asUint8Array(unknown);
567
+ if (bytes !== undefined) {
568
+ return bytes;
569
+ } else {
570
+ return s.fail("Expected Uint8Array", undefined);
571
+ }
572
+ },
573
+ s: bytes => bytes
574
+ }));
575
+
576
+ let bytesArray = S$RescriptSchema.custom("BytesArray", s => ({
577
+ p: unknown => {
578
+ if (Array.isArray(unknown)) {
579
+ return unknown.map(item => {
580
+ let bytes = asUint8Array(item);
581
+ if (bytes !== undefined) {
582
+ return bytes;
583
+ } else {
584
+ return s.fail("Expected Uint8Array", undefined);
585
+ }
586
+ });
587
+ } else {
588
+ return s.fail("Expected array of Uint8Array", undefined);
589
+ }
590
+ },
591
+ s: toPgArrayLiteral
592
+ }));
593
+
470
594
  let dbDate = S$RescriptSchema.preprocess(S$RescriptSchema.json(false), param => ({
471
595
  s: date => date.toISOString()
472
596
  }));
473
597
 
598
+ let clickHouseJson = S$RescriptSchema.preprocess(S$RescriptSchema.json(false), param => ({
599
+ s: value => {
600
+ if (value == null) {
601
+ return "null";
602
+ } else {
603
+ return value;
604
+ }
605
+ }
606
+ }));
607
+
474
608
  let clickHouseDate = S$RescriptSchema.preprocess(S$RescriptSchema.json(false), param => ({
475
609
  s: date => date.getTime()
476
610
  }));
@@ -506,7 +640,11 @@ function coerceToJsonPgType(schema) {
506
640
 
507
641
  let Schema = {
508
642
  variantTag: variantTag,
643
+ nullTolerant: nullTolerant,
644
+ bytes: bytes,
645
+ bytesArray: bytesArray,
509
646
  dbDate: dbDate,
647
+ clickHouseJson: clickHouseJson,
510
648
  clickHouseDate: clickHouseDate,
511
649
  coerceToJsonPgType: coerceToJsonPgType
512
650
  };
@@ -640,6 +778,13 @@ function prettifyExn(exn) {
640
778
  }
641
779
  }
642
780
 
781
+ function exnMessage(exn) {
782
+ let jsExn = Primitive_exceptions.internalToException(exn);
783
+ if (jsExn.RE_EXN_ID === "JsExn") {
784
+ return Stdlib_JsExn.message(jsExn._1);
785
+ }
786
+ }
787
+
643
788
  let value;
644
789
 
645
790
  try {
@@ -695,7 +840,13 @@ let $$Promise$1 = {
695
840
  isCatchable: isCatchable
696
841
  };
697
842
 
843
+ let maxBackoffExponent = 20;
844
+
698
845
  export {
846
+ jitter,
847
+ clearTimeoutRef,
848
+ maxBackoffExponent,
849
+ expBackoff,
699
850
  delay,
700
851
  $$Object,
701
852
  $$Error,
@@ -708,6 +859,7 @@ export {
708
859
  Url,
709
860
  Result,
710
861
  unwrapResultExn,
862
+ Bytes,
711
863
  Schema,
712
864
  getVariantsTags,
713
865
  $$Set,
@@ -717,6 +869,7 @@ export {
717
869
  $$Proxy,
718
870
  Hash,
719
871
  prettifyExn,
872
+ exnMessage,
720
873
  EnvioPackage,
721
874
  $$BigInt,
722
875
  $$Promise$1 as $$Promise,
package/src/Writing.res CHANGED
@@ -158,8 +158,8 @@ let runOneWrite = async (state: IndexerState.t) => {
158
158
  state->IndexerState.markCommitted(~upToCheckpointId)
159
159
 
160
160
  switch rollback {
161
- | Some({progressBlockNumberByChainId}) if RollbackCommit.callbacks->Utils.Array.notEmpty =>
162
- await RollbackCommit.fire(~progressBlockNumberByChainId)
161
+ | Some({progressedChains}) if RollbackCommit.callbacks->Utils.Array.notEmpty =>
162
+ await RollbackCommit.fire(~progressedChains)
163
163
  | _ => ()
164
164
  }
165
165
 
@@ -130,7 +130,7 @@ async function runOneWrite(state) {
130
130
  ]);
131
131
  IndexerState.markCommitted(state, upToCheckpointId);
132
132
  if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
133
- await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
133
+ await RollbackCommit.fire(rollback.progressedChains);
134
134
  }
135
135
  return await PruneStaleHistory.runForced(state, pruneTargets);
136
136
  }