envio 3.9.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +26 -0
- package/fuel.schema.json +26 -0
- package/index.d.ts +35 -3
- package/package.json +6 -7
- package/src/Batch.res +9 -0
- package/src/ChainState.res +35 -39
- package/src/ChainState.res.mjs +14 -11
- package/src/ChainState.resi +15 -7
- package/src/Config.res +52 -44
- package/src/Config.res.mjs +51 -17
- package/src/Core.res +4 -0
- package/src/CrossChainState.res +15 -23
- package/src/CrossChainState.res.mjs +10 -18
- package/src/CrossChainState.resi +4 -1
- package/src/Ecosystem.res +7 -2
- package/src/Envio.res +6 -4
- package/src/EventConfigBuilder.res +169 -26
- package/src/EventConfigBuilder.res.mjs +124 -15
- package/src/HandlerRegister.res +21 -4
- package/src/HandlerRegister.res.mjs +24 -5
- package/src/InMemoryStore.res +9 -7
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/IndexerState.res +43 -6
- package/src/IndexerState.res.mjs +36 -7
- package/src/IndexerState.resi +4 -4
- package/src/Internal.res +16 -17
- package/src/Metrics.res +72 -6
- package/src/Metrics.res.mjs +30 -2
- package/src/Persistence.res +36 -30
- package/src/Persistence.res.mjs +1 -20
- package/src/PgStorage.res +104 -56
- package/src/PgStorage.res.mjs +57 -35
- package/src/PruneStaleHistory.res +85 -52
- package/src/PruneStaleHistory.res.mjs +63 -37
- package/src/Rollback.res +68 -45
- package/src/Rollback.res.mjs +38 -19
- package/src/RollbackCommit.res +7 -9
- package/src/RollbackCommit.res.mjs +5 -6
- package/src/SafeCheckpointTracking.res +11 -11
- package/src/SafeCheckpointTracking.res.mjs +7 -6
- package/src/SimulateItems.res +33 -18
- package/src/SimulateItems.res.mjs +26 -22
- package/src/Sink.res +38 -26
- package/src/Sink.res.mjs +21 -16
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +1 -1
- package/src/Utils.res +170 -0
- package/src/Utils.res.mjs +153 -0
- package/src/Writing.res +2 -2
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +366 -698
- package/src/bindings/ClickHouse.res.mjs +371 -421
- package/src/bindings/ClickHouseSink.res +337 -0
- package/src/bindings/ClickHouseSink.res.mjs +246 -0
- package/src/bindings/EventSource.res +8 -2
- package/src/bindings/NodeJs.res +6 -0
- package/src/bindings/Vitest.res +23 -0
- package/src/bindings/Vitest.res.mjs +3 -0
- package/src/bindings/WebSocket.res +9 -10
- package/src/db/CheckpointBounds.res +53 -0
- package/src/db/CheckpointBounds.res.mjs +44 -0
- package/src/db/EntityFilter.res +36 -17
- package/src/db/EntityFilter.res.mjs +25 -14
- package/src/db/EntityHistory.res +47 -22
- package/src/db/EntityHistory.res.mjs +19 -12
- package/src/db/InternalTable.res +84 -35
- package/src/db/InternalTable.res.mjs +56 -23
- package/src/db/RollbackFloors.res +55 -0
- package/src/db/RollbackFloors.res.mjs +92 -0
- package/src/db/Table.res +22 -4
- package/src/db/Table.res.mjs +30 -9
- package/src/sources/Evm.res +2 -0
- package/src/sources/Evm.res.mjs +2 -0
- package/src/sources/EvmHyperSyncSource.res +3 -3
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcWs.res +124 -0
- package/src/sources/EvmRpcWs.res.mjs +117 -0
- package/src/sources/Fuel.res +2 -0
- package/src/sources/Fuel.res.mjs +2 -0
- package/src/sources/FuelHyperSyncSource.res +1 -1
- package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
- package/src/sources/HeightFeed.res +568 -0
- package/src/sources/HeightFeed.res.mjs +452 -0
- package/src/sources/HeightStream.res +257 -0
- package/src/sources/HeightStream.res.mjs +194 -0
- package/src/sources/HyperSync.res +5 -0
- package/src/sources/HyperSync.res.mjs +3 -0
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncSSE.res +49 -0
- package/src/sources/HyperSyncSSE.res.mjs +79 -0
- package/src/sources/RpcSource.res +1 -1
- package/src/sources/RpcSource.res.mjs +2 -2
- package/src/sources/Source.res +54 -1
- package/src/sources/Source.res.mjs +25 -0
- package/src/sources/SourceManager.res +220 -227
- package/src/sources/SourceManager.res.mjs +139 -164
- package/src/sources/SourceManager.resi +8 -0
- package/src/sources/Svm.res +2 -0
- package/src/sources/Svm.res.mjs +2 -0
- package/src/sources/SvmHyperSyncClient.res +70 -127
- package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
- package/src/sources/SvmHyperSyncSource.res +18 -15
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/svm.schema.json +1 -75
- package/src/MemoryStorage.res +0 -729
- package/src/MemoryStorage.res.mjs +0 -586
- package/src/sources/HyperSyncHeightStream.res +0 -129
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
- package/src/sources/RpcWebSocketHeightStream.res +0 -175
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
package/src/Sink.res
CHANGED
|
@@ -1,44 +1,56 @@
|
|
|
1
1
|
type t = {
|
|
2
2
|
name: string,
|
|
3
|
-
initialize: (
|
|
4
|
-
|
|
5
|
-
~
|
|
6
|
-
~
|
|
3
|
+
initialize: (~entities: array<Internal.entityConfig>) => promise<unit>,
|
|
4
|
+
resume: (
|
|
5
|
+
~checkpointId: Internal.checkpointId,
|
|
6
|
+
~chains: array<Persistence.initialChainState>,
|
|
7
|
+
~entities: array<Internal.entityConfig>,
|
|
7
8
|
) => promise<unit>,
|
|
8
|
-
resume: (~checkpointId: Internal.checkpointId) => promise<unit>,
|
|
9
9
|
writeBatch: (
|
|
10
10
|
~batch: Batch.t,
|
|
11
11
|
~updatedEntities: array<Persistence.updatedEntity>,
|
|
12
12
|
) => promise<unit>,
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
let makeClickHouse = (
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
15
|
+
let makeClickHouse = (
|
|
16
|
+
~host,
|
|
17
|
+
~database,
|
|
18
|
+
~username,
|
|
19
|
+
~password,
|
|
20
|
+
~chainIdMode: ChainId.mode=Int32,
|
|
21
|
+
): t => {
|
|
22
|
+
let sink = ClickHouse.makeSink(~host, ~username, ~password, ~database, ~chainIdMode)
|
|
23
|
+
let registry = ClickHouse.makeRegistry()
|
|
24
|
+
let mirrored = (entities: array<Internal.entityConfig>) =>
|
|
25
|
+
entities->Array.filter(e => e.storage.clickhouse)
|
|
26
26
|
|
|
27
27
|
{
|
|
28
28
|
name: "clickhouse",
|
|
29
|
-
initialize: (~
|
|
30
|
-
ClickHouse.initialize(
|
|
29
|
+
initialize: (~entities) => {
|
|
30
|
+
ClickHouse.initialize(sink, ~entities=mirrored(entities))
|
|
31
31
|
},
|
|
32
|
-
resume: (~checkpointId) => {
|
|
33
|
-
ClickHouse.resume(
|
|
32
|
+
resume: (~checkpointId, ~chains, ~entities) => {
|
|
33
|
+
ClickHouse.resume(sink, ~checkpointId, ~chains, ~entities=mirrored(entities))
|
|
34
34
|
},
|
|
35
35
|
writeBatch: async (~batch, ~updatedEntities) => {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
36
|
+
// Staging reads JS values, so it holds the isolate and runs here. The
|
|
37
|
+
// encode and the round trips happen in Rust, which also keeps the
|
|
38
|
+
// checkpoints behind the rows they cover.
|
|
39
|
+
let entities = []
|
|
40
|
+
let checkpoints = try {
|
|
41
|
+
updatedEntities->Array.forEach(({entityConfig, scope, changes}) =>
|
|
42
|
+
switch ClickHouse.stageUpdatesOrThrow(sink, ~registry, ~changes, ~entityConfig, ~scope) {
|
|
43
|
+
| Some(handle) => entities->Array.push(handle)
|
|
44
|
+
| None => ()
|
|
45
|
+
}
|
|
46
|
+
)
|
|
47
|
+
ClickHouse.stageCheckpointsOrThrow(sink, ~registry, ~batch)
|
|
48
|
+
} catch {
|
|
49
|
+
| exn =>
|
|
50
|
+
sink->ClickHouseSink.discard(entities)
|
|
51
|
+
throw(exn)
|
|
52
|
+
}
|
|
53
|
+
await ClickHouse.writeStagedOrThrow(sink, ~entities, ~checkpoints)
|
|
42
54
|
},
|
|
43
55
|
}
|
|
44
56
|
}
|
package/src/Sink.res.mjs
CHANGED
|
@@ -1,28 +1,33 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as ClickHouse from "./bindings/ClickHouse.res.mjs";
|
|
4
|
-
import * as Client from "@clickhouse/client";
|
|
5
4
|
|
|
6
5
|
function makeClickHouse(host, database, username, password, chainIdModeOpt) {
|
|
7
6
|
let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
|
|
8
|
-
let
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
password: password
|
|
12
|
-
});
|
|
13
|
-
let cache = {};
|
|
7
|
+
let sink = ClickHouse.makeSink(host, username, password, database, chainIdMode);
|
|
8
|
+
let registry = ClickHouse.makeRegistry();
|
|
9
|
+
let mirrored = entities => entities.filter(e => e.storage.clickhouse);
|
|
14
10
|
return {
|
|
15
11
|
name: "clickhouse",
|
|
16
|
-
initialize: (
|
|
17
|
-
|
|
18
|
-
let entities = $staropt$star$1 !== undefined ? $staropt$star$1 : [];
|
|
19
|
-
let enums = $staropt$star$2 !== undefined ? $staropt$star$2 : [];
|
|
20
|
-
return ClickHouse.initialize(client, database, entities, enums, chainIdMode);
|
|
21
|
-
},
|
|
22
|
-
resume: checkpointId => ClickHouse.resume(client, database, checkpointId),
|
|
12
|
+
initialize: entities => ClickHouse.initialize(sink, mirrored(entities)),
|
|
13
|
+
resume: (checkpointId, chains, entities) => ClickHouse.resume(sink, checkpointId, chains, mirrored(entities)),
|
|
23
14
|
writeBatch: async (batch, updatedEntities) => {
|
|
24
|
-
|
|
25
|
-
|
|
15
|
+
let entities = [];
|
|
16
|
+
let checkpoints;
|
|
17
|
+
try {
|
|
18
|
+
updatedEntities.forEach(param => {
|
|
19
|
+
let handle = ClickHouse.stageUpdatesOrThrow(sink, registry, param.changes, param.entityConfig, param.scope);
|
|
20
|
+
if (handle !== undefined) {
|
|
21
|
+
entities.push(handle);
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
checkpoints = ClickHouse.stageCheckpointsOrThrow(sink, registry, batch);
|
|
26
|
+
} catch (exn) {
|
|
27
|
+
sink.discard(entities);
|
|
28
|
+
throw exn;
|
|
29
|
+
}
|
|
30
|
+
return await ClickHouse.writeStagedOrThrow(sink, entities, checkpoints);
|
|
26
31
|
}
|
|
27
32
|
};
|
|
28
33
|
}
|
package/src/TestIndexer.res
CHANGED
|
@@ -589,7 +589,7 @@ let makeInMemoryStorage = (~state: testIndexerState): Persistence.storage => {
|
|
|
589
589
|
JsError.throwWithMessage(
|
|
590
590
|
"TestIndexer: initialize should not be called; the initial state is derived from config.",
|
|
591
591
|
),
|
|
592
|
-
resumeInitialState: async () =>
|
|
592
|
+
resumeInitialState: async (~entities as _, ~throwIfIncompatible as _) =>
|
|
593
593
|
JsError.throwWithMessage(
|
|
594
594
|
"TestIndexer: resumeInitialState should not be called; the initial state is derived from config.",
|
|
595
595
|
),
|
|
@@ -625,22 +625,22 @@ let makeInMemoryStorage = (~state: testIndexerState): Persistence.storage => {
|
|
|
625
625
|
dumpEffectCache: async () => (),
|
|
626
626
|
reset: async () => (),
|
|
627
627
|
setChainMeta: async _ => Obj.magic(),
|
|
628
|
-
pruneStaleCheckpoints: async (~
|
|
628
|
+
pruneStaleCheckpoints: async (~safeCheckpoints as _) => (),
|
|
629
629
|
pruneStaleEntityHistory: async (
|
|
630
630
|
~entityName as _,
|
|
631
631
|
~entityIndex as _,
|
|
632
632
|
~chainIdColumn as _,
|
|
633
|
-
~
|
|
633
|
+
~safeCheckpoints as _,
|
|
634
634
|
) => (),
|
|
635
635
|
getRollbackTargetCheckpoint: async (~reorgChainId as _, ~lastKnownValidBlockNumber as _) =>
|
|
636
636
|
JsError.throwWithMessage(
|
|
637
637
|
"TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable.",
|
|
638
638
|
),
|
|
639
|
-
getRollbackProgressDiff: async (~
|
|
639
|
+
getRollbackProgressDiff: async (~floors as _) =>
|
|
640
640
|
JsError.throwWithMessage(
|
|
641
641
|
"TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable.",
|
|
642
642
|
),
|
|
643
|
-
getRollbackData: async (~entityConfig as _, ~
|
|
643
|
+
getRollbackData: async (~entityConfig as _, ~floors as _) =>
|
|
644
644
|
JsError.throwWithMessage(
|
|
645
645
|
"TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable.",
|
|
646
646
|
),
|
package/src/TestIndexer.res.mjs
CHANGED
|
@@ -385,7 +385,7 @@ function makeInMemoryStorage(state) {
|
|
|
385
385
|
name: "test-inmemory",
|
|
386
386
|
isInitialized: async () => true,
|
|
387
387
|
initialize: async (param, param$1, param$2, param$3, param$4) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
|
|
388
|
-
resumeInitialState: async () => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
|
|
388
|
+
resumeInitialState: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
|
|
389
389
|
loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
|
|
390
390
|
ensureQueryIndexes: async (param, param$1) => {},
|
|
391
391
|
ensureSchemaIndexes: async param => {},
|
package/src/Utils.res
CHANGED
|
@@ -18,6 +18,33 @@ external importPathWithJson: (
|
|
|
18
18
|
"default": JSON.t,
|
|
19
19
|
}> = "import"
|
|
20
20
|
|
|
21
|
+
// Half the delay fixed, half spread across it. Every indexer pointed at one
|
|
22
|
+
// provider loses its stream, or gives up on a quiet chain, in the same instant
|
|
23
|
+
// that provider blinks, and putting them all back on the same schedule is how a
|
|
24
|
+
// blip becomes a stampede.
|
|
25
|
+
let jitter = delay => delay / 2 + (Math.random() *. (delay / 2)->Int.toFloat)->Float.toInt
|
|
26
|
+
|
|
27
|
+
// Clearing a pending timer and forgetting it are one action. A ref left holding
|
|
28
|
+
// a spent id reads as a timer still pending, and the next clear looks like it
|
|
29
|
+
// did something.
|
|
30
|
+
let clearTimeoutRef = timeoutId => {
|
|
31
|
+
switch timeoutId.contents {
|
|
32
|
+
| Some(id) => clearTimeout(id)
|
|
33
|
+
| None => ()
|
|
34
|
+
}
|
|
35
|
+
timeoutId := None
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Keeps the doubling from overflowing on something that has been failing for
|
|
39
|
+
// days. Every caller's delay reaches its cap long before this.
|
|
40
|
+
let maxBackoffExponent = 20
|
|
41
|
+
|
|
42
|
+
let expBackoff = (~base, ~exp, ~maxMillis) =>
|
|
43
|
+
Pervasives.min(
|
|
44
|
+
base * Math.pow(2.0, ~exp=Pervasives.min(exp, maxBackoffExponent)->Int.toFloat)->Float.toInt,
|
|
45
|
+
maxMillis,
|
|
46
|
+
)
|
|
47
|
+
|
|
21
48
|
let delay = milliseconds =>
|
|
22
49
|
Promise.make((resolve, _) => {
|
|
23
50
|
let _interval = setTimeout(_ => {
|
|
@@ -97,6 +124,15 @@ module Dict = {
|
|
|
97
124
|
@get_index
|
|
98
125
|
external dangerouslyGetNonOption: (dict<'a>, string) => option<'a> = ""
|
|
99
126
|
|
|
127
|
+
let incrementBy = (dict, key, count) =>
|
|
128
|
+
dict->Dict.set(
|
|
129
|
+
key,
|
|
130
|
+
switch dict->dangerouslyGetNonOption(key) {
|
|
131
|
+
| Some(current) => current + count
|
|
132
|
+
| None => count
|
|
133
|
+
},
|
|
134
|
+
)
|
|
135
|
+
|
|
100
136
|
let getOrInsertEmptyDict = (dict, key) => {
|
|
101
137
|
switch dict->dangerouslyGetNonOption(key) {
|
|
102
138
|
| Some(d) => d
|
|
@@ -253,6 +289,10 @@ module Array = {
|
|
|
253
289
|
|
|
254
290
|
let immutableEmpty: array<unknown> = []
|
|
255
291
|
|
|
292
|
+
/** Render names for an error message: `"a", "b"`. */
|
|
293
|
+
let quotedJoin = (names: array<string>) =>
|
|
294
|
+
names->Array.map(name => `"${name}"`)->Array.joinUnsafe(", ")
|
|
295
|
+
|
|
256
296
|
@send
|
|
257
297
|
external forEachAsync: (array<'a>, 'a => promise<unit>) => unit = "forEach"
|
|
258
298
|
|
|
@@ -491,9 +531,120 @@ let unwrapResultExn = res =>
|
|
|
491
531
|
|
|
492
532
|
external queueMicrotask: (unit => unit) => unit = "queueMicrotask"
|
|
493
533
|
|
|
534
|
+
module Bytes = {
|
|
535
|
+
let asUint8Array: unknown => option<Uint8Array.t> = %raw(`(value) =>
|
|
536
|
+
value instanceof Uint8Array
|
|
537
|
+
? value.constructor === Uint8Array
|
|
538
|
+
? value
|
|
539
|
+
: new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
|
|
540
|
+
: undefined`)
|
|
541
|
+
|
|
542
|
+
let toHex = (bytes: Uint8Array.t) =>
|
|
543
|
+
NodeJs.Buffer.fromArrayBuffer(
|
|
544
|
+
bytes->TypedArray.buffer,
|
|
545
|
+
~byteOffset=bytes->TypedArray.byteOffset,
|
|
546
|
+
~length=bytes->TypedArray.byteLength,
|
|
547
|
+
)->NodeJs.Buffer.toHex
|
|
548
|
+
|
|
549
|
+
@get_index external byteAt: (Uint8Array.t, int) => int = ""
|
|
550
|
+
|
|
551
|
+
// A bytea[] parameter as the array literal Postgres parses itself. postgres.js
|
|
552
|
+
// serializes an array parameter with the element serializer of the type the
|
|
553
|
+
// server describes, and the bytea one expects a Uint8Array — yet it types the
|
|
554
|
+
// array after its first element, so an array of Uint8Arrays binds as a single
|
|
555
|
+
// bytea. Text binds untyped and reaches the server's array parser as it is.
|
|
556
|
+
//
|
|
557
|
+
// Nests, since an `in` filter over a list column carries one array per
|
|
558
|
+
// candidate value. A sub-array is written unquoted, which is how Postgres
|
|
559
|
+
// spells a dimension rather than an element.
|
|
560
|
+
let rec toPgArrayLiteral = (values: array<unknown>) =>
|
|
561
|
+
"{" ++
|
|
562
|
+
values
|
|
563
|
+
->Array.map(value =>
|
|
564
|
+
switch value->(magic: unknown => Nullable.t<unknown>)->Nullable.toOption {
|
|
565
|
+
| None => "NULL"
|
|
566
|
+
| Some(value) =>
|
|
567
|
+
switch value->asUint8Array {
|
|
568
|
+
| Some(bytes) => `"\\\\x${bytes->toHex}"`
|
|
569
|
+
| None =>
|
|
570
|
+
if value->Array.isArray {
|
|
571
|
+
value->(magic: unknown => array<unknown>)->toPgArrayLiteral
|
|
572
|
+
} else {
|
|
573
|
+
JsError.throwWithMessage("Expected Uint8Array")
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
)
|
|
578
|
+
->Array.join(",") ++ "}"
|
|
579
|
+
|
|
580
|
+
// Bytewise, the order Postgres sorts a bytea in.
|
|
581
|
+
let compare = (a: Uint8Array.t, b: Uint8Array.t) => {
|
|
582
|
+
let aLength = a->TypedArray.length
|
|
583
|
+
let bLength = b->TypedArray.length
|
|
584
|
+
let rec loop = index =>
|
|
585
|
+
if index === aLength || index === bLength {
|
|
586
|
+
Int.compare(aLength, bLength)
|
|
587
|
+
} else {
|
|
588
|
+
switch Int.compare(a->byteAt(index), b->byteAt(index)) {
|
|
589
|
+
| 0. => loop(index + 1)
|
|
590
|
+
| order => order
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
loop(0)
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
494
597
|
module Schema = {
|
|
495
598
|
let variantTag = S.union([S.string, S.object(s => s.field("TAG", S.string))])
|
|
496
599
|
|
|
600
|
+
// A ReScript `option` is `undefined` at runtime, so `S.null`'s serializer
|
|
601
|
+
// treats only `undefined` as the empty case — a genuine `null`, which a JS
|
|
602
|
+
// handler writes for a field it doesn't set, reaches the value serializer
|
|
603
|
+
// and crashes it (`null.toString()` for a BigInt column). Collapse both to
|
|
604
|
+
// `None` before the value serializer sees them.
|
|
605
|
+
//
|
|
606
|
+
// Revisit on the Sury v11 migration: if its nullable handles a `null` value
|
|
607
|
+
// on the serialize side, this wrapper goes away.
|
|
608
|
+
let nullTolerant = schema =>
|
|
609
|
+
S.null(schema)->S.transform(_ => {
|
|
610
|
+
parser: value => value,
|
|
611
|
+
serializer: value => value->(magic: option<'a> => Nullable.t<'a>)->Nullable.toOption,
|
|
612
|
+
})
|
|
613
|
+
|
|
614
|
+
// Postgres hands a bytea back as a Buffer, while handlers are promised the
|
|
615
|
+
// plain Uint8Array of the entity type — Buffer's `slice` and `toString` behave
|
|
616
|
+
// differently.
|
|
617
|
+
let bytes = S.custom("Bytes", s => {
|
|
618
|
+
parser: unknown =>
|
|
619
|
+
switch unknown->Bytes.asUint8Array {
|
|
620
|
+
| Some(bytes) => bytes
|
|
621
|
+
| None => s.fail("Expected Uint8Array")
|
|
622
|
+
},
|
|
623
|
+
serializer: (bytes: Uint8Array.t) => bytes,
|
|
624
|
+
})
|
|
625
|
+
|
|
626
|
+
// A bytea[] value: a list column, or the values an `in` filter compares
|
|
627
|
+
// against — one array per candidate when the column is itself a list, which
|
|
628
|
+
// the literal writer nests. Only a column is ever read back, so the parser
|
|
629
|
+
// takes the one level a bytea[] column returns.
|
|
630
|
+
let bytesArray = S.custom("BytesArray", s => {
|
|
631
|
+
parser: unknown =>
|
|
632
|
+
if unknown->Array.isArray {
|
|
633
|
+
unknown
|
|
634
|
+
->(magic: unknown => array<unknown>)
|
|
635
|
+
->Array.map(item =>
|
|
636
|
+
switch item->Bytes.asUint8Array {
|
|
637
|
+
| Some(bytes) => bytes
|
|
638
|
+
| None => s.fail("Expected Uint8Array")
|
|
639
|
+
}
|
|
640
|
+
)
|
|
641
|
+
} else {
|
|
642
|
+
s.fail("Expected array of Uint8Array")
|
|
643
|
+
},
|
|
644
|
+
serializer: (values: array<Uint8Array.t>) =>
|
|
645
|
+
values->(magic: array<Uint8Array.t> => array<unknown>)->Bytes.toPgArrayLiteral,
|
|
646
|
+
})
|
|
647
|
+
|
|
497
648
|
// Don't use S.unknown, since it's not serializable to json
|
|
498
649
|
// In a nutshell, this is completely unsafe.
|
|
499
650
|
let dbDate =
|
|
@@ -501,6 +652,19 @@ module Schema = {
|
|
|
501
652
|
->(magic: S.t<JSON.t> => S.t<Date.t>)
|
|
502
653
|
->S.preprocess(_ => {serializer: date => date->magic->Date.toISOString})
|
|
503
654
|
|
|
655
|
+
// JSON `null` is a document, and Postgres stores it as one. Reaching the
|
|
656
|
+
// ClickHouse sink as a JS `null` it would instead read as a field the handler
|
|
657
|
+
// never set, which a String column has no way to hold — so it travels as the
|
|
658
|
+
// text it would have been serialized to. Every other document is left for the
|
|
659
|
+
// sink to serialize.
|
|
660
|
+
let clickHouseJson = S.json(~validate=false)->S.preprocess(_ => {
|
|
661
|
+
serializer: value =>
|
|
662
|
+
switch value->(magic: unknown => Nullable.t<unknown>)->Nullable.toOption {
|
|
663
|
+
| None => "null"->magic
|
|
664
|
+
| Some(json) => json
|
|
665
|
+
},
|
|
666
|
+
})
|
|
667
|
+
|
|
504
668
|
// ClickHouse expects timestamps as numbers (milliseconds), not ISO strings
|
|
505
669
|
let clickHouseDate =
|
|
506
670
|
S.json(~validate=false)
|
|
@@ -761,6 +925,12 @@ let prettifyExn = exn => {
|
|
|
761
925
|
}
|
|
762
926
|
}
|
|
763
927
|
|
|
928
|
+
let exnMessage = exn =>
|
|
929
|
+
switch exn->JsExn.anyToExnInternal {
|
|
930
|
+
| JsExn(jsExn) => jsExn->JsExn.message
|
|
931
|
+
| _ => None
|
|
932
|
+
}
|
|
933
|
+
|
|
764
934
|
module EnvioPackage = {
|
|
765
935
|
type t = {version: string}
|
|
766
936
|
|
package/src/Utils.res.mjs
CHANGED
|
@@ -3,6 +3,8 @@ import packageJson from '../package.json' with { type: 'json' }
|
|
|
3
3
|
|
|
4
4
|
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
5
5
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
6
|
+
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
7
|
+
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
6
8
|
import * as Stdlib_BigInt from "@rescript/runtime/lib/es6/Stdlib_BigInt.js";
|
|
7
9
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
8
10
|
import * as Stdlib_String from "@rescript/runtime/lib/es6/Stdlib_String.js";
|
|
@@ -14,6 +16,22 @@ import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js
|
|
|
14
16
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
15
17
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
16
18
|
|
|
19
|
+
function jitter(delay) {
|
|
20
|
+
return (delay / 2 | 0) + (Math.random() * (delay / 2 | 0) | 0) | 0;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function clearTimeoutRef(timeoutId) {
|
|
24
|
+
let id = timeoutId.contents;
|
|
25
|
+
if (id !== undefined) {
|
|
26
|
+
clearTimeout(Primitive_option.valFromOption(id));
|
|
27
|
+
}
|
|
28
|
+
timeoutId.contents = undefined;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function expBackoff(base, exp, maxMillis) {
|
|
32
|
+
return Primitive_int.min(base * (Math.pow(2.0, Primitive_int.min(exp, 20)) | 0) | 0, maxMillis);
|
|
33
|
+
}
|
|
34
|
+
|
|
17
35
|
function delay(milliseconds) {
|
|
18
36
|
return new Promise((resolve, param) => {
|
|
19
37
|
setTimeout(() => resolve(), milliseconds);
|
|
@@ -61,6 +79,11 @@ let Option = {
|
|
|
61
79
|
getExn: getExn
|
|
62
80
|
};
|
|
63
81
|
|
|
82
|
+
function incrementBy(dict, key, count) {
|
|
83
|
+
let current = dict[key];
|
|
84
|
+
dict[key] = current !== undefined ? current + count | 0 : count;
|
|
85
|
+
}
|
|
86
|
+
|
|
64
87
|
function getOrInsertEmptyDict(dict, key) {
|
|
65
88
|
let d = dict[key];
|
|
66
89
|
if (d !== undefined) {
|
|
@@ -169,6 +192,7 @@ let shallowCopy = ((dict) => ({...dict}));
|
|
|
169
192
|
|
|
170
193
|
let Dict = {
|
|
171
194
|
$$delete: Stdlib_Dict.$$delete,
|
|
195
|
+
incrementBy: incrementBy,
|
|
172
196
|
getOrInsertEmptyDict: getOrInsertEmptyDict,
|
|
173
197
|
push: push,
|
|
174
198
|
pushMany: pushMany,
|
|
@@ -211,6 +235,10 @@ let UnsafeIntOperators = {};
|
|
|
211
235
|
|
|
212
236
|
let immutableEmpty = [];
|
|
213
237
|
|
|
238
|
+
function quotedJoin(names) {
|
|
239
|
+
return names.map(name => `"` + name + `"`).join(", ");
|
|
240
|
+
}
|
|
241
|
+
|
|
214
242
|
function mergeSorted(f, xs, ys) {
|
|
215
243
|
if (xs.length === 0) {
|
|
216
244
|
return ys;
|
|
@@ -393,6 +421,7 @@ let $$Array$1 = {
|
|
|
393
421
|
partition: Stdlib_Array.partition,
|
|
394
422
|
unzip: Stdlib_Array.unzip,
|
|
395
423
|
immutableEmpty: immutableEmpty,
|
|
424
|
+
quotedJoin: quotedJoin,
|
|
396
425
|
mergeSorted: mergeSorted,
|
|
397
426
|
clearInPlace: clearInPlace$1,
|
|
398
427
|
setIndexImmutable: setIndexImmutable,
|
|
@@ -462,15 +491,120 @@ function unwrapResultExn(res) {
|
|
|
462
491
|
throw res._0;
|
|
463
492
|
}
|
|
464
493
|
|
|
494
|
+
let asUint8Array = ((value) =>
|
|
495
|
+
value instanceof Uint8Array
|
|
496
|
+
? value.constructor === Uint8Array
|
|
497
|
+
? value
|
|
498
|
+
: new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
|
|
499
|
+
: undefined);
|
|
500
|
+
|
|
501
|
+
function toHex(bytes) {
|
|
502
|
+
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("hex");
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
function toPgArrayLiteral(values) {
|
|
506
|
+
return "{" + values.map(value => {
|
|
507
|
+
if (value == null) {
|
|
508
|
+
return "NULL";
|
|
509
|
+
}
|
|
510
|
+
let bytes = asUint8Array(value);
|
|
511
|
+
if (bytes !== undefined) {
|
|
512
|
+
return `"\\\\x` + toHex(bytes) + `"`;
|
|
513
|
+
} else if (Array.isArray(value)) {
|
|
514
|
+
return toPgArrayLiteral(value);
|
|
515
|
+
} else {
|
|
516
|
+
return Stdlib_JsError.throwWithMessage("Expected Uint8Array");
|
|
517
|
+
}
|
|
518
|
+
}).join(",") + "}";
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
function compare(a, b) {
|
|
522
|
+
let aLength = a.length;
|
|
523
|
+
let bLength = b.length;
|
|
524
|
+
let _index = 0;
|
|
525
|
+
while (true) {
|
|
526
|
+
let index = _index;
|
|
527
|
+
if (index === aLength || index === bLength) {
|
|
528
|
+
return Primitive_int.compare(aLength, bLength);
|
|
529
|
+
}
|
|
530
|
+
let order = Primitive_int.compare(a[index], b[index]);
|
|
531
|
+
if (order !== 0) {
|
|
532
|
+
return order;
|
|
533
|
+
}
|
|
534
|
+
_index = index + 1 | 0;
|
|
535
|
+
continue;
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
let Bytes = {
|
|
540
|
+
asUint8Array: asUint8Array,
|
|
541
|
+
toHex: toHex,
|
|
542
|
+
toPgArrayLiteral: toPgArrayLiteral,
|
|
543
|
+
compare: compare
|
|
544
|
+
};
|
|
545
|
+
|
|
465
546
|
let variantTag = S$RescriptSchema.union([
|
|
466
547
|
S$RescriptSchema.string,
|
|
467
548
|
S$RescriptSchema.object(s => s.f("TAG", S$RescriptSchema.string))
|
|
468
549
|
]);
|
|
469
550
|
|
|
551
|
+
function nullTolerant(schema) {
|
|
552
|
+
return S$RescriptSchema.transform(S$RescriptSchema.$$null(schema), param => ({
|
|
553
|
+
p: value => value,
|
|
554
|
+
s: value => {
|
|
555
|
+
if (value == null) {
|
|
556
|
+
return;
|
|
557
|
+
} else {
|
|
558
|
+
return Primitive_option.some(value);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}));
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
let bytes = S$RescriptSchema.custom("Bytes", s => ({
|
|
565
|
+
p: unknown => {
|
|
566
|
+
let bytes = asUint8Array(unknown);
|
|
567
|
+
if (bytes !== undefined) {
|
|
568
|
+
return bytes;
|
|
569
|
+
} else {
|
|
570
|
+
return s.fail("Expected Uint8Array", undefined);
|
|
571
|
+
}
|
|
572
|
+
},
|
|
573
|
+
s: bytes => bytes
|
|
574
|
+
}));
|
|
575
|
+
|
|
576
|
+
let bytesArray = S$RescriptSchema.custom("BytesArray", s => ({
|
|
577
|
+
p: unknown => {
|
|
578
|
+
if (Array.isArray(unknown)) {
|
|
579
|
+
return unknown.map(item => {
|
|
580
|
+
let bytes = asUint8Array(item);
|
|
581
|
+
if (bytes !== undefined) {
|
|
582
|
+
return bytes;
|
|
583
|
+
} else {
|
|
584
|
+
return s.fail("Expected Uint8Array", undefined);
|
|
585
|
+
}
|
|
586
|
+
});
|
|
587
|
+
} else {
|
|
588
|
+
return s.fail("Expected array of Uint8Array", undefined);
|
|
589
|
+
}
|
|
590
|
+
},
|
|
591
|
+
s: toPgArrayLiteral
|
|
592
|
+
}));
|
|
593
|
+
|
|
470
594
|
let dbDate = S$RescriptSchema.preprocess(S$RescriptSchema.json(false), param => ({
|
|
471
595
|
s: date => date.toISOString()
|
|
472
596
|
}));
|
|
473
597
|
|
|
598
|
+
let clickHouseJson = S$RescriptSchema.preprocess(S$RescriptSchema.json(false), param => ({
|
|
599
|
+
s: value => {
|
|
600
|
+
if (value == null) {
|
|
601
|
+
return "null";
|
|
602
|
+
} else {
|
|
603
|
+
return value;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
}));
|
|
607
|
+
|
|
474
608
|
let clickHouseDate = S$RescriptSchema.preprocess(S$RescriptSchema.json(false), param => ({
|
|
475
609
|
s: date => date.getTime()
|
|
476
610
|
}));
|
|
@@ -506,7 +640,11 @@ function coerceToJsonPgType(schema) {
|
|
|
506
640
|
|
|
507
641
|
let Schema = {
|
|
508
642
|
variantTag: variantTag,
|
|
643
|
+
nullTolerant: nullTolerant,
|
|
644
|
+
bytes: bytes,
|
|
645
|
+
bytesArray: bytesArray,
|
|
509
646
|
dbDate: dbDate,
|
|
647
|
+
clickHouseJson: clickHouseJson,
|
|
510
648
|
clickHouseDate: clickHouseDate,
|
|
511
649
|
coerceToJsonPgType: coerceToJsonPgType
|
|
512
650
|
};
|
|
@@ -640,6 +778,13 @@ function prettifyExn(exn) {
|
|
|
640
778
|
}
|
|
641
779
|
}
|
|
642
780
|
|
|
781
|
+
function exnMessage(exn) {
|
|
782
|
+
let jsExn = Primitive_exceptions.internalToException(exn);
|
|
783
|
+
if (jsExn.RE_EXN_ID === "JsExn") {
|
|
784
|
+
return Stdlib_JsExn.message(jsExn._1);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
|
|
643
788
|
let value;
|
|
644
789
|
|
|
645
790
|
try {
|
|
@@ -695,7 +840,13 @@ let $$Promise$1 = {
|
|
|
695
840
|
isCatchable: isCatchable
|
|
696
841
|
};
|
|
697
842
|
|
|
843
|
+
let maxBackoffExponent = 20;
|
|
844
|
+
|
|
698
845
|
export {
|
|
846
|
+
jitter,
|
|
847
|
+
clearTimeoutRef,
|
|
848
|
+
maxBackoffExponent,
|
|
849
|
+
expBackoff,
|
|
699
850
|
delay,
|
|
700
851
|
$$Object,
|
|
701
852
|
$$Error,
|
|
@@ -708,6 +859,7 @@ export {
|
|
|
708
859
|
Url,
|
|
709
860
|
Result,
|
|
710
861
|
unwrapResultExn,
|
|
862
|
+
Bytes,
|
|
711
863
|
Schema,
|
|
712
864
|
getVariantsTags,
|
|
713
865
|
$$Set,
|
|
@@ -717,6 +869,7 @@ export {
|
|
|
717
869
|
$$Proxy,
|
|
718
870
|
Hash,
|
|
719
871
|
prettifyExn,
|
|
872
|
+
exnMessage,
|
|
720
873
|
EnvioPackage,
|
|
721
874
|
$$BigInt,
|
|
722
875
|
$$Promise$1 as $$Promise,
|
package/src/Writing.res
CHANGED
|
@@ -158,8 +158,8 @@ let runOneWrite = async (state: IndexerState.t) => {
|
|
|
158
158
|
state->IndexerState.markCommitted(~upToCheckpointId)
|
|
159
159
|
|
|
160
160
|
switch rollback {
|
|
161
|
-
| Some({
|
|
162
|
-
await RollbackCommit.fire(~
|
|
161
|
+
| Some({progressedChains}) if RollbackCommit.callbacks->Utils.Array.notEmpty =>
|
|
162
|
+
await RollbackCommit.fire(~progressedChains)
|
|
163
163
|
| _ => ()
|
|
164
164
|
}
|
|
165
165
|
|
package/src/Writing.res.mjs
CHANGED
|
@@ -130,7 +130,7 @@ async function runOneWrite(state) {
|
|
|
130
130
|
]);
|
|
131
131
|
IndexerState.markCommitted(state, upToCheckpointId);
|
|
132
132
|
if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
|
|
133
|
-
await RollbackCommit.fire(rollback.
|
|
133
|
+
await RollbackCommit.fire(rollback.progressedChains);
|
|
134
134
|
}
|
|
135
135
|
return await PruneStaleHistory.runForced(state, pruneTargets);
|
|
136
136
|
}
|