envio 3.11.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +3 -0
- package/package.json +6 -6
- package/src/Batch.res +111 -78
- package/src/Batch.res.mjs +53 -46
- package/src/BatchProcessing.res +1 -11
- package/src/BatchProcessing.res.mjs +1 -4
- package/src/ChainState.res +72 -19
- package/src/ChainState.res.mjs +50 -12
- package/src/ChainState.resi +5 -0
- package/src/Config.res +58 -37
- package/src/Config.res.mjs +63 -53
- package/src/CrossChainState.res +49 -52
- package/src/CrossChainState.res.mjs +25 -23
- package/src/CrossChainState.resi +6 -13
- package/src/EffectState.res +13 -4
- package/src/EffectState.resi +7 -1
- package/src/EventProcessing.res +6 -6
- package/src/EventProcessing.res.mjs +6 -6
- package/src/FetchState.res +14 -0
- package/src/FetchState.res.mjs +14 -0
- package/src/FinalizeBackfill.res +16 -12
- package/src/FinalizeBackfill.res.mjs +3 -3
- package/src/HistoryPolicy.res +46 -0
- package/src/HistoryPolicy.res.mjs +50 -0
- package/src/InMemoryStore.res +30 -32
- package/src/InMemoryStore.res.mjs +33 -28
- package/src/IndexerState.res +66 -45
- package/src/IndexerState.res.mjs +57 -29
- package/src/IndexerState.resi +8 -13
- package/src/Internal.res +4 -2
- package/src/LoadLayer.res +11 -4
- package/src/LoadLayer.res.mjs +8 -5
- package/src/Main.res +25 -24
- package/src/Main.res.mjs +24 -25
- package/src/Metrics.res +49 -1
- package/src/Metrics.res.mjs +26 -3
- package/src/Persistence.res +55 -16
- package/src/Persistence.res.mjs +18 -3
- package/src/PgStorage.res +194 -118
- package/src/PgStorage.res.mjs +132 -72
- package/src/PruneStaleHistory.res +9 -15
- package/src/PruneStaleHistory.res.mjs +11 -20
- package/src/Rollback.res +9 -14
- package/src/Rollback.res.mjs +5 -3
- package/src/Sink.res +7 -5
- package/src/Sink.res.mjs +4 -4
- package/src/Staging.res +298 -0
- package/src/Staging.res.mjs +257 -0
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +7 -6
- package/src/UserContext.res +7 -4
- package/src/UserContext.res.mjs +5 -2
- package/src/Utils.res +9 -0
- package/src/Utils.res.mjs +9 -0
- package/src/Writing.res +34 -20
- package/src/Writing.res.mjs +15 -20
- package/src/bindings/ClickHouse.res +41 -34
- package/src/bindings/ClickHouse.res.mjs +36 -30
- package/src/bindings/ClickHouseSink.res +45 -186
- package/src/bindings/ClickHouseSink.res.mjs +21 -174
- package/src/db/CheckpointSequence.res +109 -0
- package/src/db/CheckpointSequence.res.mjs +99 -0
- package/src/db/EntityHistory.res +6 -5
- package/src/db/EntityHistory.res.mjs +5 -5
- package/src/db/Frontier.res +86 -0
- package/src/db/Frontier.res.mjs +126 -0
- package/src/db/InternalTable.res +145 -27
- package/src/db/InternalTable.res.mjs +115 -29
- package/src/db/RollbackFloors.res +34 -41
- package/src/db/RollbackFloors.res.mjs +27 -71
- package/src/sources/BlockStore.res +3 -0
- package/src/sources/EvmHyperSyncSource.res +3 -1
- package/src/sources/EvmHyperSyncSource.res.mjs +2 -1
- package/src/sources/HyperSync.res +2 -0
- package/src/sources/HyperSync.res.mjs +1 -0
- package/src/sources/HyperSync.resi +1 -0
- package/src/sources/HyperSyncClient.res +3 -0
- package/src/sources/RequestStat.res +8 -1
- package/src/sources/Source.res +4 -3
- package/src/sources/SourceManager.res +30 -10
- package/src/sources/SourceManager.res.mjs +24 -7
- package/src/sources/SourceManager.resi +2 -0
- package/src/tui/Tui.res +56 -143
- package/src/tui/Tui.res.mjs +70 -166
- package/src/tui/components/TuiData.res +61 -16
- package/src/tui/components/TuiData.res.mjs +53 -15
- package/svm.schema.json +0 -7
- package/src/db/CheckpointBounds.res +0 -53
- package/src/db/CheckpointBounds.res.mjs +0 -44
package/src/Rollback.res.mjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
|
-
import * as Config from "./Config.res.mjs";
|
|
5
4
|
import * as Logging from "./Logging.res.mjs";
|
|
6
5
|
import * as Writing from "./Writing.res.mjs";
|
|
6
|
+
import * as ChainMap from "./ChainMap.res.mjs";
|
|
7
7
|
import * as ChainState from "./ChainState.res.mjs";
|
|
8
8
|
import * as Performance from "./bindings/Performance.res.mjs";
|
|
9
9
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
@@ -44,7 +44,8 @@ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, sch
|
|
|
44
44
|
await Writing.flush(state);
|
|
45
45
|
let checkpointId = await IndexerState.persistence(state).storage.getRollbackTargetCheckpoint(reorgChain, rollbackTargetBlockNumber);
|
|
46
46
|
let rollbackTargetCheckpointId = checkpointId !== undefined ? checkpointId : 0n;
|
|
47
|
-
let
|
|
47
|
+
let config = IndexerState.config(state);
|
|
48
|
+
let next = RollbackFloors.make(config.checkpointSequence, ChainMap.keys(config.chainMap), reorgChain, rollbackTargetCheckpointId, rollbackTargetBlockNumber);
|
|
48
49
|
let match = IndexerState.pendingRollback(state);
|
|
49
50
|
let floors = match !== undefined ? RollbackFloors.merge(match.floors, next) : next;
|
|
50
51
|
let progressDiffByChain = {};
|
|
@@ -85,6 +86,7 @@ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, sch
|
|
|
85
86
|
rolledBackChains.push({
|
|
86
87
|
chainId: chainId,
|
|
87
88
|
progressBlockNumber: toBlock,
|
|
89
|
+
progressBlockTime: ChainState.committedProgressBlockTime(cs),
|
|
88
90
|
sourceBlockNumber: ChainState.knownHeight(cs),
|
|
89
91
|
totalEventsProcessed: ChainState.numEventsProcessed(cs)
|
|
90
92
|
});
|
|
@@ -97,7 +99,7 @@ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, sch
|
|
|
97
99
|
});
|
|
98
100
|
}
|
|
99
101
|
});
|
|
100
|
-
let diff$1 = await InMemoryStore.prepareRollbackDiff(state, floors,
|
|
102
|
+
let diff$1 = await InMemoryStore.prepareRollbackDiff(state, floors, rolledBackChains, rolledBackAddresses);
|
|
101
103
|
Logging.childTrace(logger, {
|
|
102
104
|
msg: "Rollback entity changes",
|
|
103
105
|
deleted: diff$1.deletedEntities,
|
package/src/Sink.res
CHANGED
|
@@ -2,12 +2,13 @@ type t = {
|
|
|
2
2
|
name: string,
|
|
3
3
|
initialize: (~entities: array<Internal.entityConfig>) => promise<unit>,
|
|
4
4
|
resume: (
|
|
5
|
-
~
|
|
5
|
+
~frontier: Frontier.t,
|
|
6
6
|
~chains: array<Persistence.initialChainState>,
|
|
7
7
|
~entities: array<Internal.entityConfig>,
|
|
8
8
|
) => promise<unit>,
|
|
9
9
|
writeBatch: (
|
|
10
10
|
~batch: Batch.t,
|
|
11
|
+
~diffCheckpoints: array<InternalTable.Checkpoints.diffCheckpoint>,
|
|
11
12
|
~updatedEntities: array<Persistence.updatedEntity>,
|
|
12
13
|
) => promise<unit>,
|
|
13
14
|
}
|
|
@@ -17,6 +18,7 @@ let makeClickHouse = (
|
|
|
17
18
|
~database,
|
|
18
19
|
~username,
|
|
19
20
|
~password,
|
|
21
|
+
~sequence: CheckpointSequence.t,
|
|
20
22
|
~chainIdMode: ChainId.mode=Int32,
|
|
21
23
|
): t => {
|
|
22
24
|
let sink = ClickHouse.makeSink(~host, ~username, ~password, ~database, ~chainIdMode)
|
|
@@ -29,10 +31,10 @@ let makeClickHouse = (
|
|
|
29
31
|
initialize: (~entities) => {
|
|
30
32
|
ClickHouse.initialize(sink, ~entities=mirrored(entities))
|
|
31
33
|
},
|
|
32
|
-
resume: (~
|
|
33
|
-
ClickHouse.resume(sink, ~
|
|
34
|
+
resume: (~frontier, ~chains, ~entities) => {
|
|
35
|
+
ClickHouse.resume(sink, ~sequence, ~frontier, ~chains, ~entities=mirrored(entities))
|
|
34
36
|
},
|
|
35
|
-
writeBatch: async (~batch, ~updatedEntities) => {
|
|
37
|
+
writeBatch: async (~batch, ~diffCheckpoints, ~updatedEntities) => {
|
|
36
38
|
// Staging reads JS values, so it holds the isolate and runs here. The
|
|
37
39
|
// encode and the round trips happen in Rust, which also keeps the
|
|
38
40
|
// checkpoints behind the rows they cover.
|
|
@@ -44,7 +46,7 @@ let makeClickHouse = (
|
|
|
44
46
|
| None => ()
|
|
45
47
|
}
|
|
46
48
|
)
|
|
47
|
-
ClickHouse.stageCheckpointsOrThrow(sink, ~registry, ~batch)
|
|
49
|
+
ClickHouse.stageCheckpointsOrThrow(sink, ~registry, ~batch, ~diffCheckpoints)
|
|
48
50
|
} catch {
|
|
49
51
|
| exn =>
|
|
50
52
|
sink->ClickHouseSink.discard(entities)
|
package/src/Sink.res.mjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import * as ClickHouse from "./bindings/ClickHouse.res.mjs";
|
|
4
4
|
|
|
5
|
-
function makeClickHouse(host, database, username, password, chainIdModeOpt) {
|
|
5
|
+
function makeClickHouse(host, database, username, password, sequence, chainIdModeOpt) {
|
|
6
6
|
let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
|
|
7
7
|
let sink = ClickHouse.makeSink(host, username, password, database, chainIdMode);
|
|
8
8
|
let registry = ClickHouse.makeRegistry();
|
|
@@ -10,8 +10,8 @@ function makeClickHouse(host, database, username, password, chainIdModeOpt) {
|
|
|
10
10
|
return {
|
|
11
11
|
name: "clickhouse",
|
|
12
12
|
initialize: entities => ClickHouse.initialize(sink, mirrored(entities)),
|
|
13
|
-
resume: (
|
|
14
|
-
writeBatch: async (batch, updatedEntities) => {
|
|
13
|
+
resume: (frontier, chains, entities) => ClickHouse.resume(sink, sequence, frontier, chains, mirrored(entities)),
|
|
14
|
+
writeBatch: async (batch, diffCheckpoints, updatedEntities) => {
|
|
15
15
|
let entities = [];
|
|
16
16
|
let checkpoints;
|
|
17
17
|
try {
|
|
@@ -22,7 +22,7 @@ function makeClickHouse(host, database, username, password, chainIdModeOpt) {
|
|
|
22
22
|
return;
|
|
23
23
|
}
|
|
24
24
|
});
|
|
25
|
-
checkpoints = ClickHouse.stageCheckpointsOrThrow(sink, registry, batch);
|
|
25
|
+
checkpoints = ClickHouse.stageCheckpointsOrThrow(sink, registry, batch, diffCheckpoints);
|
|
26
26
|
} catch (exn) {
|
|
27
27
|
sink.discard(entities);
|
|
28
28
|
throw exn;
|
package/src/Staging.res
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
// Filling a batch into a Rust-owned arena. The buffers below are lent by the
|
|
2
|
+
// addon and must be written only between `begin` and `commit`/`abort` — the
|
|
3
|
+
// rules, and why breaking them corrupts memory rather than failing a test, are
|
|
4
|
+
// written down at the top of `packages/cli/src/columnar/mod.rs`.
|
|
5
|
+
|
|
6
|
+
type kind =
|
|
7
|
+
| @as(0) F64
|
|
8
|
+
| @as(1) U64
|
|
9
|
+
| @as(2) I64
|
|
10
|
+
| @as(3) Text
|
|
11
|
+
| @as(4) Bytes
|
|
12
|
+
|
|
13
|
+
let kindOfOrdinal = ordinal =>
|
|
14
|
+
switch ordinal {
|
|
15
|
+
| 0 => F64
|
|
16
|
+
| 1 => U64
|
|
17
|
+
| 2 => I64
|
|
18
|
+
| 3 => Text
|
|
19
|
+
| 4 => Bytes
|
|
20
|
+
| unknown => JsError.throwWithMessage(`Unknown staged column kind ${unknown->Int.toString}`)
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// The replacer closes over nothing but the column's name, so it is built once
|
|
24
|
+
// with the column rather than once per column per batch.
|
|
25
|
+
type column = {name: string, kind: kind, isNullable: bool, replacer: JSON.replacer}
|
|
26
|
+
|
|
27
|
+
type begun = {handle: int, buffers: array<ArrayBuffer.t>}
|
|
28
|
+
|
|
29
|
+
// What the addon exposes, so this module stays free of any one sink's API.
|
|
30
|
+
type arena = {
|
|
31
|
+
beginStage: (~table: int, ~rows: int) => begun,
|
|
32
|
+
growStage: (~handle: int, ~column: int, ~needed: int, ~stale: ArrayBuffer.t) => ArrayBuffer.t,
|
|
33
|
+
commitStage: (~handle: int, ~buffers: array<ArrayBuffer.t>) => unit,
|
|
34
|
+
abortStage: (~handle: int, ~buffers: array<ArrayBuffer.t>) => unit,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
type encodeResult = {read: int, written: int}
|
|
38
|
+
type textEncoder
|
|
39
|
+
@new external makeTextEncoder: unit => textEncoder = "TextEncoder"
|
|
40
|
+
@send external encodeInto: (textEncoder, string, Uint8Array.t) => encodeResult = "encodeInto"
|
|
41
|
+
|
|
42
|
+
%%private(let encoder = makeTextEncoder())
|
|
43
|
+
|
|
44
|
+
@send external setFrom: (Uint8Array.t, Uint8Array.t, int) => unit = "set"
|
|
45
|
+
|
|
46
|
+
// A variable-width column's payload and the offsets that cut it into rows.
|
|
47
|
+
// `cursor` is where the next value starts — a row's bytes run from the previous
|
|
48
|
+
// row's end to its own, so every row sets `ends`, the null ones included.
|
|
49
|
+
// `slot` is which of the lent buffers carries `data`, since `grow` replaces it
|
|
50
|
+
// and the commit has to detach what is live rather than what was handed out.
|
|
51
|
+
type variable = {
|
|
52
|
+
mutable data: Uint8Array.t,
|
|
53
|
+
ends: Uint32Array.t,
|
|
54
|
+
slot: int,
|
|
55
|
+
mutable cursor: int,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// One per column kind, holding only the view that kind writes through.
|
|
59
|
+
type storage =
|
|
60
|
+
| Floats(Float64Array.t)
|
|
61
|
+
| Unsigned(BigUint64Array.t)
|
|
62
|
+
| Signed(BigInt64Array.t)
|
|
63
|
+
| Text(variable)
|
|
64
|
+
| Bytes(variable)
|
|
65
|
+
|
|
66
|
+
type builder = {
|
|
67
|
+
name: string,
|
|
68
|
+
isNullable: bool,
|
|
69
|
+
replacer: JSON.replacer,
|
|
70
|
+
index: int,
|
|
71
|
+
storage: storage,
|
|
72
|
+
nulls: Uint8Array.t,
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
type t = {
|
|
76
|
+
arena: arena,
|
|
77
|
+
handle: int,
|
|
78
|
+
buffers: array<ArrayBuffer.t>,
|
|
79
|
+
builders: array<builder>,
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
let begin = (arena, ~table, ~rows, ~columns: array<column>) => {
|
|
83
|
+
let {handle, buffers} = arena.beginStage(~table, ~rows)
|
|
84
|
+
let slot = ref(0)
|
|
85
|
+
let take = () => {
|
|
86
|
+
let buffer = buffers->Array.getUnsafe(slot.contents)
|
|
87
|
+
slot := slot.contents + 1
|
|
88
|
+
buffer
|
|
89
|
+
}
|
|
90
|
+
// Buffer order is the arena's: the values, then a variable column's offsets,
|
|
91
|
+
// then the null flags.
|
|
92
|
+
let takeVariable = () => {
|
|
93
|
+
let slot = slot.contents
|
|
94
|
+
let data = Uint8Array.fromBuffer(take())
|
|
95
|
+
let ends = Uint32Array.fromBuffer(take())
|
|
96
|
+
{data, ends, slot, cursor: 0}
|
|
97
|
+
}
|
|
98
|
+
let builders = columns->Array.mapWithIndex(({name, kind, isNullable, replacer}, index) => {
|
|
99
|
+
let storage = switch kind {
|
|
100
|
+
| F64 => Floats(Float64Array.fromBuffer(take()))
|
|
101
|
+
| U64 => Unsigned(BigUint64Array.fromBuffer(take()))
|
|
102
|
+
| I64 => Signed(BigInt64Array.fromBuffer(take()))
|
|
103
|
+
| Text => Text(takeVariable())
|
|
104
|
+
| Bytes => Bytes(takeVariable())
|
|
105
|
+
}
|
|
106
|
+
{name, isNullable, replacer, index, storage, nulls: Uint8Array.fromBuffer(take())}
|
|
107
|
+
})
|
|
108
|
+
{arena, handle, buffers, builders}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
%%private(
|
|
112
|
+
let ensure = (stage, builder, variable, ~needed) =>
|
|
113
|
+
if needed > variable.data->TypedArray.length {
|
|
114
|
+
let fresh = stage.arena.growStage(
|
|
115
|
+
~handle=stage.handle,
|
|
116
|
+
~column=builder.index,
|
|
117
|
+
~needed,
|
|
118
|
+
~stale=variable.data->TypedArray.buffer,
|
|
119
|
+
)
|
|
120
|
+
stage.buffers->Array.setUnsafe(variable.slot, fresh)
|
|
121
|
+
variable.data = Uint8Array.fromBuffer(fresh)
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
external asString: unknown => string = "%identity"
|
|
126
|
+
@val external toNumber: unknown => float = "Number"
|
|
127
|
+
@val external toBigInt: unknown => bigint = "BigInt"
|
|
128
|
+
@val external stringOf: unknown => string = "String"
|
|
129
|
+
|
|
130
|
+
// No column holds NaN or Infinity, but nothing downstream refuses them well: a
|
|
131
|
+
// list renders one as `null` on the way into JSON text, which the column then
|
|
132
|
+
// refuses for a reason naming `null` rather than what the handler wrote, and a
|
|
133
|
+
// scalar Float64 takes the raw bytes and stores a value no reader can render.
|
|
134
|
+
// Refused here instead, where the value is still itself, with one message for
|
|
135
|
+
// both shapes.
|
|
136
|
+
let finiteOrThrow = (number: float, ~column) =>
|
|
137
|
+
if number->Float.isFinite {
|
|
138
|
+
number
|
|
139
|
+
} else {
|
|
140
|
+
JsError.throwWithMessage(
|
|
141
|
+
`${number->Float.toString} is not a finite number, so it cannot be stored in the \`${column}\` column. Store a finite number, or keep it out of the entity.`,
|
|
142
|
+
)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
let toText = (value: unknown, ~replacer) =>
|
|
146
|
+
switch value->typeof {
|
|
147
|
+
| #string => value->asString
|
|
148
|
+
| #bigint => value->stringOf
|
|
149
|
+
| _ => value->(Utils.magic: unknown => JSON.t)->JSON.stringify(~replacer)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Copies `text` in as one byte per character, or returns -1 at the first
|
|
153
|
+
// character that needs more than one. An id, a hash, a decimal and an enum
|
|
154
|
+
// variant are all ASCII, which is most of what a text column ever holds, and
|
|
155
|
+
// this spares them the `subarray` that `encodeInto` needs to be given an
|
|
156
|
+
// offset — one short-lived object per cell is what the batch pays otherwise.
|
|
157
|
+
%%private(
|
|
158
|
+
let writeAscii: (Uint8Array.t, string, int) => int = %raw(`(data, text, offset) => {
|
|
159
|
+
const length = text.length;
|
|
160
|
+
for (let index = 0; index < length; index++) {
|
|
161
|
+
const code = text.charCodeAt(index);
|
|
162
|
+
if (code > 0x7f) {
|
|
163
|
+
return -1;
|
|
164
|
+
}
|
|
165
|
+
data[offset + index] = code;
|
|
166
|
+
}
|
|
167
|
+
return length;
|
|
168
|
+
}`)
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
%%private(
|
|
172
|
+
let writeText = (stage, builder, variable, ~row, text) => {
|
|
173
|
+
// One byte per UTF-16 unit is what ASCII needs, so the room for the fast
|
|
174
|
+
// path is the room for its guess. `read` says when the guess was short:
|
|
175
|
+
// UTF-8 spends at most three bytes per unit, which is what a surrogate pair
|
|
176
|
+
// costs across its two.
|
|
177
|
+
let units = text->String.length
|
|
178
|
+
stage->ensure(builder, variable, ~needed=variable.cursor + units)
|
|
179
|
+
let written = switch variable.data->writeAscii(text, variable.cursor) {
|
|
180
|
+
| -1 =>
|
|
181
|
+
let {read, written} =
|
|
182
|
+
encoder->encodeInto(text, variable.data->TypedArray.subarray(~start=variable.cursor))
|
|
183
|
+
if read < units {
|
|
184
|
+
stage->ensure(builder, variable, ~needed=variable.cursor + units * 3)
|
|
185
|
+
let {written} =
|
|
186
|
+
encoder->encodeInto(text, variable.data->TypedArray.subarray(~start=variable.cursor))
|
|
187
|
+
written
|
|
188
|
+
} else {
|
|
189
|
+
written
|
|
190
|
+
}
|
|
191
|
+
| ascii => ascii
|
|
192
|
+
}
|
|
193
|
+
variable.cursor = variable.cursor + written
|
|
194
|
+
variable.ends->TypedArray.set(row, variable.cursor)
|
|
195
|
+
}
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
%%private(
|
|
199
|
+
let writeBytes = (stage, builder, variable, ~row, bytes: Uint8Array.t) => {
|
|
200
|
+
let length = bytes->TypedArray.length
|
|
201
|
+
stage->ensure(builder, variable, ~needed=variable.cursor + length)
|
|
202
|
+
variable.data->setFrom(bytes, variable.cursor)
|
|
203
|
+
variable.cursor = variable.cursor + length
|
|
204
|
+
variable.ends->TypedArray.set(row, variable.cursor)
|
|
205
|
+
}
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
%%private(
|
|
209
|
+
let markNull = (builder, ~row) => {
|
|
210
|
+
builder.nulls->TypedArray.set(row, 1)
|
|
211
|
+
// A row with no value still ends where the one before it did, or the row
|
|
212
|
+
// after it would start before its own beginning.
|
|
213
|
+
switch builder.storage {
|
|
214
|
+
| Text(variable) | Bytes(variable) => variable.ends->TypedArray.set(row, variable.cursor)
|
|
215
|
+
| Floats(_) | Unsigned(_) | Signed(_) => ()
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
// RowBinary carries the raw integer, so a value the column cannot hold is not
|
|
221
|
+
// rejected anywhere downstream — and a typed array reduces it modulo 2^64 on
|
|
222
|
+
// the way in rather than refusing it, which would leave no trace at all.
|
|
223
|
+
%%private(
|
|
224
|
+
let checkedBigInt = (value: unknown, ~builder, ~min, ~max) => {
|
|
225
|
+
let value = value->toBigInt
|
|
226
|
+
if value < min || value > max {
|
|
227
|
+
JsError.throwWithMessage(
|
|
228
|
+
`${value->BigInt.toString} is out of range for the \`${builder.name}\` column`,
|
|
229
|
+
)
|
|
230
|
+
}
|
|
231
|
+
value
|
|
232
|
+
}
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
%%private(
|
|
236
|
+
let writePresent = (stage, builder, ~row, value: unknown) =>
|
|
237
|
+
switch builder.storage {
|
|
238
|
+
| Floats(floats) =>
|
|
239
|
+
floats->TypedArray.set(row, value->toNumber->finiteOrThrow(~column=builder.name))
|
|
240
|
+
| Unsigned(unsigned) =>
|
|
241
|
+
unsigned->TypedArray.set(
|
|
242
|
+
row,
|
|
243
|
+
value->checkedBigInt(~builder, ~min=0n, ~max=18446744073709551615n),
|
|
244
|
+
)
|
|
245
|
+
| Signed(signed) =>
|
|
246
|
+
signed->TypedArray.set(
|
|
247
|
+
row,
|
|
248
|
+
value->checkedBigInt(~builder, ~min=-9223372036854775808n, ~max=9223372036854775807n),
|
|
249
|
+
)
|
|
250
|
+
| Text(variable) =>
|
|
251
|
+
stage->writeText(builder, variable, ~row, value->toText(~replacer=builder.replacer))
|
|
252
|
+
| Bytes(variable) =>
|
|
253
|
+
stage->writeBytes(builder, variable, ~row, value->(Utils.magic: unknown => Uint8Array.t))
|
|
254
|
+
}
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
%%private(let isAbsent = (value: unknown) => value === %raw(`undefined`) || value === %raw(`null`))
|
|
258
|
+
|
|
259
|
+
// Writes one value of a row the handler set. `undefined`/`null` marks the row's
|
|
260
|
+
// null bit, which a column that accepts NULL stores as such — and which one that
|
|
261
|
+
// does not has no way to store: RowBinary carries no "absent", so the row would
|
|
262
|
+
// land holding the type's zero, a value the handler never chose and that nothing
|
|
263
|
+
// downstream could tell from one it did.
|
|
264
|
+
let writeValue = (stage, ~column, ~row, value: unknown) => {
|
|
265
|
+
let builder = stage.builders->Array.getUnsafe(column)
|
|
266
|
+
if value->isAbsent {
|
|
267
|
+
if builder.isNullable {
|
|
268
|
+
builder->markNull(~row)
|
|
269
|
+
} else {
|
|
270
|
+
JsError.throwWithMessage(
|
|
271
|
+
`No value for the \`${builder.name}\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`,
|
|
272
|
+
)
|
|
273
|
+
}
|
|
274
|
+
} else {
|
|
275
|
+
stage->writePresent(builder, ~row, value)
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
let writeDeletedValue = (stage, ~column, ~row, value: unknown) => {
|
|
280
|
+
let builder = stage.builders->Array.getUnsafe(column)
|
|
281
|
+
if value->isAbsent {
|
|
282
|
+
builder->markNull(~row)
|
|
283
|
+
} else {
|
|
284
|
+
stage->writePresent(builder, ~row, value)
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
let columnCount = stage => stage.builders->Array.length
|
|
289
|
+
|
|
290
|
+
// Ends the filling phase and returns the handle the write is asked for. Every
|
|
291
|
+
// lent buffer is detached here, so a view left over from this stage throws on
|
|
292
|
+
// its next write instead of reaching memory Rust is reading.
|
|
293
|
+
let commit = stage => {
|
|
294
|
+
stage.arena.commitStage(~handle=stage.handle, ~buffers=stage.buffers)
|
|
295
|
+
stage.handle
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
let abort = stage => stage.arena.abortStage(~handle=stage.handle, ~buffers=stage.buffers)
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
4
|
+
|
|
5
|
+
function kindOfOrdinal(ordinal) {
|
|
6
|
+
switch (ordinal) {
|
|
7
|
+
case 0 :
|
|
8
|
+
return 0;
|
|
9
|
+
case 1 :
|
|
10
|
+
return 1;
|
|
11
|
+
case 2 :
|
|
12
|
+
return 2;
|
|
13
|
+
case 3 :
|
|
14
|
+
return 3;
|
|
15
|
+
case 4 :
|
|
16
|
+
return 4;
|
|
17
|
+
default:
|
|
18
|
+
return Stdlib_JsError.throwWithMessage(`Unknown staged column kind ` + ordinal.toString());
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
let encoder = new TextEncoder();
|
|
23
|
+
|
|
24
|
+
function begin(arena, table, rows, columns) {
|
|
25
|
+
let match = arena.beginStage(table, rows);
|
|
26
|
+
let buffers = match.buffers;
|
|
27
|
+
let slot = {
|
|
28
|
+
contents: 0
|
|
29
|
+
};
|
|
30
|
+
let take = () => {
|
|
31
|
+
let buffer = buffers[slot.contents];
|
|
32
|
+
slot.contents = slot.contents + 1 | 0;
|
|
33
|
+
return buffer;
|
|
34
|
+
};
|
|
35
|
+
let takeVariable = () => {
|
|
36
|
+
let slot$1 = slot.contents;
|
|
37
|
+
let data = new Uint8Array(take());
|
|
38
|
+
let ends = new Uint32Array(take());
|
|
39
|
+
return {
|
|
40
|
+
data: data,
|
|
41
|
+
ends: ends,
|
|
42
|
+
slot: slot$1,
|
|
43
|
+
cursor: 0
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
let builders = columns.map((param, index) => {
|
|
47
|
+
let storage;
|
|
48
|
+
switch (param.kind) {
|
|
49
|
+
case 0 :
|
|
50
|
+
storage = {
|
|
51
|
+
TAG: "Floats",
|
|
52
|
+
_0: new Float64Array(take())
|
|
53
|
+
};
|
|
54
|
+
break;
|
|
55
|
+
case 1 :
|
|
56
|
+
storage = {
|
|
57
|
+
TAG: "Unsigned",
|
|
58
|
+
_0: new BigUint64Array(take())
|
|
59
|
+
};
|
|
60
|
+
break;
|
|
61
|
+
case 2 :
|
|
62
|
+
storage = {
|
|
63
|
+
TAG: "Signed",
|
|
64
|
+
_0: new BigInt64Array(take())
|
|
65
|
+
};
|
|
66
|
+
break;
|
|
67
|
+
case 3 :
|
|
68
|
+
storage = {
|
|
69
|
+
TAG: "Text",
|
|
70
|
+
_0: takeVariable()
|
|
71
|
+
};
|
|
72
|
+
break;
|
|
73
|
+
case 4 :
|
|
74
|
+
storage = {
|
|
75
|
+
TAG: "Bytes",
|
|
76
|
+
_0: takeVariable()
|
|
77
|
+
};
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
return {
|
|
81
|
+
name: param.name,
|
|
82
|
+
isNullable: param.isNullable,
|
|
83
|
+
replacer: param.replacer,
|
|
84
|
+
index: index,
|
|
85
|
+
storage: storage,
|
|
86
|
+
nulls: new Uint8Array(take())
|
|
87
|
+
};
|
|
88
|
+
});
|
|
89
|
+
return {
|
|
90
|
+
arena: arena,
|
|
91
|
+
handle: match.handle,
|
|
92
|
+
buffers: buffers,
|
|
93
|
+
builders: builders
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function ensure(stage, builder, variable, needed) {
|
|
98
|
+
if (needed <= variable.data.length) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
let fresh = stage.arena.growStage(stage.handle, builder.index, needed, variable.data.buffer);
|
|
102
|
+
stage.buffers[variable.slot] = fresh;
|
|
103
|
+
variable.data = new Uint8Array(fresh);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function finiteOrThrow(number, column) {
|
|
107
|
+
if (isFinite(number)) {
|
|
108
|
+
return number;
|
|
109
|
+
} else {
|
|
110
|
+
return Stdlib_JsError.throwWithMessage(number.toString() + ` is not a finite number, so it cannot be stored in the \`` + column + `\` column. Store a finite number, or keep it out of the entity.`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function toText(value, replacer) {
|
|
115
|
+
let match = typeof value;
|
|
116
|
+
if (match === "bigint") {
|
|
117
|
+
return String(value);
|
|
118
|
+
} else if (match === "string") {
|
|
119
|
+
return value;
|
|
120
|
+
} else {
|
|
121
|
+
return JSON.stringify(value, replacer);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let writeAscii = ((data, text, offset) => {
|
|
126
|
+
const length = text.length;
|
|
127
|
+
for (let index = 0; index < length; index++) {
|
|
128
|
+
const code = text.charCodeAt(index);
|
|
129
|
+
if (code > 0x7f) {
|
|
130
|
+
return -1;
|
|
131
|
+
}
|
|
132
|
+
data[offset + index] = code;
|
|
133
|
+
}
|
|
134
|
+
return length;
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
function markNull(builder, row) {
|
|
138
|
+
builder.nulls[row] = 1;
|
|
139
|
+
let match = builder.storage;
|
|
140
|
+
switch (match.TAG) {
|
|
141
|
+
case "Text" :
|
|
142
|
+
case "Bytes" :
|
|
143
|
+
break;
|
|
144
|
+
default:
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
let variable = match._0;
|
|
148
|
+
variable.ends[row] = variable.cursor;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function checkedBigInt(value, builder, min, max) {
|
|
152
|
+
let value$1 = BigInt(value);
|
|
153
|
+
if (value$1 < min || value$1 > max) {
|
|
154
|
+
Stdlib_JsError.throwWithMessage(value$1.toString() + ` is out of range for the \`` + builder.name + `\` column`);
|
|
155
|
+
}
|
|
156
|
+
return value$1;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function writePresent(stage, builder, row, value) {
|
|
160
|
+
let floats = builder.storage;
|
|
161
|
+
switch (floats.TAG) {
|
|
162
|
+
case "Floats" :
|
|
163
|
+
floats._0[row] = finiteOrThrow(Number(value), builder.name);
|
|
164
|
+
return;
|
|
165
|
+
case "Unsigned" :
|
|
166
|
+
floats._0[row] = checkedBigInt(value, builder, 0n, 18446744073709551615n);
|
|
167
|
+
return;
|
|
168
|
+
case "Signed" :
|
|
169
|
+
floats._0[row] = checkedBigInt(value, builder, -9223372036854775808n, 9223372036854775807n);
|
|
170
|
+
return;
|
|
171
|
+
case "Text" :
|
|
172
|
+
let variable = floats._0;
|
|
173
|
+
let text = toText(value, builder.replacer);
|
|
174
|
+
let units = text.length;
|
|
175
|
+
ensure(stage, builder, variable, variable.cursor + units | 0);
|
|
176
|
+
let ascii = writeAscii(variable.data, text, variable.cursor);
|
|
177
|
+
let written;
|
|
178
|
+
if (ascii !== -1) {
|
|
179
|
+
written = ascii;
|
|
180
|
+
} else {
|
|
181
|
+
let match = encoder.encodeInto(text, variable.data.subarray(variable.cursor));
|
|
182
|
+
if (match.read < units) {
|
|
183
|
+
ensure(stage, builder, variable, variable.cursor + (units * 3 | 0) | 0);
|
|
184
|
+
written = encoder.encodeInto(text, variable.data.subarray(variable.cursor)).written;
|
|
185
|
+
} else {
|
|
186
|
+
written = match.written;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
variable.cursor = variable.cursor + written | 0;
|
|
190
|
+
variable.ends[row] = variable.cursor;
|
|
191
|
+
return;
|
|
192
|
+
case "Bytes" :
|
|
193
|
+
let variable$1 = floats._0;
|
|
194
|
+
let length = value.length;
|
|
195
|
+
ensure(stage, builder, variable$1, variable$1.cursor + length | 0);
|
|
196
|
+
variable$1.data.set(value, variable$1.cursor);
|
|
197
|
+
variable$1.cursor = variable$1.cursor + length | 0;
|
|
198
|
+
variable$1.ends[row] = variable$1.cursor;
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function isAbsent(value) {
|
|
204
|
+
if (value === undefined) {
|
|
205
|
+
return true;
|
|
206
|
+
} else {
|
|
207
|
+
return value === null;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function writeValue(stage, column, row, value) {
|
|
212
|
+
let builder = stage.builders[column];
|
|
213
|
+
if (isAbsent(value)) {
|
|
214
|
+
if (builder.isNullable) {
|
|
215
|
+
return markNull(builder, row);
|
|
216
|
+
} else {
|
|
217
|
+
return Stdlib_JsError.throwWithMessage(`No value for the \`` + builder.name + `\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`);
|
|
218
|
+
}
|
|
219
|
+
} else {
|
|
220
|
+
return writePresent(stage, builder, row, value);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function writeDeletedValue(stage, column, row, value) {
|
|
225
|
+
let builder = stage.builders[column];
|
|
226
|
+
if (isAbsent(value)) {
|
|
227
|
+
return markNull(builder, row);
|
|
228
|
+
} else {
|
|
229
|
+
return writePresent(stage, builder, row, value);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function columnCount(stage) {
|
|
234
|
+
return stage.builders.length;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function commit(stage) {
|
|
238
|
+
stage.arena.commitStage(stage.handle, stage.buffers);
|
|
239
|
+
return stage.handle;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function abort(stage) {
|
|
243
|
+
stage.arena.abortStage(stage.handle, stage.buffers);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export {
|
|
247
|
+
kindOfOrdinal,
|
|
248
|
+
begin,
|
|
249
|
+
finiteOrThrow,
|
|
250
|
+
toText,
|
|
251
|
+
writeValue,
|
|
252
|
+
writeDeletedValue,
|
|
253
|
+
columnCount,
|
|
254
|
+
commit,
|
|
255
|
+
abort,
|
|
256
|
+
}
|
|
257
|
+
/* encoder Not a pure module */
|