envio 3.11.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +3 -0
- package/package.json +6 -6
- package/src/Batch.res +111 -78
- package/src/Batch.res.mjs +53 -46
- package/src/BatchProcessing.res +1 -11
- package/src/BatchProcessing.res.mjs +1 -4
- package/src/ChainState.res +72 -19
- package/src/ChainState.res.mjs +50 -12
- package/src/ChainState.resi +5 -0
- package/src/Config.res +58 -37
- package/src/Config.res.mjs +63 -53
- package/src/CrossChainState.res +49 -52
- package/src/CrossChainState.res.mjs +25 -23
- package/src/CrossChainState.resi +6 -13
- package/src/EffectState.res +13 -4
- package/src/EffectState.resi +7 -1
- package/src/EventProcessing.res +6 -6
- package/src/EventProcessing.res.mjs +6 -6
- package/src/FetchState.res +14 -0
- package/src/FetchState.res.mjs +14 -0
- package/src/FinalizeBackfill.res +16 -12
- package/src/FinalizeBackfill.res.mjs +3 -3
- package/src/HistoryPolicy.res +46 -0
- package/src/HistoryPolicy.res.mjs +50 -0
- package/src/InMemoryStore.res +30 -32
- package/src/InMemoryStore.res.mjs +33 -28
- package/src/IndexerState.res +66 -45
- package/src/IndexerState.res.mjs +57 -29
- package/src/IndexerState.resi +8 -13
- package/src/Internal.res +4 -2
- package/src/LoadLayer.res +11 -4
- package/src/LoadLayer.res.mjs +8 -5
- package/src/Main.res +25 -24
- package/src/Main.res.mjs +24 -25
- package/src/Metrics.res +49 -1
- package/src/Metrics.res.mjs +26 -3
- package/src/Persistence.res +55 -16
- package/src/Persistence.res.mjs +18 -3
- package/src/PgStorage.res +194 -118
- package/src/PgStorage.res.mjs +132 -72
- package/src/PruneStaleHistory.res +9 -15
- package/src/PruneStaleHistory.res.mjs +11 -20
- package/src/Rollback.res +9 -14
- package/src/Rollback.res.mjs +5 -3
- package/src/Sink.res +7 -5
- package/src/Sink.res.mjs +4 -4
- package/src/Staging.res +298 -0
- package/src/Staging.res.mjs +257 -0
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +7 -6
- package/src/UserContext.res +7 -4
- package/src/UserContext.res.mjs +5 -2
- package/src/Utils.res +9 -0
- package/src/Utils.res.mjs +9 -0
- package/src/Writing.res +34 -20
- package/src/Writing.res.mjs +15 -20
- package/src/bindings/ClickHouse.res +41 -34
- package/src/bindings/ClickHouse.res.mjs +36 -30
- package/src/bindings/ClickHouseSink.res +45 -186
- package/src/bindings/ClickHouseSink.res.mjs +21 -174
- package/src/db/CheckpointSequence.res +109 -0
- package/src/db/CheckpointSequence.res.mjs +99 -0
- package/src/db/EntityHistory.res +6 -5
- package/src/db/EntityHistory.res.mjs +5 -5
- package/src/db/Frontier.res +86 -0
- package/src/db/Frontier.res.mjs +126 -0
- package/src/db/InternalTable.res +145 -27
- package/src/db/InternalTable.res.mjs +115 -29
- package/src/db/RollbackFloors.res +34 -41
- package/src/db/RollbackFloors.res.mjs +27 -71
- package/src/sources/BlockStore.res +3 -0
- package/src/sources/EvmHyperSyncSource.res +3 -1
- package/src/sources/EvmHyperSyncSource.res.mjs +2 -1
- package/src/sources/HyperSync.res +2 -0
- package/src/sources/HyperSync.res.mjs +1 -0
- package/src/sources/HyperSync.resi +1 -0
- package/src/sources/HyperSyncClient.res +3 -0
- package/src/sources/RequestStat.res +8 -1
- package/src/sources/Source.res +4 -3
- package/src/sources/SourceManager.res +30 -10
- package/src/sources/SourceManager.res.mjs +24 -7
- package/src/sources/SourceManager.resi +2 -0
- package/src/tui/Tui.res +56 -143
- package/src/tui/Tui.res.mjs +70 -166
- package/src/tui/components/TuiData.res +61 -16
- package/src/tui/components/TuiData.res.mjs +53 -15
- package/svm.schema.json +0 -7
- package/src/db/CheckpointBounds.res +0 -53
- package/src/db/CheckpointBounds.res.mjs +0 -44
|
@@ -5,6 +5,8 @@ import * as Table from "../db/Table.res.mjs";
|
|
|
5
5
|
import * as Utils from "../Utils.res.mjs";
|
|
6
6
|
import * as ChainId from "../ChainId.res.mjs";
|
|
7
7
|
import * as Logging from "../Logging.res.mjs";
|
|
8
|
+
import * as Staging from "../Staging.res.mjs";
|
|
9
|
+
import * as Frontier from "../db/Frontier.res.mjs";
|
|
8
10
|
import * as Internal from "../Internal.res.mjs";
|
|
9
11
|
import * as Persistence from "../Persistence.res.mjs";
|
|
10
12
|
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
@@ -176,7 +178,8 @@ let checkpointColumns = Stdlib_Array.filterMap(InternalTable.Checkpoints.columns
|
|
|
176
178
|
let match = field._0;
|
|
177
179
|
return {
|
|
178
180
|
spec: makeColumnSpec(match.fieldName, undefined, param.clickHouseFieldType, match.isNullable, undefined),
|
|
179
|
-
valuesOf: param.valuesOf
|
|
181
|
+
valuesOf: param.valuesOf,
|
|
182
|
+
diffValuesOf: param.diffValuesOf
|
|
180
183
|
};
|
|
181
184
|
});
|
|
182
185
|
|
|
@@ -303,27 +306,24 @@ function makeConverters(entityConfig, scope, table) {
|
|
|
303
306
|
};
|
|
304
307
|
}
|
|
305
308
|
|
|
306
|
-
function
|
|
307
|
-
let
|
|
308
|
-
let
|
|
309
|
-
let columns = builders.length;
|
|
310
|
-
for (let row = 0; row < rows; ++row) {
|
|
309
|
+
function fillStage(stage, converters, changes) {
|
|
310
|
+
let columns = Staging.columnCount(stage);
|
|
311
|
+
for (let row = 0, row_finish = changes.length; row < row_finish; ++row) {
|
|
311
312
|
let change = changes[row];
|
|
312
313
|
let match;
|
|
313
314
|
match = change.type === "SET" ? [
|
|
314
315
|
converters.setCells,
|
|
315
|
-
|
|
316
|
+
Staging.writeValue
|
|
316
317
|
] : [
|
|
317
318
|
converters.deleteCells,
|
|
318
|
-
|
|
319
|
+
Staging.writeDeletedValue
|
|
319
320
|
];
|
|
320
321
|
let write = match[1];
|
|
321
322
|
let cells = match[0];
|
|
322
323
|
for (let column = 0; column < columns; ++column) {
|
|
323
|
-
write(
|
|
324
|
+
write(stage, column, row, cells[column](change));
|
|
324
325
|
}
|
|
325
326
|
}
|
|
326
|
-
return builders;
|
|
327
327
|
}
|
|
328
328
|
|
|
329
329
|
function entityTable(sink, registry, entityConfig) {
|
|
@@ -347,27 +347,25 @@ function checkpointsTable(sink, registry) {
|
|
|
347
347
|
return table$1;
|
|
348
348
|
}
|
|
349
349
|
|
|
350
|
-
function
|
|
351
|
-
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
function stageCheckpointsOrThrow(sink, registry, batch) {
|
|
355
|
-
let rows = batch.checkpointIds.length;
|
|
350
|
+
function stageCheckpointsOrThrow(sink, registry, batch, diffCheckpoints) {
|
|
351
|
+
let rows = batch.checkpointIds.length + diffCheckpoints.length | 0;
|
|
356
352
|
if (rows === 0) {
|
|
357
353
|
return null;
|
|
358
354
|
}
|
|
359
355
|
let table = checkpointsTable(sink, registry);
|
|
356
|
+
let stage = Staging.begin(ClickHouseSink.arena(sink), table.handle, rows, table.columns);
|
|
360
357
|
try {
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
let columnValues =
|
|
358
|
+
table.columns.forEach((param, column) => {
|
|
359
|
+
let source = checkpointColumns[column];
|
|
360
|
+
let columnValues = source.valuesOf(batch).concat(source.diffValuesOf(diffCheckpoints));
|
|
364
361
|
for (let row = 0; row < rows; ++row) {
|
|
365
|
-
|
|
362
|
+
Staging.writeValue(stage, column, row, columnValues[row]);
|
|
366
363
|
}
|
|
367
364
|
});
|
|
368
|
-
return
|
|
365
|
+
return Staging.commit(stage);
|
|
369
366
|
} catch (raw_exn) {
|
|
370
367
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
368
|
+
Staging.abort(stage);
|
|
371
369
|
throw {
|
|
372
370
|
RE_EXN_ID: Persistence.StorageError,
|
|
373
371
|
message: `Failed to convert checkpoints for ClickHouse table "` + table.name + `"`,
|
|
@@ -385,6 +383,7 @@ function stageUpdatesOrThrow(sink, registry, changes, entityConfig, scope) {
|
|
|
385
383
|
let table = entityTable(sink, registry, entityConfig);
|
|
386
384
|
let tableName = table.name;
|
|
387
385
|
let cacheKey = entityConfig.name + `|` + Internal.chainScopeToString(scope);
|
|
386
|
+
let stage = Staging.begin(ClickHouseSink.arena(sink), table.handle, rows, table.columns);
|
|
388
387
|
try {
|
|
389
388
|
let cached = registry.converters[cacheKey];
|
|
390
389
|
let converters;
|
|
@@ -395,10 +394,11 @@ function stageUpdatesOrThrow(sink, registry, changes, entityConfig, scope) {
|
|
|
395
394
|
registry.converters[cacheKey] = cached$1;
|
|
396
395
|
converters = cached$1;
|
|
397
396
|
}
|
|
398
|
-
|
|
399
|
-
return
|
|
397
|
+
fillStage(stage, converters, changes);
|
|
398
|
+
return Staging.commit(stage);
|
|
400
399
|
} catch (raw_exn) {
|
|
401
400
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
401
|
+
Staging.abort(stage);
|
|
402
402
|
throw {
|
|
403
403
|
RE_EXN_ID: Persistence.StorageError,
|
|
404
404
|
message: `Failed to convert items for ClickHouse table "` + tableName + `"`,
|
|
@@ -443,18 +443,25 @@ async function initialize(sink, entities) {
|
|
|
443
443
|
}
|
|
444
444
|
}
|
|
445
445
|
|
|
446
|
-
async function resume(sink,
|
|
446
|
+
async function resume(sink, sequence, frontier, chains, entities) {
|
|
447
447
|
let chainProgress = chains.map(chain => ({
|
|
448
448
|
chainId: ChainId.toString(chain.id),
|
|
449
|
-
progressBlockNumber: chain.progressBlockNumber
|
|
449
|
+
progressBlockNumber: chain.progressBlockNumber,
|
|
450
|
+
committedCheckpointId: Frontier.get(frontier, chain.id).toString()
|
|
450
451
|
}));
|
|
451
452
|
try {
|
|
453
|
+
let tmp;
|
|
454
|
+
tmp = sequence !== "SharedAcrossChains";
|
|
452
455
|
return await sink.resume({
|
|
453
|
-
|
|
456
|
+
perChain: tmp,
|
|
454
457
|
chainProgress: chainProgress,
|
|
455
|
-
historyTables: entities.map(entityConfig =>
|
|
456
|
-
|
|
457
|
-
|
|
458
|
+
historyTables: entities.map(entityConfig => {
|
|
459
|
+
let spec = entitySpec(entityConfig);
|
|
460
|
+
return {
|
|
461
|
+
name: spec.historyTable,
|
|
462
|
+
chainIdColumn: spec.chainIdColumn
|
|
463
|
+
};
|
|
464
|
+
})
|
|
458
465
|
});
|
|
459
466
|
} catch (raw_exn) {
|
|
460
467
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
@@ -480,7 +487,6 @@ export {
|
|
|
480
487
|
makeRegistry,
|
|
481
488
|
entityTable,
|
|
482
489
|
checkpointsTable,
|
|
483
|
-
stageBuilders,
|
|
484
490
|
stageCheckpointsOrThrow,
|
|
485
491
|
stageUpdatesOrThrow,
|
|
486
492
|
writeStagedOrThrow,
|
|
@@ -9,9 +9,16 @@ type historySchema = {
|
|
|
9
9
|
checkpointsTable: string,
|
|
10
10
|
checkpointChainIdColumn: string,
|
|
11
11
|
checkpointBlockNumberColumn: string,
|
|
12
|
+
chainsTable: string,
|
|
13
|
+
chainsCheckpointIdColumn: string,
|
|
12
14
|
}
|
|
13
15
|
|
|
14
|
-
type chainProgressInput = {
|
|
16
|
+
type chainProgressInput = {
|
|
17
|
+
chainId: string,
|
|
18
|
+
progressBlockNumber: int,
|
|
19
|
+
// Where the chain's own sequence stands, as Postgres has it committed.
|
|
20
|
+
committedCheckpointId: string,
|
|
21
|
+
}
|
|
15
22
|
|
|
16
23
|
type options = {
|
|
17
24
|
url: string,
|
|
@@ -62,12 +69,14 @@ type initializeInput = {
|
|
|
62
69
|
databaseEngine?: string,
|
|
63
70
|
}
|
|
64
71
|
|
|
72
|
+
// A history table and the column naming the chain its rows belong to. Absent
|
|
73
|
+
// only for a cross-chain entity, which no per-chain sequence can produce.
|
|
74
|
+
type historyTableInput = {name: string, chainIdColumn?: string}
|
|
75
|
+
|
|
65
76
|
type resumeInput = {
|
|
66
|
-
|
|
77
|
+
perChain: bool,
|
|
67
78
|
chainProgress: array<chainProgressInput>,
|
|
68
|
-
historyTables: array<
|
|
69
|
-
replicated: bool,
|
|
70
|
-
databaseEngine?: string,
|
|
79
|
+
historyTables: array<historyTableInput>,
|
|
71
80
|
}
|
|
72
81
|
|
|
73
82
|
type registeredTable = {
|
|
@@ -77,15 +86,6 @@ type registeredTable = {
|
|
|
77
86
|
nullable: array<bool>,
|
|
78
87
|
}
|
|
79
88
|
|
|
80
|
-
type columnValuesInput = {
|
|
81
|
-
numbers?: Float64Array.t,
|
|
82
|
-
unsigned64?: BigUint64Array.t,
|
|
83
|
-
signed64?: BigInt64Array.t,
|
|
84
|
-
texts?: array<string>,
|
|
85
|
-
bytes?: array<Uint8Array.t>,
|
|
86
|
-
nulls?: Uint8Array.t,
|
|
87
|
-
}
|
|
88
|
-
|
|
89
89
|
@send
|
|
90
90
|
external registerEntityTable: (t, entitySpec) => registeredTable = "registerEntityTable"
|
|
91
91
|
|
|
@@ -99,7 +99,30 @@ external registerCheckpointsTable: (t, array<columnSpec>) => registeredTable =
|
|
|
99
99
|
external resume: (t, resumeInput) => promise<unit> = "resume"
|
|
100
100
|
|
|
101
101
|
@send
|
|
102
|
-
external
|
|
102
|
+
external beginStage: (t, ~table: int, ~rows: int) => Staging.begun = "beginStage"
|
|
103
|
+
|
|
104
|
+
@send
|
|
105
|
+
external growStage: (
|
|
106
|
+
t,
|
|
107
|
+
~handle: int,
|
|
108
|
+
~column: int,
|
|
109
|
+
~needed: int,
|
|
110
|
+
~stale: ArrayBuffer.t,
|
|
111
|
+
) => ArrayBuffer.t = "growStage"
|
|
112
|
+
|
|
113
|
+
@send
|
|
114
|
+
external commitStage: (t, ~handle: int, ~buffers: array<ArrayBuffer.t>) => unit = "commitStage"
|
|
115
|
+
|
|
116
|
+
@send
|
|
117
|
+
external abortStage: (t, ~handle: int, ~buffers: array<ArrayBuffer.t>) => unit = "abortStage"
|
|
118
|
+
|
|
119
|
+
let arena = (sink): Staging.arena => {
|
|
120
|
+
beginStage: (~table, ~rows) => sink->beginStage(~table, ~rows),
|
|
121
|
+
growStage: (~handle, ~column, ~needed, ~stale) =>
|
|
122
|
+
sink->growStage(~handle, ~column, ~needed, ~stale),
|
|
123
|
+
commitStage: (~handle, ~buffers) => sink->commitStage(~handle, ~buffers),
|
|
124
|
+
abortStage: (~handle, ~buffers) => sink->abortStage(~handle, ~buffers),
|
|
125
|
+
}
|
|
103
126
|
|
|
104
127
|
@send
|
|
105
128
|
external writeBatch: (t, ~entities: array<int>, ~checkpoints: Null.t<int>) => promise<unit> =
|
|
@@ -116,6 +139,8 @@ let historySchema = (): historySchema => {
|
|
|
116
139
|
checkpointsTable: InternalTable.Checkpoints.table.tableName,
|
|
117
140
|
checkpointChainIdColumn: (#chain_id: InternalTable.Checkpoints.field :> string),
|
|
118
141
|
checkpointBlockNumberColumn: (#block_number: InternalTable.Checkpoints.field :> string),
|
|
142
|
+
chainsTable: InternalTable.Chains.table.tableName,
|
|
143
|
+
chainsCheckpointIdColumn: (#checkpoint_id: InternalTable.Chains.field :> string),
|
|
119
144
|
}
|
|
120
145
|
|
|
121
146
|
let make = (~url, ~username, ~password, ~database, ~chainIdMode: ChainId.mode, ~onWarning) =>
|
|
@@ -131,45 +156,6 @@ let make = (~url, ~username, ~password, ~database, ~chainIdMode: ChainId.mode, ~
|
|
|
131
156
|
onWarning,
|
|
132
157
|
)
|
|
133
158
|
|
|
134
|
-
type kind =
|
|
135
|
-
| @as(0) F64
|
|
136
|
-
| @as(1) U64
|
|
137
|
-
| @as(2) I64
|
|
138
|
-
| @as(3) Text
|
|
139
|
-
| @as(4) Bytes
|
|
140
|
-
|
|
141
|
-
let kindOfOrdinal = ordinal =>
|
|
142
|
-
switch ordinal {
|
|
143
|
-
| 0 => F64
|
|
144
|
-
| 1 => U64
|
|
145
|
-
| 2 => I64
|
|
146
|
-
| 3 => Text
|
|
147
|
-
| 4 => Bytes
|
|
148
|
-
| unknown => JsError.throwWithMessage(`Unknown ClickHouse column kind ${unknown->Int.toString}`)
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
external asString: unknown => string = "%identity"
|
|
152
|
-
@val external toNumber: unknown => float = "Number"
|
|
153
|
-
@val external toBigInt: unknown => bigint = "BigInt"
|
|
154
|
-
@val external stringOf: unknown => string = "String"
|
|
155
|
-
|
|
156
|
-
// No column holds NaN or Infinity, but nothing downstream refuses them well: a
|
|
157
|
-
// list renders one as `null` on the way into JSON text, which the column then
|
|
158
|
-
// refuses for a reason naming `null` rather than what the handler wrote, and a
|
|
159
|
-
// scalar Float64 takes the raw bytes and stores a value no reader can render.
|
|
160
|
-
// Refused here instead, where the value is still itself, with one message for
|
|
161
|
-
// both shapes.
|
|
162
|
-
%%private(
|
|
163
|
-
let finiteOrThrow = (number: float, ~column) =>
|
|
164
|
-
if number->Float.isFinite {
|
|
165
|
-
number
|
|
166
|
-
} else {
|
|
167
|
-
JsError.throwWithMessage(
|
|
168
|
-
`${number->Float.toString} is not a finite number, so it cannot be stored in the \`${column}\` column. Store a finite number, or keep it out of the entity.`,
|
|
169
|
-
)
|
|
170
|
-
}
|
|
171
|
-
)
|
|
172
|
-
|
|
173
159
|
// Visits every node on the way into JSON text. A bigint would make
|
|
174
160
|
// `JSON.stringify` throw and fail the whole batch, so it travels as its digits.
|
|
175
161
|
%%private(
|
|
@@ -177,7 +163,7 @@ external asString: unknown => string = "%identity"
|
|
|
177
163
|
(_, value: JSON.t) =>
|
|
178
164
|
switch value->typeof {
|
|
179
165
|
| #bigint =>
|
|
180
|
-
value->(Utils.magic: JSON.t => unknown)->stringOf->(Utils.magic: string => JSON.t)
|
|
166
|
+
value->(Utils.magic: JSON.t => unknown)->Staging.stringOf->(Utils.magic: string => JSON.t)
|
|
181
167
|
// A Uint8Array inside a list column travels as its byte values, which the
|
|
182
168
|
// sink reads back into raw bytes.
|
|
183
169
|
| #object =>
|
|
@@ -190,148 +176,21 @@ external asString: unknown => string = "%identity"
|
|
|
190
176
|
| None => value
|
|
191
177
|
}
|
|
192
178
|
| #number =>
|
|
193
|
-
let _ = value->(Utils.magic: JSON.t => float)->finiteOrThrow(~column)
|
|
179
|
+
let _ = value->(Utils.magic: JSON.t => float)->Staging.finiteOrThrow(~column)
|
|
194
180
|
value
|
|
195
181
|
| _ => value
|
|
196
182
|
}
|
|
197
183
|
)
|
|
198
184
|
|
|
199
|
-
|
|
200
|
-
// with the column rather than once per column per batch.
|
|
201
|
-
type column = {name: string, kind: kind, isNullable: bool, replacer: JSON.replacer}
|
|
202
|
-
type table = {handle: int, name: string, columns: array<column>}
|
|
185
|
+
type table = {handle: int, name: string, columns: array<Staging.column>}
|
|
203
186
|
|
|
204
187
|
let makeTable = (~name, {handle, names, kinds, nullable}: registeredTable) => {
|
|
205
188
|
handle,
|
|
206
189
|
name,
|
|
207
|
-
columns: names->Array.mapWithIndex((name, index) => {
|
|
190
|
+
columns: names->Array.mapWithIndex((name, index): Staging.column => {
|
|
208
191
|
name,
|
|
209
|
-
kind: kinds->Array.getUnsafe(index)->kindOfOrdinal,
|
|
192
|
+
kind: kinds->Array.getUnsafe(index)->Staging.kindOfOrdinal,
|
|
210
193
|
isNullable: nullable->Array.getUnsafe(index),
|
|
211
194
|
replacer: Replacer(jsonSafe(~column=name)),
|
|
212
195
|
}),
|
|
213
196
|
}
|
|
214
|
-
|
|
215
|
-
let toText = (value: unknown, ~replacer) =>
|
|
216
|
-
switch value->typeof {
|
|
217
|
-
| #string => value->asString
|
|
218
|
-
| #bigint => value->stringOf
|
|
219
|
-
| _ => value->(Utils.magic: unknown => JSON.t)->JSON.stringify(~replacer)
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
type builder = {
|
|
223
|
-
name: string,
|
|
224
|
-
kind: kind,
|
|
225
|
-
isNullable: bool,
|
|
226
|
-
floats: Float64Array.t,
|
|
227
|
-
unsigned: BigUint64Array.t,
|
|
228
|
-
signed: BigInt64Array.t,
|
|
229
|
-
texts: array<string>,
|
|
230
|
-
bytes: array<Uint8Array.t>,
|
|
231
|
-
replacer: JSON.replacer,
|
|
232
|
-
mutable nulls: option<Uint8Array.t>,
|
|
233
|
-
rows: int,
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
%%private(let noFloats = Float64Array.fromLength(0))
|
|
237
|
-
%%private(let noUnsigned = BigUint64Array.fromLength(0))
|
|
238
|
-
%%private(let noSigned = BigInt64Array.fromLength(0))
|
|
239
|
-
%%private(let noBytes = Uint8Array.fromLength(0))
|
|
240
|
-
|
|
241
|
-
let makeBuilder = ({name, kind, isNullable, replacer}: column, ~rows) => {
|
|
242
|
-
name,
|
|
243
|
-
kind,
|
|
244
|
-
isNullable,
|
|
245
|
-
floats: kind === F64 ? Float64Array.fromLength(rows) : noFloats,
|
|
246
|
-
unsigned: kind === U64 ? BigUint64Array.fromLength(rows) : noUnsigned,
|
|
247
|
-
signed: kind === I64 ? BigInt64Array.fromLength(rows) : noSigned,
|
|
248
|
-
texts: kind === Text ? Array.make(~length=rows, "") : [],
|
|
249
|
-
bytes: kind === Bytes ? Array.make(~length=rows, noBytes) : [],
|
|
250
|
-
replacer,
|
|
251
|
-
nulls: None,
|
|
252
|
-
rows,
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
let markNull = (builder, ~row) => {
|
|
256
|
-
let nulls = switch builder.nulls {
|
|
257
|
-
| Some(nulls) => nulls
|
|
258
|
-
| None =>
|
|
259
|
-
let nulls = Uint8Array.fromLength(builder.rows)
|
|
260
|
-
builder.nulls = Some(nulls)
|
|
261
|
-
nulls
|
|
262
|
-
}
|
|
263
|
-
nulls->TypedArray.set(row, 1)
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
// RowBinary carries the raw integer, so a value the column cannot hold is not
|
|
267
|
-
// rejected anywhere downstream — and a typed array reduces it modulo 2^64 on
|
|
268
|
-
// the way in rather than refusing it, which would leave no trace at all.
|
|
269
|
-
%%private(
|
|
270
|
-
let checkedBigInt = (value: unknown, ~builder, ~min, ~max) => {
|
|
271
|
-
let value = value->toBigInt
|
|
272
|
-
if value < min || value > max {
|
|
273
|
-
JsError.throwWithMessage(
|
|
274
|
-
`${value->BigInt.toString} is out of range for the \`${builder.name}\` column`,
|
|
275
|
-
)
|
|
276
|
-
}
|
|
277
|
-
value
|
|
278
|
-
}
|
|
279
|
-
)
|
|
280
|
-
|
|
281
|
-
%%private(
|
|
282
|
-
let writePresent = (builder, ~row, value: unknown) =>
|
|
283
|
-
switch builder.kind {
|
|
284
|
-
| F64 =>
|
|
285
|
-
builder.floats->TypedArray.set(row, value->toNumber->finiteOrThrow(~column=builder.name))
|
|
286
|
-
| U64 =>
|
|
287
|
-
builder.unsigned->TypedArray.set(
|
|
288
|
-
row,
|
|
289
|
-
value->checkedBigInt(~builder, ~min=0n, ~max=18446744073709551615n),
|
|
290
|
-
)
|
|
291
|
-
| I64 =>
|
|
292
|
-
builder.signed->TypedArray.set(
|
|
293
|
-
row,
|
|
294
|
-
value->checkedBigInt(~builder, ~min=-9223372036854775808n, ~max=9223372036854775807n),
|
|
295
|
-
)
|
|
296
|
-
| Text => builder.texts->Array.setUnsafe(row, value->toText(~replacer=builder.replacer))
|
|
297
|
-
| Bytes => builder.bytes->Array.setUnsafe(row, value->(Utils.magic: unknown => Uint8Array.t))
|
|
298
|
-
}
|
|
299
|
-
)
|
|
300
|
-
|
|
301
|
-
%%private(let isAbsent = (value: unknown) => value === %raw(`undefined`) || value === %raw(`null`))
|
|
302
|
-
|
|
303
|
-
// Writes one value of a row the handler set. `undefined`/`null` marks the row's
|
|
304
|
-
// null bit, which a column that accepts NULL stores as such — and which one that
|
|
305
|
-
// does not has no way to store: RowBinary carries no "absent", so the row would
|
|
306
|
-
// land holding the type's zero, a value the handler never chose and that nothing
|
|
307
|
-
// downstream could tell from one it did.
|
|
308
|
-
let writeValue = (builder, ~row, value: unknown) =>
|
|
309
|
-
if value->isAbsent {
|
|
310
|
-
if builder.isNullable {
|
|
311
|
-
builder->markNull(~row)
|
|
312
|
-
} else {
|
|
313
|
-
JsError.throwWithMessage(
|
|
314
|
-
`No value for the \`${builder.name}\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`,
|
|
315
|
-
)
|
|
316
|
-
}
|
|
317
|
-
} else {
|
|
318
|
-
builder->writePresent(~row, value)
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
let writeDeletedValue = (builder, ~row, value: unknown) =>
|
|
322
|
-
if value->isAbsent {
|
|
323
|
-
builder->markNull(~row)
|
|
324
|
-
} else {
|
|
325
|
-
builder->writePresent(~row, value)
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
let builderPayload = (builder): columnValuesInput => {
|
|
329
|
-
let base = {nulls: ?builder.nulls}
|
|
330
|
-
switch builder.kind {
|
|
331
|
-
| F64 => {...base, numbers: builder.floats}
|
|
332
|
-
| U64 => {...base, unsigned64: builder.unsigned}
|
|
333
|
-
| I64 => {...base, signed64: builder.signed}
|
|
334
|
-
| Text => {...base, texts: builder.texts}
|
|
335
|
-
| Bytes => {...base, bytes: builder.bytes}
|
|
336
|
-
}
|
|
337
|
-
}
|
|
@@ -3,10 +3,22 @@
|
|
|
3
3
|
import * as Core from "../Core.res.mjs";
|
|
4
4
|
import * as Table from "../db/Table.res.mjs";
|
|
5
5
|
import * as Utils from "../Utils.res.mjs";
|
|
6
|
-
import * as
|
|
6
|
+
import * as Staging from "../Staging.res.mjs";
|
|
7
7
|
import * as EntityHistory from "../db/EntityHistory.res.mjs";
|
|
8
8
|
import * as InternalTable from "../db/InternalTable.res.mjs";
|
|
9
|
-
|
|
9
|
+
|
|
10
|
+
function arena(sink) {
|
|
11
|
+
return {
|
|
12
|
+
beginStage: (table, rows) => sink.beginStage(table, rows),
|
|
13
|
+
growStage: (handle, column, needed, stale) => sink.growStage(handle, column, needed, stale),
|
|
14
|
+
commitStage: (handle, buffers) => {
|
|
15
|
+
sink.commitStage(handle, buffers);
|
|
16
|
+
},
|
|
17
|
+
abortStage: (handle, buffers) => {
|
|
18
|
+
sink.abortStage(handle, buffers);
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
}
|
|
10
22
|
|
|
11
23
|
function historySchema() {
|
|
12
24
|
return {
|
|
@@ -17,7 +29,9 @@ function historySchema() {
|
|
|
17
29
|
setVariant: "SET",
|
|
18
30
|
checkpointsTable: InternalTable.Checkpoints.table.tableName,
|
|
19
31
|
checkpointChainIdColumn: "chain_id",
|
|
20
|
-
checkpointBlockNumberColumn: "block_number"
|
|
32
|
+
checkpointBlockNumberColumn: "block_number",
|
|
33
|
+
chainsTable: InternalTable.Chains.table.tableName,
|
|
34
|
+
chainsCheckpointIdColumn: "checkpoint_id"
|
|
21
35
|
};
|
|
22
36
|
}
|
|
23
37
|
|
|
@@ -32,31 +46,6 @@ function make(url, username, password, database, chainIdMode, onWarning) {
|
|
|
32
46
|
}, onWarning);
|
|
33
47
|
}
|
|
34
48
|
|
|
35
|
-
function kindOfOrdinal(ordinal) {
|
|
36
|
-
switch (ordinal) {
|
|
37
|
-
case 0 :
|
|
38
|
-
return 0;
|
|
39
|
-
case 1 :
|
|
40
|
-
return 1;
|
|
41
|
-
case 2 :
|
|
42
|
-
return 2;
|
|
43
|
-
case 3 :
|
|
44
|
-
return 3;
|
|
45
|
-
case 4 :
|
|
46
|
-
return 4;
|
|
47
|
-
default:
|
|
48
|
-
return Stdlib_JsError.throwWithMessage(`Unknown ClickHouse column kind ` + ordinal.toString());
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
function finiteOrThrow(number, column) {
|
|
53
|
-
if (isFinite(number)) {
|
|
54
|
-
return number;
|
|
55
|
-
} else {
|
|
56
|
-
return Stdlib_JsError.throwWithMessage(number.toString() + ` is not a finite number, so it cannot be stored in the \`` + column + `\` column. Store a finite number, or keep it out of the entity.`);
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
49
|
function jsonSafe(column) {
|
|
61
50
|
return (param, value) => {
|
|
62
51
|
let match = typeof value;
|
|
@@ -65,7 +54,7 @@ function jsonSafe(column) {
|
|
|
65
54
|
}
|
|
66
55
|
if (match !== "object") {
|
|
67
56
|
if (match === "number") {
|
|
68
|
-
finiteOrThrow(value, column);
|
|
57
|
+
Staging.finiteOrThrow(value, column);
|
|
69
58
|
return value;
|
|
70
59
|
} else {
|
|
71
60
|
return value;
|
|
@@ -88,159 +77,17 @@ function makeTable(name, param) {
|
|
|
88
77
|
name: name,
|
|
89
78
|
columns: param.names.map((name, index) => ({
|
|
90
79
|
name: name,
|
|
91
|
-
kind: kindOfOrdinal(kinds[index]),
|
|
80
|
+
kind: Staging.kindOfOrdinal(kinds[index]),
|
|
92
81
|
isNullable: nullable[index],
|
|
93
82
|
replacer: jsonSafe(name)
|
|
94
83
|
}))
|
|
95
84
|
};
|
|
96
85
|
}
|
|
97
86
|
|
|
98
|
-
function toText(value, replacer) {
|
|
99
|
-
let match = typeof value;
|
|
100
|
-
if (match === "bigint") {
|
|
101
|
-
return String(value);
|
|
102
|
-
} else if (match === "string") {
|
|
103
|
-
return value;
|
|
104
|
-
} else {
|
|
105
|
-
return JSON.stringify(value, replacer);
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
let noFloats = new Float64Array(0);
|
|
110
|
-
|
|
111
|
-
let noUnsigned = new BigUint64Array(0);
|
|
112
|
-
|
|
113
|
-
let noSigned = new BigInt64Array(0);
|
|
114
|
-
|
|
115
|
-
let noBytes = new Uint8Array(0);
|
|
116
|
-
|
|
117
|
-
function makeBuilder(param, rows) {
|
|
118
|
-
let kind = param.kind;
|
|
119
|
-
return {
|
|
120
|
-
name: param.name,
|
|
121
|
-
kind: kind,
|
|
122
|
-
isNullable: param.isNullable,
|
|
123
|
-
floats: kind === 0 ? new Float64Array(rows) : noFloats,
|
|
124
|
-
unsigned: kind === 1 ? new BigUint64Array(rows) : noUnsigned,
|
|
125
|
-
signed: kind === 2 ? new BigInt64Array(rows) : noSigned,
|
|
126
|
-
texts: kind === 3 ? Stdlib_Array.make(rows, "") : [],
|
|
127
|
-
bytes: kind === 4 ? Stdlib_Array.make(rows, noBytes) : [],
|
|
128
|
-
replacer: param.replacer,
|
|
129
|
-
nulls: undefined,
|
|
130
|
-
rows: rows
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
function markNull(builder, row) {
|
|
135
|
-
let nulls = builder.nulls;
|
|
136
|
-
let nulls$1;
|
|
137
|
-
if (nulls !== undefined) {
|
|
138
|
-
nulls$1 = nulls;
|
|
139
|
-
} else {
|
|
140
|
-
let nulls$2 = new Uint8Array(builder.rows);
|
|
141
|
-
builder.nulls = nulls$2;
|
|
142
|
-
nulls$1 = nulls$2;
|
|
143
|
-
}
|
|
144
|
-
nulls$1[row] = 1;
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
function checkedBigInt(value, builder, min, max) {
|
|
148
|
-
let value$1 = BigInt(value);
|
|
149
|
-
if (value$1 < min || value$1 > max) {
|
|
150
|
-
Stdlib_JsError.throwWithMessage(value$1.toString() + ` is out of range for the \`` + builder.name + `\` column`);
|
|
151
|
-
}
|
|
152
|
-
return value$1;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
function writePresent(builder, row, value) {
|
|
156
|
-
let match = builder.kind;
|
|
157
|
-
switch (match) {
|
|
158
|
-
case 0 :
|
|
159
|
-
builder.floats[row] = finiteOrThrow(Number(value), builder.name);
|
|
160
|
-
return;
|
|
161
|
-
case 1 :
|
|
162
|
-
builder.unsigned[row] = checkedBigInt(value, builder, 0n, 18446744073709551615n);
|
|
163
|
-
return;
|
|
164
|
-
case 2 :
|
|
165
|
-
builder.signed[row] = checkedBigInt(value, builder, -9223372036854775808n, 9223372036854775807n);
|
|
166
|
-
return;
|
|
167
|
-
case 3 :
|
|
168
|
-
builder.texts[row] = toText(value, builder.replacer);
|
|
169
|
-
return;
|
|
170
|
-
case 4 :
|
|
171
|
-
builder.bytes[row] = value;
|
|
172
|
-
return;
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
function isAbsent(value) {
|
|
177
|
-
if (value === undefined) {
|
|
178
|
-
return true;
|
|
179
|
-
} else {
|
|
180
|
-
return value === null;
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
function writeValue(builder, row, value) {
|
|
185
|
-
if (isAbsent(value)) {
|
|
186
|
-
if (builder.isNullable) {
|
|
187
|
-
return markNull(builder, row);
|
|
188
|
-
} else {
|
|
189
|
-
return Stdlib_JsError.throwWithMessage(`No value for the \`` + builder.name + `\` column, which is not nullable. Set the field before saving the entity, or make it optional in the schema.`);
|
|
190
|
-
}
|
|
191
|
-
} else {
|
|
192
|
-
return writePresent(builder, row, value);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
function writeDeletedValue(builder, row, value) {
|
|
197
|
-
if (isAbsent(value)) {
|
|
198
|
-
return markNull(builder, row);
|
|
199
|
-
} else {
|
|
200
|
-
return writePresent(builder, row, value);
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
function builderPayload(builder) {
|
|
205
|
-
let base_nulls = builder.nulls;
|
|
206
|
-
let base = {
|
|
207
|
-
nulls: base_nulls
|
|
208
|
-
};
|
|
209
|
-
let match = builder.kind;
|
|
210
|
-
switch (match) {
|
|
211
|
-
case 0 :
|
|
212
|
-
let newrecord = {...base};
|
|
213
|
-
newrecord.numbers = builder.floats;
|
|
214
|
-
return newrecord;
|
|
215
|
-
case 1 :
|
|
216
|
-
let newrecord$1 = {...base};
|
|
217
|
-
newrecord$1.unsigned64 = builder.unsigned;
|
|
218
|
-
return newrecord$1;
|
|
219
|
-
case 2 :
|
|
220
|
-
let newrecord$2 = {...base};
|
|
221
|
-
newrecord$2.signed64 = builder.signed;
|
|
222
|
-
return newrecord$2;
|
|
223
|
-
case 3 :
|
|
224
|
-
let newrecord$3 = {...base};
|
|
225
|
-
newrecord$3.texts = builder.texts;
|
|
226
|
-
return newrecord$3;
|
|
227
|
-
case 4 :
|
|
228
|
-
let newrecord$4 = {...base};
|
|
229
|
-
newrecord$4.bytes = builder.bytes;
|
|
230
|
-
return newrecord$4;
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
|
|
234
87
|
export {
|
|
88
|
+
arena,
|
|
235
89
|
historySchema,
|
|
236
90
|
make,
|
|
237
|
-
kindOfOrdinal,
|
|
238
91
|
makeTable,
|
|
239
|
-
toText,
|
|
240
|
-
makeBuilder,
|
|
241
|
-
markNull,
|
|
242
|
-
writeValue,
|
|
243
|
-
writeDeletedValue,
|
|
244
|
-
builderPayload,
|
|
245
92
|
}
|
|
246
|
-
/*
|
|
93
|
+
/* Core Not a pure module */
|