envio 3.15.0 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -34
- package/evm.schema.json +11 -2
- package/fuel.schema.json +10 -1
- package/index.d.ts +6 -0
- package/licenses/README.md +1 -1
- package/package.json +8 -8
- package/src/Batch.res.mjs +5 -5
- package/src/BatchProcessing.res +1 -0
- package/src/BatchProcessing.res.mjs +1 -0
- package/src/ChainState.res +48 -7
- package/src/ChainState.res.mjs +27 -11
- package/src/Clock.res +57 -0
- package/src/Clock.res.mjs +72 -0
- package/src/Config.res +10 -3
- package/src/Config.res.mjs +4 -2
- package/src/Core.res +4 -40
- package/src/EffectState.res +10 -2
- package/src/EffectState.res.mjs +8 -4
- package/src/EffectState.resi +3 -1
- package/src/EntityId.res +2 -7
- package/src/EntityId.res.mjs +1 -8
- package/src/EntityTables.res +4 -1
- package/src/EntityTables.res.mjs +1 -1
- package/src/FetchState.res +157 -69
- package/src/FetchState.res.mjs +144 -49
- package/src/FinalizeBackfill.res +2 -4
- package/src/FinalizeBackfill.res.mjs +3 -4
- package/src/InMemoryTable.res +17 -16
- package/src/InMemoryTable.res.mjs +14 -15
- package/src/IndexerState.res +28 -4
- package/src/IndexerState.res.mjs +19 -6
- package/src/IndexerState.resi +4 -0
- package/src/Internal.res +0 -49
- package/src/Internal.res.mjs +1 -80
- package/src/LoadLayer.res +14 -8
- package/src/LoadLayer.res.mjs +8 -6
- package/src/Main.res.mjs +1 -1
- package/src/Persistence.res +3 -23
- package/src/Persistence.res.mjs +3 -3
- package/src/Rollback.res +0 -1
- package/src/Rollback.res.mjs +1 -1
- package/src/SafeCheckpointTracking.res +4 -4
- package/src/SafeCheckpointTracking.res.mjs +1 -1
- package/src/StagedRows.res +106 -0
- package/src/StagedRows.res.mjs +102 -0
- package/src/Staging.res +32 -14
- package/src/Staging.res.mjs +17 -20
- package/src/Storage.res +277 -948
- package/src/Storage.res.mjs +215 -707
- package/src/Supervisor.res +75 -56
- package/src/Supervisor.res.mjs +55 -26
- package/src/TestIndexer.res +22 -27
- package/src/TestIndexer.res.mjs +20 -17
- package/src/Throttler.res +15 -8
- package/src/Throttler.res.mjs +9 -6
- package/src/Throttler.resi +1 -1
- package/src/Utils.res +0 -6
- package/src/Utils.res.mjs +0 -5
- package/src/Writing.res +0 -1
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +0 -322
- package/src/bindings/ClickHouse.res.mjs +0 -338
- package/src/bindings/StorageClient.res +107 -174
- package/src/bindings/StorageClient.res.mjs +19 -8
- package/src/bindings/Vitest.res +0 -26
- package/src/bindings/Vitest.res.mjs +0 -3
- package/src/db/Table.res +24 -150
- package/src/db/Table.res.mjs +38 -272
- package/src/sources/ChainSources.res +2 -0
- package/src/sources/ChainSources.res.mjs +2 -2
- package/src/sources/EvmChain.res +3 -0
- package/src/sources/EvmChain.res.mjs +4 -2
- package/src/sources/EvmHyperSyncSource.res +2 -2
- package/src/sources/EvmHyperSyncSource.res.mjs +1 -1
- package/src/sources/EvmRpcClient.res +13 -3
- package/src/sources/EvmRpcClient.res.mjs +4 -3
- package/src/sources/EvmRpcWs.res +4 -3
- package/src/sources/EvmRpcWs.res.mjs +2 -2
- package/src/sources/HeightFeed.res +31 -10
- package/src/sources/HeightFeed.res.mjs +10 -6
- package/src/sources/HeightStream.res +32 -17
- package/src/sources/HeightStream.res.mjs +10 -10
- package/src/sources/HyperSyncSSE.res +4 -6
- package/src/sources/HyperSyncSSE.res.mjs +2 -2
- package/src/sources/RpcSource.res +10 -1
- package/src/sources/RpcSource.res.mjs +4 -3
- package/src/sources/Source.res +3 -0
- package/src/sources/SourceManager.res +84 -77
- package/src/sources/SourceManager.res.mjs +17 -20
- package/src/sources/SourceManager.resi +3 -4
- package/src/sources/StartBlockResolver.res +1 -0
- package/src/sources/StartBlockResolver.res.mjs +3 -2
- package/src/sources/SvmHyperSyncSource.res +2 -2
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
- package/svm.schema.json +11 -2
- package/src/PgWriting.res +0 -36
- package/src/PgWriting.res.mjs +0 -34
- package/src/db/IndexDefinition.res +0 -47
- package/src/db/IndexDefinition.res.mjs +0 -67
|
@@ -1,340 +1,8 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Env from "../Env.res.mjs";
|
|
4
|
-
import * as Sql from "../db/Sql.res.mjs";
|
|
5
|
-
import * as Table from "../db/Table.res.mjs";
|
|
6
4
|
import * as Utils from "../Utils.res.mjs";
|
|
7
|
-
import * as ChainId from "../ChainId.res.mjs";
|
|
8
|
-
import * as Staging from "../Staging.res.mjs";
|
|
9
|
-
import * as Internal from "../Internal.res.mjs";
|
|
10
|
-
import * as Persistence from "../Persistence.res.mjs";
|
|
11
|
-
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
12
|
-
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
13
|
-
import * as EntityHistory from "../db/EntityHistory.res.mjs";
|
|
14
|
-
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
15
5
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
16
|
-
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
17
|
-
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
18
|
-
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
19
|
-
|
|
20
|
-
function fieldTypeName(fieldType) {
|
|
21
|
-
if (typeof fieldType !== "object") {
|
|
22
|
-
switch (fieldType) {
|
|
23
|
-
case "String" :
|
|
24
|
-
return "String";
|
|
25
|
-
case "Boolean" :
|
|
26
|
-
return "Boolean";
|
|
27
|
-
case "Uint32" :
|
|
28
|
-
return "Uint32";
|
|
29
|
-
case "UInt52" :
|
|
30
|
-
return "UInt52";
|
|
31
|
-
case "SmallInt" :
|
|
32
|
-
return Stdlib_JsError.throwWithMessage("ClickHouse doesn't support the internal SmallInt column type");
|
|
33
|
-
case "Bytea" :
|
|
34
|
-
return "Bytea";
|
|
35
|
-
case "UInt64" :
|
|
36
|
-
return "UInt64";
|
|
37
|
-
case "Int32" :
|
|
38
|
-
return "Int32";
|
|
39
|
-
case "ChainId" :
|
|
40
|
-
return "ChainId";
|
|
41
|
-
case "Number" :
|
|
42
|
-
return "Number";
|
|
43
|
-
case "Serial" :
|
|
44
|
-
return "Serial";
|
|
45
|
-
case "BigSerial" :
|
|
46
|
-
return "BigSerial";
|
|
47
|
-
case "Json" :
|
|
48
|
-
return "Json";
|
|
49
|
-
case "Date" :
|
|
50
|
-
return "Date";
|
|
51
|
-
}
|
|
52
|
-
} else {
|
|
53
|
-
switch (fieldType.type) {
|
|
54
|
-
case "BigInt" :
|
|
55
|
-
return "BigInt";
|
|
56
|
-
case "BigDecimal" :
|
|
57
|
-
return "BigDecimal";
|
|
58
|
-
case "Enum" :
|
|
59
|
-
return "Enum";
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
function column(name, rowName, fieldName, fieldType, isNullableOpt, isArrayOpt) {
|
|
65
|
-
let isNullable = isNullableOpt !== undefined ? isNullableOpt : false;
|
|
66
|
-
let isArray = isArrayOpt !== undefined ? isArrayOpt : false;
|
|
67
|
-
let tmp;
|
|
68
|
-
if (typeof fieldType !== "object") {
|
|
69
|
-
tmp = undefined;
|
|
70
|
-
} else {
|
|
71
|
-
switch (fieldType.type) {
|
|
72
|
-
case "BigInt" :
|
|
73
|
-
tmp = fieldType.precision;
|
|
74
|
-
break;
|
|
75
|
-
case "BigDecimal" :
|
|
76
|
-
tmp = Stdlib_Option.map(fieldType.config, param => param[0]);
|
|
77
|
-
break;
|
|
78
|
-
default:
|
|
79
|
-
tmp = undefined;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
let tmp$1;
|
|
83
|
-
tmp$1 = typeof fieldType !== "object" || fieldType.type !== "BigDecimal" ? undefined : Stdlib_Option.map(fieldType.config, param => param[1]);
|
|
84
|
-
let tmp$2;
|
|
85
|
-
tmp$2 = typeof fieldType !== "object" || fieldType.type !== "Enum" ? undefined : fieldType.config.variants.map(variant => variant);
|
|
86
|
-
return {
|
|
87
|
-
name: name,
|
|
88
|
-
rowName: rowName,
|
|
89
|
-
fieldName: fieldName,
|
|
90
|
-
fieldType: fieldTypeName(fieldType),
|
|
91
|
-
isNullable: isNullable,
|
|
92
|
-
isArray: isArray,
|
|
93
|
-
precision: tmp,
|
|
94
|
-
scale: tmp$1,
|
|
95
|
-
enumVariants: tmp$2
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
function fieldColumn(field) {
|
|
100
|
-
return column(Table.getClickHouseDbFieldName(field), Table.getPgDbFieldName(field), field.fieldName, field.fieldType, field.isNullable, field.isArray);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
function entityTable(entityConfig) {
|
|
104
|
-
let options = entityConfig.storage.clickhouseOptions;
|
|
105
|
-
return {
|
|
106
|
-
name: entityConfig.name,
|
|
107
|
-
historyTable: EntityHistory.historyTableName(entityConfig.name, entityConfig.index),
|
|
108
|
-
columns: Stdlib_Array.filterMap(entityConfig.table.fields, field => {
|
|
109
|
-
if (field.TAG === "Field") {
|
|
110
|
-
return fieldColumn(field._0);
|
|
111
|
-
}
|
|
112
|
-
}),
|
|
113
|
-
chainIdColumn: Stdlib_Option.map(Table.getChainIdField(entityConfig.table), Table.getClickHouseDbFieldName),
|
|
114
|
-
partitionBy: Stdlib_Option.flatMap(options, options => options.partitionBy),
|
|
115
|
-
orderBy: Stdlib_Option.flatMap(options, options => options.orderBy),
|
|
116
|
-
ttl: Stdlib_Option.flatMap(options, options => options.ttl),
|
|
117
|
-
skippingIndexes: Stdlib_Option.flatMap(options, options => Stdlib_Option.map(options.skippingIndexes, indexes => indexes.map(index => ({
|
|
118
|
-
name: index.name,
|
|
119
|
-
expr: index.expr,
|
|
120
|
-
indexType: index.type,
|
|
121
|
-
granularity: index.granularity
|
|
122
|
-
}))))
|
|
123
|
-
};
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
function jsonSafe(column) {
|
|
127
|
-
return (param, value) => {
|
|
128
|
-
let match = typeof value;
|
|
129
|
-
if (match === "bigint") {
|
|
130
|
-
return String(value);
|
|
131
|
-
}
|
|
132
|
-
if (match !== "object") {
|
|
133
|
-
if (match === "number") {
|
|
134
|
-
Staging.finiteOrThrow(value, column);
|
|
135
|
-
return value;
|
|
136
|
-
} else {
|
|
137
|
-
return value;
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
let bytes = Utils.Bytes.asUint8Array(value);
|
|
141
|
-
if (bytes !== undefined) {
|
|
142
|
-
return Array.from(bytes);
|
|
143
|
-
} else {
|
|
144
|
-
return value;
|
|
145
|
-
}
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function makeTable(handle, name, param) {
|
|
150
|
-
let nullable = param.nullable;
|
|
151
|
-
let kinds = param.kinds;
|
|
152
|
-
return {
|
|
153
|
-
handle: handle,
|
|
154
|
-
name: name,
|
|
155
|
-
columns: param.names.map((name, index) => ({
|
|
156
|
-
name: name,
|
|
157
|
-
kind: Staging.kindOfOrdinal(kinds[index]),
|
|
158
|
-
isNullable: nullable[index],
|
|
159
|
-
replacer: jsonSafe(name)
|
|
160
|
-
})),
|
|
161
|
-
historyCells: {}
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
function documentText(value) {
|
|
166
|
-
if (value == null) {
|
|
167
|
-
return "null";
|
|
168
|
-
} else {
|
|
169
|
-
return Sql.canonicalDocument(value);
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
function compileSerializer(schema) {
|
|
174
|
-
return S$RescriptSchema.compile(schema, "Output", "Json", "Sync", false);
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
function serializerOf(f) {
|
|
178
|
-
let shaped = baseSchema => {
|
|
179
|
-
if (f.isNullable) {
|
|
180
|
-
return Utils.Schema.nullTolerant(baseSchema);
|
|
181
|
-
} else if (f.isArray) {
|
|
182
|
-
return S$RescriptSchema.array(baseSchema);
|
|
183
|
-
} else {
|
|
184
|
-
return baseSchema;
|
|
185
|
-
}
|
|
186
|
-
};
|
|
187
|
-
let match = f.fieldType;
|
|
188
|
-
if (typeof match === "object") {
|
|
189
|
-
return compileSerializer(f.fieldSchema);
|
|
190
|
-
}
|
|
191
|
-
switch (match) {
|
|
192
|
-
case "Bytea" :
|
|
193
|
-
return compileSerializer(shaped(S$RescriptSchema.json(false)));
|
|
194
|
-
case "ChainId" :
|
|
195
|
-
return compileSerializer(ChainId.schema);
|
|
196
|
-
case "Json" :
|
|
197
|
-
if (f.isArray) {
|
|
198
|
-
return value => {
|
|
199
|
-
if (value == null) {
|
|
200
|
-
return value;
|
|
201
|
-
} else {
|
|
202
|
-
return value.map(documentText);
|
|
203
|
-
}
|
|
204
|
-
};
|
|
205
|
-
} else {
|
|
206
|
-
return documentText;
|
|
207
|
-
}
|
|
208
|
-
case "Date" :
|
|
209
|
-
return compileSerializer(shaped(Utils.Schema.clickHouseDate));
|
|
210
|
-
default:
|
|
211
|
-
return compileSerializer(f.fieldSchema);
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
let absent = undefined;
|
|
216
|
-
|
|
217
|
-
function makeHistoryCells(table, entityConfig, scope) {
|
|
218
|
-
let sourceTable = entityConfig.table;
|
|
219
|
-
let match = Table.getChainIdField(sourceTable);
|
|
220
|
-
let match$1 = Internal.chainScopeChainId(scope);
|
|
221
|
-
let chainIdTag = match !== undefined && match$1 !== undefined ? [
|
|
222
|
-
Table.getClickHouseDbFieldName(match),
|
|
223
|
-
Primitive_option.valFromOption(match$1)
|
|
224
|
-
] : undefined;
|
|
225
|
-
let fieldsByColumn = {};
|
|
226
|
-
sourceTable.fields.forEach(field => {
|
|
227
|
-
if (field.TAG !== "Field") {
|
|
228
|
-
return;
|
|
229
|
-
}
|
|
230
|
-
let f = field._0;
|
|
231
|
-
fieldsByColumn[Table.getClickHouseDbFieldName(f)] = f;
|
|
232
|
-
});
|
|
233
|
-
let setCells = [];
|
|
234
|
-
let deleteCells = [];
|
|
235
|
-
let registeredNames = new Set(table.columns.map(column => column.name));
|
|
236
|
-
table.columns.forEach(param => {
|
|
237
|
-
let name = param.name;
|
|
238
|
-
let match;
|
|
239
|
-
if (name === EntityHistory.checkpointIdFieldName) {
|
|
240
|
-
let cell = change => change.checkpointId;
|
|
241
|
-
match = [
|
|
242
|
-
cell,
|
|
243
|
-
cell
|
|
244
|
-
];
|
|
245
|
-
} else if (name === EntityHistory.changeFieldName) {
|
|
246
|
-
match = [
|
|
247
|
-
param => "SET",
|
|
248
|
-
param => "DELETE"
|
|
249
|
-
];
|
|
250
|
-
} else {
|
|
251
|
-
let match$1 = fieldsByColumn[name];
|
|
252
|
-
let exit = 0;
|
|
253
|
-
if (chainIdTag !== undefined && chainIdTag[0] === name) {
|
|
254
|
-
let chainId = chainIdTag[1];
|
|
255
|
-
match = [
|
|
256
|
-
param => chainId,
|
|
257
|
-
param => chainId
|
|
258
|
-
];
|
|
259
|
-
} else {
|
|
260
|
-
exit = 1;
|
|
261
|
-
}
|
|
262
|
-
if (exit === 1) {
|
|
263
|
-
if (match$1 !== undefined) {
|
|
264
|
-
let serialize = serializerOf(match$1);
|
|
265
|
-
let apiName = Table.getApiFieldName(match$1);
|
|
266
|
-
if (apiName === Table.idFieldName) {
|
|
267
|
-
let cell$1 = change => serialize(change.entityId);
|
|
268
|
-
match = [
|
|
269
|
-
cell$1,
|
|
270
|
-
cell$1
|
|
271
|
-
];
|
|
272
|
-
} else {
|
|
273
|
-
match = [
|
|
274
|
-
change => serialize(change.entity[apiName]),
|
|
275
|
-
param => absent
|
|
276
|
-
];
|
|
277
|
-
}
|
|
278
|
-
} else {
|
|
279
|
-
match = Stdlib_JsError.throwWithMessage(`ClickHouse table "` + table.name + `" registered a column "` + name + `" that entity "` + entityConfig.name + `" has no value for`);
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
setCells.push(match[0]);
|
|
284
|
-
deleteCells.push(match[1]);
|
|
285
|
-
});
|
|
286
|
-
Stdlib_Dict.forEachWithKey(fieldsByColumn, (f, column) => {
|
|
287
|
-
if (!registeredNames.has(column)) {
|
|
288
|
-
return Stdlib_JsError.throwWithMessage(`ClickHouse table "` + table.name + `" registered no column for field "` + f.fieldName + `" of entity "` + entityConfig.name + `"`);
|
|
289
|
-
}
|
|
290
|
-
});
|
|
291
|
-
return {
|
|
292
|
-
setCells: setCells,
|
|
293
|
-
deleteCells: deleteCells
|
|
294
|
-
};
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
function stageChangesOrThrow(arena, table, entityConfig, scope, changes) {
|
|
298
|
-
let stage = Staging.begin(arena, table.handle, changes.length, table.columns);
|
|
299
|
-
try {
|
|
300
|
-
let cacheKey = Internal.chainScopeToString(scope);
|
|
301
|
-
let cached = table.historyCells[cacheKey];
|
|
302
|
-
let cells;
|
|
303
|
-
if (cached !== undefined) {
|
|
304
|
-
cells = cached;
|
|
305
|
-
} else {
|
|
306
|
-
let cells$1 = makeHistoryCells(table, entityConfig, scope);
|
|
307
|
-
table.historyCells[cacheKey] = cells$1;
|
|
308
|
-
cells = cells$1;
|
|
309
|
-
}
|
|
310
|
-
let columns = Staging.columnCount(stage);
|
|
311
|
-
changes.forEach((change, row) => {
|
|
312
|
-
let match;
|
|
313
|
-
match = change.type === "SET" ? [
|
|
314
|
-
cells.setCells,
|
|
315
|
-
Staging.writeValue
|
|
316
|
-
] : [
|
|
317
|
-
cells.deleteCells,
|
|
318
|
-
Staging.writeDeletedValue
|
|
319
|
-
];
|
|
320
|
-
let write = match[1];
|
|
321
|
-
let cells$2 = match[0];
|
|
322
|
-
for (let column = 0; column < columns; ++column) {
|
|
323
|
-
write(stage, column, row, cells$2[column](change));
|
|
324
|
-
}
|
|
325
|
-
});
|
|
326
|
-
return Staging.commit(stage);
|
|
327
|
-
} catch (raw_exn) {
|
|
328
|
-
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
329
|
-
Staging.abort(stage);
|
|
330
|
-
throw {
|
|
331
|
-
RE_EXN_ID: Persistence.StorageError,
|
|
332
|
-
message: `Failed to convert items for ClickHouse table "` + table.name + `"`,
|
|
333
|
-
reason: Utils.prettifyExn(exn),
|
|
334
|
-
Error: new Error()
|
|
335
|
-
};
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
6
|
|
|
339
7
|
function optionsFromEnvOrThrow() {
|
|
340
8
|
let missing = [];
|
|
@@ -364,12 +32,6 @@ function optionsFromEnvOrThrow() {
|
|
|
364
32
|
}
|
|
365
33
|
|
|
366
34
|
export {
|
|
367
|
-
fieldTypeName,
|
|
368
|
-
column,
|
|
369
|
-
fieldColumn,
|
|
370
|
-
entityTable,
|
|
371
|
-
makeTable,
|
|
372
|
-
stageChangesOrThrow,
|
|
373
35
|
optionsFromEnvOrThrow,
|
|
374
36
|
}
|
|
375
37
|
/* Env Not a pure module */
|