envio 3.5.0-alpha.2 → 3.5.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/BatchProcessing.res +14 -0
- package/src/BatchProcessing.res.mjs +8 -0
- package/src/ChainState.res +10 -4
- package/src/ChainState.res.mjs +7 -6
- package/src/ChainState.resi +1 -1
- package/src/Config.res +2 -2
- package/src/Config.res.mjs +2 -2
- package/src/CrossChainState.res +33 -10
- package/src/CrossChainState.res.mjs +21 -6
- package/src/CrossChainState.resi +3 -0
- package/src/FetchState.res +230 -66
- package/src/FetchState.res.mjs +213 -67
- package/src/FinalizeBackfill.res +34 -0
- package/src/FinalizeBackfill.res.mjs +26 -0
- package/src/InMemoryTable.res +15 -15
- package/src/InMemoryTable.res.mjs +14 -14
- package/src/IndexerState.res +12 -0
- package/src/IndexerState.res.mjs +19 -0
- package/src/IndexerState.resi +3 -0
- package/src/LoadLayer.res +9 -1
- package/src/LoadLayer.res.mjs +1 -0
- package/src/Persistence.res +11 -0
- package/src/PgStorage.res +356 -35
- package/src/PgStorage.res.mjs +237 -24
- package/src/TestIndexer.res +3 -0
- package/src/TestIndexer.res.mjs +2 -0
- package/src/db/EntityFilter.res +0 -4
- package/src/db/EntityFilter.res.mjs +1 -8
- package/src/db/IndexRegistry.res +219 -0
- package/src/db/IndexRegistry.res.mjs +195 -0
- package/src/db/InternalTable.res +7 -0
- package/src/db/InternalTable.res.mjs +7 -0
- package/src/db/Table.res +13 -13
- package/src/db/Table.res.mjs +12 -12
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Utils from "../Utils.res.mjs";
|
|
4
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
5
|
+
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
6
|
+
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
7
|
+
|
|
8
|
+
function fromIndexFields(indexFields) {
|
|
9
|
+
return indexFields.map(param => ({
|
|
10
|
+
name: param.fieldName,
|
|
11
|
+
direction: param.direction
|
|
12
|
+
}));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function makeKey(tableName, columns, method) {
|
|
16
|
+
return tableName + `|` + method + `|` + columns.map(param => {
|
|
17
|
+
let name = param.name;
|
|
18
|
+
if (param.direction === "Asc") {
|
|
19
|
+
return name;
|
|
20
|
+
} else {
|
|
21
|
+
return name + " DESC";
|
|
22
|
+
}
|
|
23
|
+
}).join(",");
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let catalogRowsSchema = S$RescriptSchema.array(S$RescriptSchema.object(s => ({
|
|
27
|
+
tableName: s.f("tableName", S$RescriptSchema.string),
|
|
28
|
+
indexName: s.f("indexName", S$RescriptSchema.string),
|
|
29
|
+
method: s.f("method", S$RescriptSchema.string),
|
|
30
|
+
isValid: s.f("isValid", S$RescriptSchema.int),
|
|
31
|
+
isPlain: s.f("isPlain", S$RescriptSchema.int),
|
|
32
|
+
columns: s.f("columns", S$RescriptSchema.array(S$RescriptSchema.string)),
|
|
33
|
+
directions: s.f("directions", S$RescriptSchema.array(S$RescriptSchema.string))
|
|
34
|
+
})));
|
|
35
|
+
|
|
36
|
+
function makeCatalogQuery(pgSchema) {
|
|
37
|
+
return `SELECT
|
|
38
|
+
t.relname AS "tableName",
|
|
39
|
+
i.relname AS "indexName",
|
|
40
|
+
am.amname AS "method",
|
|
41
|
+
CASE WHEN ix.indisvalid AND ix.indisready THEN 1 ELSE 0 END AS "isValid",
|
|
42
|
+
CASE
|
|
43
|
+
WHEN ix.indisunique OR ix.indpred IS NOT NULL OR ix.indexprs IS NOT NULL THEN 0
|
|
44
|
+
ELSE 1
|
|
45
|
+
END AS "isPlain",
|
|
46
|
+
array_agg(
|
|
47
|
+
CASE WHEN k.attnum = 0
|
|
48
|
+
THEN pg_get_indexdef(ix.indexrelid, k.ord::int, true)
|
|
49
|
+
ELSE a.attname
|
|
50
|
+
END ORDER BY k.ord
|
|
51
|
+
) AS "columns",
|
|
52
|
+
array_agg(
|
|
53
|
+
CASE WHEN (ix.indoption[k.ord - 1] & 1) = 1 THEN 'DESC' ELSE 'ASC' END ORDER BY k.ord
|
|
54
|
+
) AS "directions"
|
|
55
|
+
FROM pg_index ix
|
|
56
|
+
JOIN pg_class i ON i.oid = ix.indexrelid
|
|
57
|
+
JOIN pg_class t ON t.oid = ix.indrelid
|
|
58
|
+
JOIN pg_namespace n ON n.oid = t.relnamespace
|
|
59
|
+
JOIN pg_am am ON am.oid = i.relam
|
|
60
|
+
JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) ON k.ord <= ix.indnkeyatts
|
|
61
|
+
LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
|
|
62
|
+
WHERE n.nspname = '` + pgSchema + `'
|
|
63
|
+
GROUP BY t.relname, i.relname, am.amname, ix.indisvalid, ix.indisready,
|
|
64
|
+
ix.indisunique, ix.indpred, ix.indexprs;`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function make() {
|
|
68
|
+
return {
|
|
69
|
+
keys: new Set(),
|
|
70
|
+
invalidByName: {},
|
|
71
|
+
inflight: {},
|
|
72
|
+
tableQueue: {}
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function has(registry, key) {
|
|
77
|
+
return registry.keys.has(key);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function add(registry, key) {
|
|
81
|
+
registry.keys.add(key);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function size(registry) {
|
|
85
|
+
return registry.keys.size;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function toArray(registry) {
|
|
89
|
+
return Array.from(registry.keys).toSorted(Primitive_string.compare);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function getInvalid(registry, name) {
|
|
93
|
+
return registry.invalidByName[name];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function clearInvalidName(registry, name) {
|
|
97
|
+
Utils.Dict.deleteInPlace(registry.invalidByName, name);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function reload(registry, rows) {
|
|
101
|
+
registry.keys.clear();
|
|
102
|
+
Object.keys(registry.invalidByName).forEach(name => Utils.Dict.deleteInPlace(registry.invalidByName, name));
|
|
103
|
+
let invalidIndexNames = [];
|
|
104
|
+
rows.forEach(row => {
|
|
105
|
+
let key = makeKey(row.tableName, row.columns.map((name, idx) => {
|
|
106
|
+
let match = row.directions[idx];
|
|
107
|
+
let tmp;
|
|
108
|
+
tmp = match === "DESC" ? "Desc" : "Asc";
|
|
109
|
+
return {
|
|
110
|
+
name: name,
|
|
111
|
+
direction: tmp
|
|
112
|
+
};
|
|
113
|
+
}), row.method);
|
|
114
|
+
if (row.isValid === 0) {
|
|
115
|
+
invalidIndexNames.push(row.indexName);
|
|
116
|
+
registry.invalidByName[row.indexName] = {
|
|
117
|
+
key: key,
|
|
118
|
+
isRepairable: row.isPlain === 1
|
|
119
|
+
};
|
|
120
|
+
return;
|
|
121
|
+
} else {
|
|
122
|
+
return add(registry, key);
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
return invalidIndexNames;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function ensure(registry, key, tableName, build) {
|
|
129
|
+
if (registry.keys.has(key)) {
|
|
130
|
+
return Promise.resolve();
|
|
131
|
+
}
|
|
132
|
+
let promise = registry.inflight[key];
|
|
133
|
+
if (promise !== undefined) {
|
|
134
|
+
return promise;
|
|
135
|
+
}
|
|
136
|
+
let tail = registry.tableQueue[tableName];
|
|
137
|
+
let tail$1 = tail !== undefined ? tail : Promise.resolve();
|
|
138
|
+
let promise$1 = tail$1.then(() => {
|
|
139
|
+
if (registry.keys.has(key)) {
|
|
140
|
+
return Promise.resolve();
|
|
141
|
+
} else {
|
|
142
|
+
return build();
|
|
143
|
+
}
|
|
144
|
+
}).then(() => add(registry, key));
|
|
145
|
+
registry.inflight[key] = promise$1;
|
|
146
|
+
registry.tableQueue[tableName] = Utils.$$Promise.silentCatch(promise$1);
|
|
147
|
+
return promise$1.finally(() => Utils.Dict.deleteInPlace(registry.inflight, key));
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function truncateIndexName(description) {
|
|
151
|
+
if (description.length > 63) {
|
|
152
|
+
return description.slice(0, 63);
|
|
153
|
+
} else {
|
|
154
|
+
return description;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function validateIndexNamesOrThrow(descriptions) {
|
|
159
|
+
let byName = {};
|
|
160
|
+
descriptions.forEach(description => {
|
|
161
|
+
let name = truncateIndexName(description);
|
|
162
|
+
let existing = byName[name];
|
|
163
|
+
if (existing !== undefined && existing !== description) {
|
|
164
|
+
return Stdlib_JsError.throwWithMessage(`Index names "` + existing + `" and "` + description + `" both truncate to "` + name + `" at PostgreSQL's ` + (63).toString() + `-character identifier limit. Rename a field or an entity so the generated index names stay distinct.`);
|
|
165
|
+
} else {
|
|
166
|
+
byName[name] = description;
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
let btree = "btree";
|
|
173
|
+
|
|
174
|
+
let pgMaxIdentifierLength = 63;
|
|
175
|
+
|
|
176
|
+
export {
|
|
177
|
+
btree,
|
|
178
|
+
fromIndexFields,
|
|
179
|
+
makeKey,
|
|
180
|
+
catalogRowsSchema,
|
|
181
|
+
makeCatalogQuery,
|
|
182
|
+
make,
|
|
183
|
+
has,
|
|
184
|
+
add,
|
|
185
|
+
size,
|
|
186
|
+
toArray,
|
|
187
|
+
getInvalid,
|
|
188
|
+
clearInvalidName,
|
|
189
|
+
reload,
|
|
190
|
+
ensure,
|
|
191
|
+
pgMaxIdentifierLength,
|
|
192
|
+
truncateIndexName,
|
|
193
|
+
validateIndexNamesOrThrow,
|
|
194
|
+
}
|
|
195
|
+
/* catalogRowsSchema Not a pure module */
|
package/src/db/InternalTable.res
CHANGED
|
@@ -164,6 +164,13 @@ SET ${setClauses->Array.joinUnsafe(",\n ")}
|
|
|
164
164
|
WHERE "${(#id: field :> string)}" = $1;`
|
|
165
165
|
}
|
|
166
166
|
|
|
167
|
+
// Written in the same transaction as the deferred schema indexes, so a chain
|
|
168
|
+
// is never reported ready without the indexes the schema promises.
|
|
169
|
+
let makeSetReadyAtQuery = (~pgSchema) =>
|
|
170
|
+
`UPDATE "${pgSchema}"."${table.tableName}"
|
|
171
|
+
SET "${(#ready_at: field :> string)}" = $1
|
|
172
|
+
WHERE "${(#id: field :> string)}" = ANY($2::int[]);`
|
|
173
|
+
|
|
167
174
|
type rawInitialState = {
|
|
168
175
|
id: int,
|
|
169
176
|
startBlock: int,
|
|
@@ -101,6 +101,12 @@ SET ` + setClauses.join(",\n ") + `
|
|
|
101
101
|
WHERE "` + "id" + `" = $1;`;
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
function makeSetReadyAtQuery(pgSchema) {
|
|
105
|
+
return `UPDATE "` + pgSchema + `"."` + table.tableName + `"
|
|
106
|
+
SET "` + "ready_at" + `" = $1
|
|
107
|
+
WHERE "` + "id" + `" = ANY($2::int[]);`;
|
|
108
|
+
}
|
|
109
|
+
|
|
104
110
|
function makeGetInitialStateQuery(pgSchema) {
|
|
105
111
|
return `SELECT "` + "id" + `" as "id",
|
|
106
112
|
"` + "start_block" + `" as "startBlock",
|
|
@@ -213,6 +219,7 @@ let Chains = {
|
|
|
213
219
|
makeInitialValuesQuery: makeInitialValuesQuery,
|
|
214
220
|
metaFields: metaFields,
|
|
215
221
|
makeMetaFieldsUpdateQuery: makeMetaFieldsUpdateQuery,
|
|
222
|
+
makeSetReadyAtQuery: makeSetReadyAtQuery,
|
|
216
223
|
makeGetInitialStateQuery: makeGetInitialStateQuery,
|
|
217
224
|
makeGetIndexingAddressesQuery: makeGetIndexingAddressesQuery,
|
|
218
225
|
getInitialState: getInitialState,
|
package/src/db/Table.res
CHANGED
|
@@ -190,14 +190,14 @@ type compositeIndexField = {
|
|
|
190
190
|
type table = {
|
|
191
191
|
tableName: string,
|
|
192
192
|
fields: array<fieldOrDerived>,
|
|
193
|
-
|
|
193
|
+
compositeIndexes: array<array<compositeIndexField>>,
|
|
194
194
|
description: option<string>,
|
|
195
195
|
}
|
|
196
196
|
|
|
197
|
-
let mkTable = (tableName, ~
|
|
197
|
+
let mkTable = (tableName, ~compositeIndexes=[], ~fields, ~description=?) => {
|
|
198
198
|
tableName,
|
|
199
199
|
fields,
|
|
200
|
-
|
|
200
|
+
compositeIndexes,
|
|
201
201
|
description,
|
|
202
202
|
}
|
|
203
203
|
|
|
@@ -355,11 +355,11 @@ let pgRowsSchema: table => S.t<array<unknown>> = Utils.WeakMap.memoize(table =>
|
|
|
355
355
|
exception NonExistingTableField(string)
|
|
356
356
|
|
|
357
357
|
/*
|
|
358
|
-
Gets all composite
|
|
358
|
+
Gets all composite indexes (whether they are single indexes or not)
|
|
359
359
|
And maps the fields defined to their actual db name (some have _id suffix)
|
|
360
360
|
*/
|
|
361
|
-
let
|
|
362
|
-
table.
|
|
361
|
+
let getUnfilteredCompositeIndexesUnsafe = (table): array<array<compositeIndexField>> => {
|
|
362
|
+
table.compositeIndexes->Array.map(compositeIndex =>
|
|
363
363
|
compositeIndex->Array.map(indexField => {
|
|
364
364
|
let dbFieldName = switch table->getFieldByName(indexField.fieldName) {
|
|
365
365
|
| Some(field) => field->getPgFieldName
|
|
@@ -469,10 +469,10 @@ let toSqlParams = (table: table, ~schema, ~pgSchema) => {
|
|
|
469
469
|
}
|
|
470
470
|
|
|
471
471
|
/*
|
|
472
|
-
Gets all single
|
|
472
|
+
Gets all single indexes
|
|
473
473
|
And maps the fields defined to their actual db name (some have _id suffix)
|
|
474
474
|
*/
|
|
475
|
-
let
|
|
475
|
+
let getSingleIndexes = (table): array<string> => {
|
|
476
476
|
let indexFields = table.fields->Array.filterMap(field =>
|
|
477
477
|
switch field {
|
|
478
478
|
| Field(field) if field.isIndex => Some(field->getPgDbFieldName)
|
|
@@ -481,8 +481,8 @@ let getSingleIndices = (table): array<string> => {
|
|
|
481
481
|
)
|
|
482
482
|
|
|
483
483
|
table
|
|
484
|
-
->
|
|
485
|
-
//get all composite
|
|
484
|
+
->getUnfilteredCompositeIndexesUnsafe
|
|
485
|
+
//get all composite indexes with only 1 field defined
|
|
486
486
|
//this is still a single index
|
|
487
487
|
->Array.filterMap(cidx =>
|
|
488
488
|
switch cidx {
|
|
@@ -498,11 +498,11 @@ let getSingleIndices = (table): array<string> => {
|
|
|
498
498
|
}
|
|
499
499
|
|
|
500
500
|
/*
|
|
501
|
-
Gets all composite
|
|
501
|
+
Gets all composite indexes
|
|
502
502
|
And maps the fields defined to their actual db name (some have _id suffix)
|
|
503
503
|
*/
|
|
504
|
-
let
|
|
504
|
+
let getCompositeIndexes = (table): array<array<compositeIndexField>> => {
|
|
505
505
|
table
|
|
506
|
-
->
|
|
506
|
+
->getUnfilteredCompositeIndexesUnsafe
|
|
507
507
|
->Array.filter(ind => ind->Array.length > 1)
|
|
508
508
|
}
|
package/src/db/Table.res.mjs
CHANGED
|
@@ -153,12 +153,12 @@ function getPgFieldType(fieldType, pgSchema, isArray, isNumericArrayAsText, isNu
|
|
|
153
153
|
);
|
|
154
154
|
}
|
|
155
155
|
|
|
156
|
-
function mkTable(tableName,
|
|
157
|
-
let
|
|
156
|
+
function mkTable(tableName, compositeIndexesOpt, fields, description) {
|
|
157
|
+
let compositeIndexes = compositeIndexesOpt !== undefined ? compositeIndexesOpt : [];
|
|
158
158
|
return {
|
|
159
159
|
tableName: tableName,
|
|
160
160
|
fields: fields,
|
|
161
|
-
|
|
161
|
+
compositeIndexes: compositeIndexes,
|
|
162
162
|
description: description
|
|
163
163
|
};
|
|
164
164
|
}
|
|
@@ -304,8 +304,8 @@ let pgRowsSchema = Utils.$$WeakMap.memoize(table => makeRowsSchema(table, getPgD
|
|
|
304
304
|
|
|
305
305
|
let NonExistingTableField = /* @__PURE__ */Primitive_exceptions.create("Table.NonExistingTableField");
|
|
306
306
|
|
|
307
|
-
function
|
|
308
|
-
return table.
|
|
307
|
+
function getUnfilteredCompositeIndexesUnsafe(table) {
|
|
308
|
+
return table.compositeIndexes.map(compositeIndex => compositeIndex.map(indexField => {
|
|
309
309
|
let field = getFieldByName(table, indexField.fieldName);
|
|
310
310
|
let dbFieldName;
|
|
311
311
|
if (field !== undefined) {
|
|
@@ -419,7 +419,7 @@ function toSqlParams(table, schema, pgSchema) {
|
|
|
419
419
|
};
|
|
420
420
|
}
|
|
421
421
|
|
|
422
|
-
function
|
|
422
|
+
function getSingleIndexes(table) {
|
|
423
423
|
let indexFields = Stdlib_Array.filterMap(table.fields, field => {
|
|
424
424
|
if (field.TAG !== "Field") {
|
|
425
425
|
return;
|
|
@@ -429,7 +429,7 @@ function getSingleIndices(table) {
|
|
|
429
429
|
return getPgDbFieldName(field$1);
|
|
430
430
|
}
|
|
431
431
|
});
|
|
432
|
-
return Array.from(new Set(Stdlib_Array.filterMap(
|
|
432
|
+
return Array.from(new Set(Stdlib_Array.filterMap(getUnfilteredCompositeIndexesUnsafe(table), cidx => {
|
|
433
433
|
if (cidx.length !== 1) {
|
|
434
434
|
return;
|
|
435
435
|
}
|
|
@@ -438,8 +438,8 @@ function getSingleIndices(table) {
|
|
|
438
438
|
}).concat([indexFields]).flat())).toSorted(Primitive_string.compare);
|
|
439
439
|
}
|
|
440
440
|
|
|
441
|
-
function
|
|
442
|
-
return
|
|
441
|
+
function getCompositeIndexes(table) {
|
|
442
|
+
return getUnfilteredCompositeIndexesUnsafe(table).filter(ind => ind.length > 1);
|
|
443
443
|
}
|
|
444
444
|
|
|
445
445
|
export {
|
|
@@ -473,9 +473,9 @@ export {
|
|
|
473
473
|
rowsSchema,
|
|
474
474
|
pgRowsSchema,
|
|
475
475
|
NonExistingTableField,
|
|
476
|
-
|
|
476
|
+
getUnfilteredCompositeIndexesUnsafe,
|
|
477
477
|
toSqlParams,
|
|
478
|
-
|
|
479
|
-
|
|
478
|
+
getSingleIndexes,
|
|
479
|
+
getCompositeIndexes,
|
|
480
480
|
}
|
|
481
481
|
/* idsArraySchema Not a pure module */
|