envio 3.8.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/index.d.ts +158 -104
- package/package.json +6 -6
- package/src/AddressRows.res +121 -0
- package/src/AddressRows.res.mjs +143 -0
- package/src/Batch.res +2 -0
- package/src/Batch.res.mjs +2 -1
- package/src/ChainState.res +76 -70
- package/src/ChainState.res.mjs +76 -65
- package/src/ChainState.resi +14 -9
- package/src/Config.res +23 -82
- package/src/Config.res.mjs +19 -63
- package/src/ContractMapping.res +66 -0
- package/src/ContractMapping.res.mjs +73 -0
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +4 -0
- package/src/EventConfigBuilder.res +0 -2
- package/src/EventConfigBuilder.res.mjs +0 -2
- package/src/FetchState.res +59 -67
- package/src/FetchState.res.mjs +34 -44
- package/src/InMemoryStore.res +17 -30
- package/src/InMemoryStore.res.mjs +11 -23
- package/src/IndexerState.res +21 -1
- package/src/IndexerState.res.mjs +9 -3
- package/src/IndexerState.resi +1 -0
- package/src/Main.res +25 -15
- package/src/Main.res.mjs +19 -14
- package/src/MemoryStorage.res +39 -66
- package/src/MemoryStorage.res.mjs +28 -65
- package/src/Metrics.res +15 -1
- package/src/Metrics.res.mjs +5 -1
- package/src/Persistence.res +30 -16
- package/src/Persistence.res.mjs +8 -5
- package/src/PgStorage.res +183 -65
- package/src/PgStorage.res.mjs +92 -58
- package/src/Rollback.res +14 -10
- package/src/Rollback.res.mjs +4 -2
- package/src/SimulateItems.res +5 -12
- package/src/SimulateItems.res.mjs +6 -10
- package/src/TestIndexer.res +93 -111
- package/src/TestIndexer.res.mjs +61 -88
- package/src/Utils.res +7 -0
- package/src/Utils.res.mjs +9 -2
- package/src/Writing.res +2 -0
- package/src/Writing.res.mjs +2 -1
- package/src/bindings/ClickHouse.res +6 -0
- package/src/bindings/ClickHouse.res.mjs +4 -0
- package/src/bindings/NodeJs.res +9 -0
- package/src/bindings/NodeJs.res.mjs +8 -1
- package/src/bindings/Postgres.res +9 -0
- package/src/bindings/Postgres.res.mjs +3 -0
- package/src/db/EntityHistory.res +12 -18
- package/src/db/EntityHistory.res.mjs +3 -14
- package/src/db/InternalTable.res +171 -65
- package/src/db/InternalTable.res.mjs +176 -73
- package/src/db/Table.res +24 -0
- package/src/db/Table.res.mjs +20 -1
- package/src/sources/AddressSet.res +0 -22
- package/src/sources/AddressStore.res +48 -88
- package/src/sources/AddressStore.res.mjs +11 -22
- package/src/sources/SvmHyperSyncClient.res +17 -21
- package/src/sources/SvmHyperSyncSource.res +4 -9
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -13
- package/svm.schema.json +10 -4
package/src/Writing.res
CHANGED
|
@@ -120,6 +120,7 @@ let runOneWrite = async (state: IndexerState.t) => {
|
|
|
120
120
|
}
|
|
121
121
|
})
|
|
122
122
|
let updatedEffectsCache = snapshotEffects(state, ~cache)
|
|
123
|
+
let registeredAddresses = batch.registeredAddresses
|
|
123
124
|
|
|
124
125
|
let writtenEntityNames = Utils.Set.make()
|
|
125
126
|
updatedEntities->Array.forEach(({entityConfig}) =>
|
|
@@ -145,6 +146,7 @@ let runOneWrite = async (state: IndexerState.t) => {
|
|
|
145
146
|
~config,
|
|
146
147
|
~allEntities=persistence.allEntities,
|
|
147
148
|
~updatedEntities,
|
|
149
|
+
~registeredAddresses,
|
|
148
150
|
~updatedEffectsCache,
|
|
149
151
|
~chainMetaData,
|
|
150
152
|
~onWrite=(~storage, ~timeSeconds) =>
|
package/src/Writing.res.mjs
CHANGED
|
@@ -118,13 +118,14 @@ async function runOneWrite(state) {
|
|
|
118
118
|
}
|
|
119
119
|
});
|
|
120
120
|
let updatedEffectsCache = snapshotEffects(state, cache);
|
|
121
|
+
let registeredAddresses = batch.registeredAddresses;
|
|
121
122
|
let writtenEntityNames = new Set();
|
|
122
123
|
updatedEntities.forEach(param => {
|
|
123
124
|
writtenEntityNames.add(param.entityConfig.name);
|
|
124
125
|
});
|
|
125
126
|
let pruneTargets = PruneStaleHistory.select(state, writtenEntityNames, Stdlib_Option.isSome(rollback));
|
|
126
127
|
await Promise.all([
|
|
127
|
-
persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
|
|
128
|
+
persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
|
|
128
129
|
PruneStaleHistory.runConcurrent(state, pruneTargets)
|
|
129
130
|
]);
|
|
130
131
|
IndexerState.markCommitted(state, upToCheckpointId);
|
|
@@ -60,6 +60,12 @@ let getClickHouseFieldType = (
|
|
|
60
60
|
}
|
|
61
61
|
| Uint32 => "UInt32"
|
|
62
62
|
| UInt52 => "UInt64"
|
|
63
|
+
// Internal-only column types, never on an entity the sink mirrors.
|
|
64
|
+
| SmallInt
|
|
65
|
+
| Bytea =>
|
|
66
|
+
JsError.throwWithMessage(
|
|
67
|
+
"ClickHouse doesn't support the internal SmallInt and Bytea column types",
|
|
68
|
+
)
|
|
63
69
|
| UInt64 => "UInt64"
|
|
64
70
|
| Serial => "Int32"
|
|
65
71
|
| BigSerial => "Int64"
|
|
@@ -26,6 +26,10 @@ function getClickHouseFieldType(fieldType, isNullable, isArray, chainIdModeOpt)
|
|
|
26
26
|
case "Uint32" :
|
|
27
27
|
baseType = "UInt32";
|
|
28
28
|
break;
|
|
29
|
+
case "SmallInt" :
|
|
30
|
+
case "Bytea" :
|
|
31
|
+
baseType = Stdlib_JsError.throwWithMessage("ClickHouse doesn't support the internal SmallInt and Bytea column types");
|
|
32
|
+
break;
|
|
29
33
|
case "UInt52" :
|
|
30
34
|
case "UInt64" :
|
|
31
35
|
baseType = "UInt64";
|
package/src/bindings/NodeJs.res
CHANGED
|
@@ -66,6 +66,15 @@ module Process = {
|
|
|
66
66
|
external getActiveResourcesInfo: unit => array<string> = "getActiveResourcesInfo"
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
module Buffer = {
|
|
70
|
+
type t
|
|
71
|
+
@val @scope("Buffer") external concat: array<t> => t = "concat"
|
|
72
|
+
@val @scope("Buffer") external alloc: int => t = "alloc"
|
|
73
|
+
@get external length: t => int = "length"
|
|
74
|
+
@send external toBase64: (t, @as("base64") _) => string = "toString"
|
|
75
|
+
let empty = alloc(0)
|
|
76
|
+
}
|
|
77
|
+
|
|
69
78
|
module V8 = {
|
|
70
79
|
type heapSpaceStatistics = {
|
|
71
80
|
@as("space_name") spaceName: string,
|
|
@@ -18,6 +18,12 @@ let Util$1 = {
|
|
|
18
18
|
|
|
19
19
|
let Process = {};
|
|
20
20
|
|
|
21
|
+
let empty = globalThis.Buffer.alloc(0);
|
|
22
|
+
|
|
23
|
+
let Buffer = {
|
|
24
|
+
empty: empty
|
|
25
|
+
};
|
|
26
|
+
|
|
21
27
|
let V8 = {};
|
|
22
28
|
|
|
23
29
|
let PerfHooks = {};
|
|
@@ -49,6 +55,7 @@ let Fs = {
|
|
|
49
55
|
export {
|
|
50
56
|
Util$1 as Util,
|
|
51
57
|
Process,
|
|
58
|
+
Buffer,
|
|
52
59
|
V8,
|
|
53
60
|
PerfHooks,
|
|
54
61
|
ChildProcess,
|
|
@@ -59,4 +66,4 @@ export {
|
|
|
59
66
|
WorkerThreads,
|
|
60
67
|
Fs,
|
|
61
68
|
}
|
|
62
|
-
/*
|
|
69
|
+
/* empty Not a pure module */
|
|
@@ -99,6 +99,13 @@ external makeSql: (~config: poolConfig) => sql = "default"
|
|
|
99
99
|
// @send @variadic
|
|
100
100
|
// external sql: array<string> => (sql, array<string>) => int = "sql"
|
|
101
101
|
|
|
102
|
+
// postgres.js infers a Buffer array's type from its first element, so a bytea
|
|
103
|
+
// column's array parameter comes out typed `bytea` and the server refuses the
|
|
104
|
+
// cast. Naming the array type explicitly is what makes it bind. 1001 is
|
|
105
|
+
// Postgres' `bytea[]` OID.
|
|
106
|
+
let byteaArrayOid = 1001
|
|
107
|
+
@send external typed: (sql, 'a, int) => unknown = "typed"
|
|
108
|
+
|
|
102
109
|
@send external unsafe: (sql, string) => promise<'a> = "unsafe"
|
|
103
110
|
@send external unpreparedUnsafe: (sql, string, unknown) => promise<'a> = "unsafe"
|
|
104
111
|
@send
|
|
@@ -107,8 +114,10 @@ external preparedUnsafe: (sql, string, unknown, @as(json`{prepare: true}`) _) =>
|
|
|
107
114
|
|
|
108
115
|
@unboxed
|
|
109
116
|
type columnType =
|
|
117
|
+
| @as("SMALLINT") SmallInt
|
|
110
118
|
| @as("INTEGER") Integer
|
|
111
119
|
| @as("BIGINT") BigInt
|
|
120
|
+
| @as("BYTEA") Bytea
|
|
112
121
|
| @as("BOOLEAN") Boolean
|
|
113
122
|
| @as("NUMERIC") Numeric
|
|
114
123
|
| @as("DOUBLE PRECISION") DoublePrecision
|
package/src/db/EntityHistory.res
CHANGED
|
@@ -50,18 +50,14 @@ type pgEntityHistory<'entity> = {
|
|
|
50
50
|
setChangeSchemaRows: S.t<array<Change.t<'entity>>>,
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
let maxPgTableNameLength = 63
|
|
54
53
|
let historyTablePrefix = "envio_history_"
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
fullName
|
|
63
|
-
}
|
|
64
|
-
}
|
|
54
|
+
// `$` can't occur in a GraphQL entity name, so it marks where a truncated name
|
|
55
|
+
// stops and the index that keeps it unique begins. Without that boundary two
|
|
56
|
+
// long names whose indexes differ in digit count can truncate onto the same
|
|
57
|
+
// identifier, and `CREATE TABLE IF NOT EXISTS` would hand both entities one
|
|
58
|
+
// history table.
|
|
59
|
+
let historyTableName = (~entityName, ~entityIndex) =>
|
|
60
|
+
fitPgTableName(historyTablePrefix ++ entityName, ~uniqueSuffix=`$${entityIndex->Int.toString}`)
|
|
65
61
|
|
|
66
62
|
type safeReorgBlocks = {
|
|
67
63
|
chainIds: array<ChainId.t>,
|
|
@@ -136,7 +132,7 @@ let pruneStaleEntityHistory = (
|
|
|
136
132
|
// If an entity doesn't have a history before the update
|
|
137
133
|
// we create it automatically with envio_checkpoint_id 0
|
|
138
134
|
// The ids belong to a single chain (the flush group's scope), so the chain is
|
|
139
|
-
//
|
|
135
|
+
// named once in the query rather than unnested alongside them.
|
|
140
136
|
let makeBackfillHistoryQuery = (
|
|
141
137
|
~pgSchema,
|
|
142
138
|
~entityName,
|
|
@@ -146,8 +142,11 @@ let makeBackfillHistoryQuery = (
|
|
|
146
142
|
~chainId: option<ChainId.t>,
|
|
147
143
|
) => {
|
|
148
144
|
let historyTableRef = `"${pgSchema}"."${historyTableName(~entityName, ~entityIndex)}"`
|
|
145
|
+
// Written into the SQL rather than bound: this scans the entity table, which
|
|
146
|
+
// is partitioned by the chain-id column, and Postgres can only prune a plan
|
|
147
|
+
// it caches when that column is a constant.
|
|
149
148
|
let chainFilter = switch (chainIdColumn, chainId) {
|
|
150
|
-
| (Some(column), Some(
|
|
149
|
+
| (Some(column), Some(chainId)) => ` AND e."${column}" = ${chainId->ChainId.toString}`
|
|
151
150
|
| _ => ""
|
|
152
151
|
}
|
|
153
152
|
`WITH target_ids AS (
|
|
@@ -176,11 +175,6 @@ let backfillHistory = (
|
|
|
176
175
|
let idPgType = table->Table.getIdPgFieldType(~pgSchema)
|
|
177
176
|
let chainIdColumn = table->Table.getChainIdField->Option.map(Table.getPgDbFieldName)
|
|
178
177
|
let params = [table->Table.encodeIdsToJson(ids)->(Utils.magic: JSON.t => unknown)]
|
|
179
|
-
switch (chainIdColumn, chainId) {
|
|
180
|
-
| (Some(_), Some(chainId)) =>
|
|
181
|
-
params->Array.push(chainId->(Utils.magic: ChainId.t => unknown))->ignore
|
|
182
|
-
| _ => ()
|
|
183
|
-
}
|
|
184
178
|
sql
|
|
185
179
|
->Postgres.preparedUnsafe(
|
|
186
180
|
makeBackfillHistoryQuery(
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Table from "./Table.res.mjs";
|
|
4
|
-
import * as
|
|
4
|
+
import * as ChainId from "../ChainId.res.mjs";
|
|
5
5
|
import * as Stdlib_BigInt from "@rescript/runtime/lib/es6/Stdlib_BigInt.js";
|
|
6
6
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
7
7
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
@@ -65,12 +65,7 @@ function makeSetUpdateSchema(idSchema, entitySchema) {
|
|
|
65
65
|
let historyTablePrefix = "envio_history_";
|
|
66
66
|
|
|
67
67
|
function historyTableName(entityName, entityIndex) {
|
|
68
|
-
|
|
69
|
-
if (fullName.length <= 63) {
|
|
70
|
-
return fullName;
|
|
71
|
-
}
|
|
72
|
-
let entityIndexStr = entityIndex.toString();
|
|
73
|
-
return Js_string.slice(0, 63 - entityIndexStr.length | 0, fullName) + entityIndexStr;
|
|
68
|
+
return Table.fitPgTableName(historyTablePrefix + entityName, `$` + entityIndex.toString());
|
|
74
69
|
}
|
|
75
70
|
|
|
76
71
|
function makeKeyColumns(chainIdColumn) {
|
|
@@ -112,7 +107,7 @@ function pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainId
|
|
|
112
107
|
|
|
113
108
|
function makeBackfillHistoryQuery(pgSchema, entityName, entityIndex, idPgType, chainIdColumn, chainId) {
|
|
114
109
|
let historyTableRef = `"` + pgSchema + `"."` + historyTableName(entityName, entityIndex) + `"`;
|
|
115
|
-
let chainFilter = chainIdColumn !== undefined && chainId !== undefined ? ` AND e."` + chainIdColumn + `" =
|
|
110
|
+
let chainFilter = chainIdColumn !== undefined && chainId !== undefined ? ` AND e."` + chainIdColumn + `" = ` + ChainId.toString(Primitive_option.valFromOption(chainId)) : "";
|
|
116
111
|
return `WITH target_ids AS (
|
|
117
112
|
SELECT UNNEST($1::` + idPgType + `[]) AS id
|
|
118
113
|
),
|
|
@@ -132,9 +127,6 @@ function backfillHistory(sql, pgSchema, table, entityIndex, chainId, ids) {
|
|
|
132
127
|
let idPgType = Table.getIdPgFieldType(table, pgSchema);
|
|
133
128
|
let chainIdColumn = Stdlib_Option.map(Table.getChainIdField(table), Table.getPgDbFieldName);
|
|
134
129
|
let params = [Table.encodeIdsToJson(table, ids)];
|
|
135
|
-
if (chainIdColumn !== undefined && chainId !== undefined) {
|
|
136
|
-
params.push(Primitive_option.valFromOption(chainId));
|
|
137
|
-
}
|
|
138
130
|
return sql.unsafe(makeBackfillHistoryQuery(pgSchema, table.tableName, entityIndex, idPgType, chainIdColumn, chainId), params, {prepare: true});
|
|
139
131
|
}
|
|
140
132
|
|
|
@@ -144,8 +136,6 @@ function rollback(sql, pgSchema, entityName, entityIndex, rollbackTargetCheckpoi
|
|
|
144
136
|
|
|
145
137
|
let checkpointIdFieldType = "UInt64";
|
|
146
138
|
|
|
147
|
-
let maxPgTableNameLength = 63;
|
|
148
|
-
|
|
149
139
|
export {
|
|
150
140
|
RowAction,
|
|
151
141
|
changeFieldName,
|
|
@@ -154,7 +144,6 @@ export {
|
|
|
154
144
|
changeFieldType,
|
|
155
145
|
unsafeCheckpointIdSchema,
|
|
156
146
|
makeSetUpdateSchema,
|
|
157
|
-
maxPgTableNameLength,
|
|
158
147
|
historyTablePrefix,
|
|
159
148
|
historyTableName,
|
|
160
149
|
makeKeyColumns,
|
package/src/db/InternalTable.res
CHANGED
|
@@ -5,7 +5,167 @@ let isPrimaryKey = true
|
|
|
5
5
|
let isNullable = true
|
|
6
6
|
let isIndex = true
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
// Postgres SQLSTATE for "undefined_table" — what a read gets when the schema was
|
|
9
|
+
// initialized by an older envio that didn't have the table.
|
|
10
|
+
let undefinedTableSqlState = "42P01"
|
|
11
|
+
|
|
12
|
+
@get external getSqlStateCode: JsExn.t => option<string> = "code"
|
|
13
|
+
|
|
14
|
+
let isUndefinedTable = exn =>
|
|
15
|
+
switch exn->JsExn.anyToExnInternal {
|
|
16
|
+
| JsExn(e) => e->getSqlStateCode === Some(undefinedTableSqlState)
|
|
17
|
+
| _ => false
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// The array type an unnest binds a chain-id column to. Resolved from the
|
|
21
|
+
// config's mode, so every internal query casts the parameter the same way the
|
|
22
|
+
// column was created.
|
|
23
|
+
let chainIdArrayType = (~pgSchema, ~chainIdMode: ChainId.mode) =>
|
|
24
|
+
Table.getPgFieldType(
|
|
25
|
+
~fieldType=ChainId,
|
|
26
|
+
~pgSchema,
|
|
27
|
+
~isArray=true,
|
|
28
|
+
~isNumericArrayAsText=false,
|
|
29
|
+
~isNullable=false,
|
|
30
|
+
~chainIdMode,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
// The canonical contract ids. Written once at initialize from the config's
|
|
34
|
+
// contract names in byte order, so an id names the same contract on every
|
|
35
|
+
// chain and across restarts; read back on resume, where the stored mapping is
|
|
36
|
+
// what every address row means.
|
|
37
|
+
module EnvioContracts = {
|
|
38
|
+
let table = mkTable(
|
|
39
|
+
"envio_contracts",
|
|
40
|
+
~fields=[
|
|
41
|
+
mkField("id", SmallInt, ~fieldSchema=S.int, ~isPrimaryKey),
|
|
42
|
+
mkField("name", String, ~fieldSchema=S.string),
|
|
43
|
+
],
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
let makeInsertQuery = (~pgSchema) =>
|
|
47
|
+
`INSERT INTO "${pgSchema}"."${table.tableName}" ("id", "name")
|
|
48
|
+
SELECT * FROM unnest($1::${(SmallInt: Postgres.columnType :> string)}[],$2::${(Text: Postgres.columnType :> string)}[]);`
|
|
49
|
+
|
|
50
|
+
// `contractNames` is the canonical list: a name's position is its id.
|
|
51
|
+
let insert = (sql, ~pgSchema, ~contractNames: array<string>) =>
|
|
52
|
+
sql
|
|
53
|
+
->Postgres.preparedUnsafe(
|
|
54
|
+
makeInsertQuery(~pgSchema),
|
|
55
|
+
(contractNames->Array.mapWithIndex((_, idx) => idx), contractNames)->(
|
|
56
|
+
Utils.magic: ((array<int>, array<string>)) => unknown
|
|
57
|
+
),
|
|
58
|
+
)
|
|
59
|
+
->Utils.Promise.ignoreValue
|
|
60
|
+
|
|
61
|
+
// Ordered by id, so the result is the canonical list itself. None when the
|
|
62
|
+
// schema has no such table: it was written by an envio that predates the
|
|
63
|
+
// contract mapping, and every address row in it is shaped differently — so a
|
|
64
|
+
// resume has to stop at the compat check rather than at a missing column.
|
|
65
|
+
let read = async (sql, ~pgSchema): option<array<string>> =>
|
|
66
|
+
try {
|
|
67
|
+
let rows: array<{
|
|
68
|
+
"name": string,
|
|
69
|
+
}> = await sql->Postgres.unsafe(
|
|
70
|
+
`SELECT "name" FROM "${pgSchema}"."${table.tableName}" ORDER BY "id";`,
|
|
71
|
+
)
|
|
72
|
+
Some(rows->Array.map(row => row["name"]))
|
|
73
|
+
} catch {
|
|
74
|
+
| exn => isUndefinedTable(exn) ? None : throw(exn)
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
module EnvioAddresses = {
|
|
79
|
+
let name = "envio_addresses"
|
|
80
|
+
|
|
81
|
+
let table = mkTable(
|
|
82
|
+
name,
|
|
83
|
+
~fields=[
|
|
84
|
+
mkField("chain_id", ChainId, ~fieldSchema=ChainId.schema, ~isPrimaryKey),
|
|
85
|
+
// The field schemas are unused: this table is read and written by the
|
|
86
|
+
// hand-written queries below, never through the generic row encoding.
|
|
87
|
+
mkField("address", Bytea, ~fieldSchema=S.string, ~isPrimaryKey),
|
|
88
|
+
mkField("contract_id", SmallInt, ~fieldSchema=S.int, ~isPrimaryKey),
|
|
89
|
+
mkField("registration_block", Int32, ~fieldSchema=S.int),
|
|
90
|
+
],
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
let makeInsertQuery = (~pgSchema, ~chainIdMode: ChainId.mode=Int32) => {
|
|
94
|
+
let chainIdArrayType = chainIdArrayType(~pgSchema, ~chainIdMode)
|
|
95
|
+
`INSERT INTO "${pgSchema}"."${table.tableName}" ("chain_id", "address", "contract_id", "registration_block")
|
|
96
|
+
SELECT * FROM unnest($1::${chainIdArrayType},$2::${(Bytea: Postgres.columnType :> string)}[],$3::${(SmallInt: Postgres.columnType :> string)}[],$4::${(Integer: Postgres.columnType :> string)}[])
|
|
97
|
+
ON CONFLICT ("chain_id", "address", "contract_id") DO NOTHING;`
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
let insert = (
|
|
101
|
+
sql,
|
|
102
|
+
~pgSchema,
|
|
103
|
+
~rows: array<AddressRows.row>,
|
|
104
|
+
~chainIdMode: ChainId.mode=Int32,
|
|
105
|
+
) => {
|
|
106
|
+
let chainIds = []
|
|
107
|
+
let addresses = []
|
|
108
|
+
let contractIds = []
|
|
109
|
+
let registrationBlocks = []
|
|
110
|
+
rows->Array.forEach(row => {
|
|
111
|
+
chainIds->Array.push(row.chainId)->ignore
|
|
112
|
+
addresses->Array.push(row.address)->ignore
|
|
113
|
+
contractIds->Array.push(row.contractId)->ignore
|
|
114
|
+
registrationBlocks->Array.push(row.registrationBlock)->ignore
|
|
115
|
+
})
|
|
116
|
+
sql
|
|
117
|
+
->Postgres.preparedUnsafe(
|
|
118
|
+
makeInsertQuery(~pgSchema, ~chainIdMode),
|
|
119
|
+
(
|
|
120
|
+
chainIds,
|
|
121
|
+
sql->Postgres.typed(addresses, Postgres.byteaArrayOid),
|
|
122
|
+
contractIds,
|
|
123
|
+
registrationBlocks,
|
|
124
|
+
)->(Utils.magic: ((array<ChainId.t>, unknown, array<int>, array<int>)) => unknown),
|
|
125
|
+
)
|
|
126
|
+
->Utils.Promise.ignoreValue
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
let makeDeleteQuery = (~pgSchema, ~chainIdMode: ChainId.mode=Int32) => {
|
|
130
|
+
let chainIdArrayType = chainIdArrayType(~pgSchema, ~chainIdMode)
|
|
131
|
+
`DELETE FROM "${pgSchema}"."${table.tableName}"
|
|
132
|
+
USING unnest($1::${chainIdArrayType},$2::${(Bytea: Postgres.columnType :> string)}[],$3::${(SmallInt: Postgres.columnType :> string)}[]) AS dead(chain_id, address, contract_id)
|
|
133
|
+
WHERE "${table.tableName}"."chain_id" = dead.chain_id
|
|
134
|
+
AND "${table.tableName}"."address" = dead.address
|
|
135
|
+
AND "${table.tableName}"."contract_id" = dead.contract_id;`
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
let delete = (
|
|
139
|
+
sql,
|
|
140
|
+
~pgSchema,
|
|
141
|
+
~keys: array<AddressRows.key>,
|
|
142
|
+
~chainIdMode: ChainId.mode=Int32,
|
|
143
|
+
) => {
|
|
144
|
+
let chainIds = []
|
|
145
|
+
let addresses = []
|
|
146
|
+
let contractIds = []
|
|
147
|
+
keys->Array.forEach(key => {
|
|
148
|
+
chainIds->Array.push(key.chainId)->ignore
|
|
149
|
+
addresses->Array.push(key.address)->ignore
|
|
150
|
+
contractIds->Array.push(key.contractId)->ignore
|
|
151
|
+
})
|
|
152
|
+
sql
|
|
153
|
+
->Postgres.preparedUnsafe(
|
|
154
|
+
makeDeleteQuery(~pgSchema, ~chainIdMode),
|
|
155
|
+
(chainIds, sql->Postgres.typed(addresses, Postgres.byteaArrayOid), contractIds)->(
|
|
156
|
+
Utils.magic: ((array<ChainId.t>, unknown, array<int>)) => unknown
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
->Utils.Promise.ignoreValue
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
let makeGetRowsQuery = (~pgSchema) =>
|
|
163
|
+
`SELECT "chain_id" as "chainId",
|
|
164
|
+
"address" as "address",
|
|
165
|
+
"contract_id" as "contractId",
|
|
166
|
+
"registration_block" as "registrationBlock"
|
|
167
|
+
FROM "${pgSchema}"."${table.tableName}";`
|
|
168
|
+
}
|
|
9
169
|
|
|
10
170
|
module Chains = {
|
|
11
171
|
type progressFields = [
|
|
@@ -198,7 +358,7 @@ WHERE "${(#id: field :> string)}" = $2
|
|
|
198
358
|
timestampCaughtUpToHeadOrEndblock: Null.t<Date.t>,
|
|
199
359
|
numEventsProcessed: float,
|
|
200
360
|
progressBlockNumber: int,
|
|
201
|
-
|
|
361
|
+
addressRows: AddressRows.seedRows,
|
|
202
362
|
sourceBlockNumber: int,
|
|
203
363
|
}
|
|
204
364
|
|
|
@@ -215,67 +375,30 @@ WHERE "${(#id: field :> string)}" = $2
|
|
|
215
375
|
FROM "${pgSchema}"."${table.tableName}";`
|
|
216
376
|
}
|
|
217
377
|
|
|
218
|
-
type rawIndexingAddress = {
|
|
219
|
-
chainId: ChainId.t,
|
|
220
|
-
address: Address.t,
|
|
221
|
-
contractName: string,
|
|
222
|
-
registrationBlock: int,
|
|
223
|
-
}
|
|
224
|
-
|
|
225
378
|
// Addresses are read as plain rows rather than aggregated per chain with
|
|
226
379
|
// json_agg: a single chain's aggregate can exceed V8's max string length
|
|
227
380
|
// (postgres.js decodes the column with Buffer.toString and throws
|
|
228
381
|
// ERR_STRING_TOO_LONG). Grouping happens in JS instead — see getInitialState.
|
|
229
|
-
let makeGetIndexingAddressesQuery = (~pgSchema) => {
|
|
230
|
-
// envio_addresses.id is a composite "{chainId}-{address}" string produced by
|
|
231
|
-
// Config.EnvioAddresses.makeId; extract the address by taking everything
|
|
232
|
-
// after the first '-'. Keep in sync with makeId / getAddress.
|
|
233
|
-
`SELECT "chain_id" as "chainId",
|
|
234
|
-
SUBSTRING("id" FROM POSITION('-' IN "id") + 1) as "address",
|
|
235
|
-
"contract_name" as "contractName",
|
|
236
|
-
"registration_block" as "registrationBlock"
|
|
237
|
-
FROM "${pgSchema}"."${EnvioAddresses.table.tableName}";`
|
|
238
|
-
}
|
|
239
|
-
|
|
240
382
|
let getInitialState = async (sql, ~pgSchema) => {
|
|
241
|
-
let (rawInitialStates,
|
|
383
|
+
let (rawInitialStates, rawAddressRows) = await Promise.all2((
|
|
242
384
|
sql
|
|
243
385
|
->Postgres.unsafe(makeGetInitialStateQuery(~pgSchema))
|
|
244
386
|
->(Utils.magic: promise<array<unknown>> => promise<array<rawInitialState>>),
|
|
245
387
|
sql
|
|
246
|
-
->Postgres.unsafe(
|
|
247
|
-
->(Utils.magic: promise<array<unknown>> => promise<array<
|
|
388
|
+
->Postgres.unsafe(EnvioAddresses.makeGetRowsQuery(~pgSchema))
|
|
389
|
+
->(Utils.magic: promise<array<unknown>> => promise<array<AddressRows.row>>),
|
|
248
390
|
))
|
|
249
391
|
|
|
250
|
-
let
|
|
251
|
-
rawIndexingAddresses->Array.forEach(row => {
|
|
252
|
-
// BIGINT chain ids come back as strings; normalizing here keeps the
|
|
253
|
-
// grouping key identical to the one derived from the chains rows below.
|
|
254
|
-
let key = row.chainId->ChainId.normalizeOrThrow->ChainId.toString
|
|
255
|
-
let addresses = switch indexingAddressesByChainId->Dict.get(key) {
|
|
256
|
-
| Some(addresses) => addresses
|
|
257
|
-
| None =>
|
|
258
|
-
let addresses: array<Internal.indexingAddress> = []
|
|
259
|
-
indexingAddressesByChainId->Dict.set(key, addresses)
|
|
260
|
-
addresses
|
|
261
|
-
}
|
|
262
|
-
addresses
|
|
263
|
-
->Array.push({
|
|
264
|
-
address: row.address,
|
|
265
|
-
contractName: row.contractName,
|
|
266
|
-
registrationBlock: row.registrationBlock,
|
|
267
|
-
})
|
|
268
|
-
->ignore
|
|
269
|
-
})
|
|
392
|
+
let addressRowsByChainId = rawAddressRows->AddressRows.group
|
|
270
393
|
|
|
271
394
|
rawInitialStates->Array.map(rawInitialState => {
|
|
272
395
|
let id = rawInitialState.id->ChainId.normalizeOrThrow
|
|
273
396
|
{
|
|
274
397
|
...rawInitialState,
|
|
275
398
|
id,
|
|
276
|
-
|
|
277
|
-
->Dict.
|
|
278
|
-
->Option.getOr(
|
|
399
|
+
addressRows: addressRowsByChainId
|
|
400
|
+
->Utils.Dict.dangerouslyGetNonOption(id->ChainId.toString)
|
|
401
|
+
->Option.getOr(AddressRows.emptySeedRows()),
|
|
279
402
|
}
|
|
280
403
|
})
|
|
281
404
|
}
|
|
@@ -371,23 +494,13 @@ module EnvioInfo = {
|
|
|
371
494
|
],
|
|
372
495
|
)
|
|
373
496
|
|
|
374
|
-
// Postgres SQLSTATE for "undefined_table" — what we get when the schema
|
|
375
|
-
// was initialized by an older envio that didn't have `envio_info`.
|
|
376
|
-
let undefinedTableSqlState = "42P01"
|
|
377
|
-
|
|
378
|
-
@get external getCode: JsExn.t => option<string> = "code"
|
|
379
|
-
|
|
380
497
|
let read = async (sql, ~pgSchema): option<JSON.t> => {
|
|
381
498
|
let rows: array<{
|
|
382
499
|
"config": string,
|
|
383
500
|
}> = try await sql->Postgres.unsafe(
|
|
384
501
|
`SELECT "config" FROM "${pgSchema}"."${table.tableName}" LIMIT 1;`,
|
|
385
502
|
) catch {
|
|
386
|
-
| exn =>
|
|
387
|
-
switch exn->JsExn.anyToExnInternal {
|
|
388
|
-
| JsExn(e) if e->getCode === Some(undefinedTableSqlState) => []
|
|
389
|
-
| _ => throw(exn)
|
|
390
|
-
}
|
|
503
|
+
| exn => isUndefinedTable(exn) ? [] : throw(exn)
|
|
391
504
|
}
|
|
392
505
|
rows->Array.get(0)->Option.map(row => row["config"]->JSON.parseOrThrow)
|
|
393
506
|
}
|
|
@@ -485,14 +598,7 @@ WHERE cp."${(#block_hash: field :> string)}" IS NOT NULL
|
|
|
485
598
|
}
|
|
486
599
|
|
|
487
600
|
let makeInsertCheckpointQuery = (~pgSchema, ~chainIdMode: ChainId.mode=Int32) => {
|
|
488
|
-
let chainIdArrayType =
|
|
489
|
-
~fieldType=ChainId,
|
|
490
|
-
~pgSchema,
|
|
491
|
-
~isArray=true,
|
|
492
|
-
~isNumericArrayAsText=false,
|
|
493
|
-
~isNullable=false,
|
|
494
|
-
~chainIdMode,
|
|
495
|
-
)
|
|
601
|
+
let chainIdArrayType = chainIdArrayType(~pgSchema, ~chainIdMode)
|
|
496
602
|
`INSERT INTO "${pgSchema}"."${table.tableName}" ("${(#id: field :> string)}", "${(#chain_id: field :> string)}", "${(#block_number: field :> string)}", "${(#block_hash: field :> string)}", "${(#events_processed: field :> string)}")
|
|
497
603
|
SELECT * FROM unnest($1::${(BigInt: Postgres.columnType :> string)}[],$2::${chainIdArrayType},$3::${(Integer: Postgres.columnType :> string)}[],$4::${(Text: Postgres.columnType :> string)}[],$5::${(Integer: Postgres.columnType :> string)}[]);`
|
|
498
604
|
}
|