envio 2.32.3 → 3.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.js +5 -3
- package/evm.schema.json +23 -6
- package/fuel.schema.json +17 -5
- package/index.d.ts +1 -27
- package/index.js +9 -6
- package/package.json +12 -7
- package/rescript.json +4 -3
- package/src/{Address.res.js → Address.res.mjs} +7 -6
- package/src/Batch.res +4 -4
- package/src/{Batch.res.js → Batch.res.mjs} +19 -18
- package/src/{ChainMap.res.js → ChainMap.res.mjs} +20 -19
- package/src/Change.res +9 -0
- package/src/Config.res +5 -5
- package/src/{Config.res.js → Config.res.mjs} +13 -10
- package/src/Envio.gen.ts +4 -17
- package/src/Envio.res +13 -38
- package/src/{Envio.res.js → Envio.res.mjs} +5 -16
- package/src/{ErrorHandling.res.js → ErrorHandling.res.mjs} +10 -9
- package/src/EventRegister.res +10 -11
- package/src/{EventRegister.res.js → EventRegister.res.mjs} +33 -26
- package/src/EventRegister.resi +2 -1
- package/src/{EventUtils.res.js → EventUtils.res.mjs} +10 -9
- package/src/{EvmTypes.res.js → EvmTypes.res.mjs} +5 -4
- package/src/{FetchState.res.js → FetchState.res.mjs} +44 -43
- package/src/{Hasura.res.js → Hasura.res.mjs} +25 -24
- package/src/InMemoryStore.res +7 -15
- package/src/{InMemoryStore.res.js → InMemoryStore.res.mjs} +21 -22
- package/src/InMemoryTable.res +50 -35
- package/src/{InMemoryTable.res.js → InMemoryTable.res.mjs} +73 -104
- package/src/Internal.gen.ts +1 -5
- package/src/Internal.res +29 -40
- package/src/Internal.res.mjs +58 -0
- package/src/{LazyLoader.res.js → LazyLoader.res.mjs} +14 -13
- package/src/{LoadManager.res.js → LoadManager.res.mjs} +11 -10
- package/src/{LogSelection.res.js → LogSelection.res.mjs} +14 -13
- package/src/{Logging.res.js → Logging.res.mjs} +37 -36
- package/src/Persistence.res +190 -38
- package/src/Persistence.res.mjs +213 -0
- package/src/PgStorage.res +700 -14
- package/src/{PgStorage.res.js → PgStorage.res.mjs} +478 -65
- package/src/Platform.res +140 -0
- package/src/Platform.res.mjs +170 -0
- package/src/Prometheus.res +41 -0
- package/src/{Prometheus.res.js → Prometheus.res.mjs} +106 -60
- package/src/{ReorgDetection.res.js → ReorgDetection.res.mjs} +12 -11
- package/src/SafeCheckpointTracking.res +5 -4
- package/src/{SafeCheckpointTracking.res.js → SafeCheckpointTracking.res.mjs} +6 -5
- package/src/Sink.res +47 -0
- package/src/Sink.res.mjs +37 -0
- package/src/{TableIndices.res.js → TableIndices.res.mjs} +8 -7
- package/src/{Throttler.res.js → Throttler.res.mjs} +7 -6
- package/src/{Time.res.js → Time.res.mjs} +8 -7
- package/src/{TopicFilter.res.js → TopicFilter.res.mjs} +18 -17
- package/src/Utils.res +42 -0
- package/src/{Utils.res.js → Utils.res.mjs} +71 -34
- package/src/bindings/BigDecimal.gen.ts +2 -2
- package/src/bindings/BigDecimal.res +5 -5
- package/src/bindings/BigDecimal.res.d.mts +5 -0
- package/src/bindings/{BigDecimal.res.js → BigDecimal.res.mjs} +10 -9
- package/src/bindings/BigInt.gen.ts +2 -2
- package/src/bindings/BigInt.res.d.mts +5 -0
- package/src/bindings/{BigInt.res.js → BigInt.res.mjs} +25 -24
- package/src/bindings/ClickHouse.res +387 -0
- package/src/bindings/ClickHouse.res.mjs +275 -0
- package/src/bindings/Ethers.gen.ts +2 -2
- package/src/bindings/Ethers.res.d.mts +5 -0
- package/src/bindings/{Ethers.res.js → Ethers.res.mjs} +18 -17
- package/src/bindings/Express.res.mjs +2 -0
- package/src/bindings/{Hrtime.res.js → Hrtime.res.mjs} +13 -12
- package/src/bindings/{Lodash.res.js → Lodash.res.mjs} +4 -3
- package/src/bindings/NodeJs.res +13 -1
- package/src/bindings/NodeJs.res.mjs +48 -0
- package/src/bindings/Pino.res +4 -4
- package/src/bindings/{Pino.res.js → Pino.res.mjs} +16 -15
- package/src/bindings/Postgres.res +17 -2
- package/src/bindings/{Postgres.res.js → Postgres.res.mjs} +4 -3
- package/src/bindings/{PromClient.res.js → PromClient.res.mjs} +6 -5
- package/src/bindings/{Promise.res.js → Promise.res.mjs} +6 -5
- package/src/bindings/{SDSL.res.js → SDSL.res.mjs} +3 -2
- package/src/bindings/{Viem.res.js → Viem.res.mjs} +9 -8
- package/src/bindings/vendored-lodash-fns.js +3 -35
- package/src/db/EntityHistory.res +33 -156
- package/src/db/EntityHistory.res.mjs +121 -0
- package/src/db/InternalTable.gen.ts +2 -2
- package/src/db/InternalTable.res +55 -56
- package/src/db/{InternalTable.res.js → InternalTable.res.mjs} +68 -71
- package/src/db/{Schema.res.js → Schema.res.mjs} +10 -9
- package/src/db/Table.res +86 -22
- package/src/db/{Table.res.js → Table.res.mjs} +108 -40
- package/src/sources/{EventRouter.res.js → EventRouter.res.mjs} +17 -16
- package/src/sources/{Fuel.res.js → Fuel.res.mjs} +6 -5
- package/src/sources/{HyperFuel.res.js → HyperFuel.res.mjs} +14 -13
- package/src/sources/{HyperFuelClient.res.js → HyperFuelClient.res.mjs} +7 -6
- package/src/sources/{HyperFuelSource.res.js → HyperFuelSource.res.mjs} +28 -27
- package/src/sources/{HyperSync.res.js → HyperSync.res.mjs} +21 -20
- package/src/sources/HyperSyncClient.gen.ts +1 -1
- package/src/sources/HyperSyncClient.res +78 -20
- package/src/sources/{HyperSyncClient.res.js → HyperSyncClient.res.mjs} +32 -15
- package/src/sources/{HyperSyncJsonApi.res.js → HyperSyncJsonApi.res.mjs} +10 -9
- package/src/sources/HyperSyncSource.res +77 -9
- package/src/sources/{HyperSyncSource.res.js → HyperSyncSource.res.mjs} +99 -38
- package/src/sources/{Rpc.res.js → Rpc.res.mjs} +16 -15
- package/src/sources/{RpcSource.res.js → RpcSource.res.mjs} +40 -39
- package/src/sources/Source.res +1 -1
- package/src/sources/{Source.res.js → Source.res.mjs} +4 -3
- package/src/sources/SourceManager.res +12 -1
- package/src/sources/{SourceManager.res.js → SourceManager.res.mjs} +32 -21
- package/src/sources/vendored-fuel-abi-coder.js +94 -149
- package/src/vendored/{Rest.res.js → Rest.res.mjs} +12 -11
- package/src/Internal.res.js +0 -62
- package/src/Persistence.res.js +0 -159
- package/src/bindings/NodeJs.res.js +0 -35
- package/src/db/EntityHistory.res.js +0 -195
- /package/src/{Indexer.res.js → Change.res.mjs} +0 -0
- /package/src/{bindings/Express.res.js → Indexer.res.mjs} +0 -0
package/src/db/EntityHistory.res
CHANGED
|
@@ -5,41 +5,47 @@ module RowAction = {
|
|
|
5
5
|
let variants = [SET, DELETE]
|
|
6
6
|
let name = "ENVIO_HISTORY_CHANGE"
|
|
7
7
|
let schema = S.enum(variants)
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
type entityUpdate<'entityType> = {
|
|
15
|
-
entityId: string,
|
|
16
|
-
entityUpdateAction: entityUpdateAction<'entityType>,
|
|
17
|
-
checkpointId: int,
|
|
8
|
+
let config: Table.enumConfig<t> = {
|
|
9
|
+
name,
|
|
10
|
+
variants,
|
|
11
|
+
schema,
|
|
12
|
+
}
|
|
18
13
|
}
|
|
19
14
|
|
|
20
15
|
// Prefix with envio_ to avoid colleasions
|
|
21
16
|
let changeFieldName = "envio_change"
|
|
22
|
-
let checkpointIdFieldName = "
|
|
17
|
+
let checkpointIdFieldName = "envio_checkpoint_id"
|
|
18
|
+
let checkpointIdFieldType = Uint32
|
|
19
|
+
let changeFieldType = Enum({config: RowAction.config->Table.fromGenericEnumConfig})
|
|
20
|
+
|
|
21
|
+
let unsafeCheckpointIdSchema =
|
|
22
|
+
S.string
|
|
23
|
+
->S.setName("CheckpointId")
|
|
24
|
+
->S.transform(s => {
|
|
25
|
+
parser: string =>
|
|
26
|
+
switch string->Belt.Float.fromString {
|
|
27
|
+
| Some(float) => float
|
|
28
|
+
| None => s.fail("The string is not valid CheckpointId")
|
|
29
|
+
},
|
|
30
|
+
serializer: float => float->Belt.Float.toString,
|
|
31
|
+
})
|
|
23
32
|
|
|
24
|
-
let makeSetUpdateSchema: S.t<'entity> => S.t<
|
|
33
|
+
let makeSetUpdateSchema: S.t<'entity> => S.t<Change.t<'entity>> = entitySchema => {
|
|
25
34
|
S.object(s => {
|
|
26
35
|
s.tag(changeFieldName, RowAction.SET)
|
|
27
|
-
{
|
|
28
|
-
checkpointId: s.field(checkpointIdFieldName,
|
|
29
|
-
entityId: s.field(
|
|
30
|
-
|
|
31
|
-
}
|
|
36
|
+
Change.Set({
|
|
37
|
+
checkpointId: s.field(checkpointIdFieldName, unsafeCheckpointIdSchema),
|
|
38
|
+
entityId: s.field(Table.idFieldName, S.string),
|
|
39
|
+
entity: s.flatten(entitySchema),
|
|
40
|
+
})
|
|
32
41
|
})
|
|
33
42
|
}
|
|
34
43
|
|
|
35
|
-
type
|
|
36
|
-
table: table,
|
|
37
|
-
|
|
44
|
+
type pgEntityHistory<'entity> = {
|
|
45
|
+
table: Table.table,
|
|
46
|
+
setChangeSchema: S.t<Change.t<'entity>>,
|
|
38
47
|
// Used for parsing
|
|
39
|
-
|
|
40
|
-
makeInsertDeleteUpdatesQuery: (~pgSchema: string) => string,
|
|
41
|
-
makeGetRollbackRemovedIdsQuery: (~pgSchema: string) => string,
|
|
42
|
-
makeGetRollbackRestoredEntitiesQuery: (~pgSchema: string) => string,
|
|
48
|
+
setChangeSchemaRows: S.t<array<Change.t<'entity>>>,
|
|
43
49
|
}
|
|
44
50
|
|
|
45
51
|
let maxPgTableNameLength = 63
|
|
@@ -55,120 +61,6 @@ let historyTableName = (~entityName, ~entityIndex) => {
|
|
|
55
61
|
}
|
|
56
62
|
}
|
|
57
63
|
|
|
58
|
-
let fromTable = (table: table, ~schema: S.t<'entity>, ~entityIndex): t<'entity> => {
|
|
59
|
-
let id = "id"
|
|
60
|
-
|
|
61
|
-
let dataFields = table.fields->Belt.Array.keepMap(field =>
|
|
62
|
-
switch field {
|
|
63
|
-
| Field(field) =>
|
|
64
|
-
switch field.fieldName {
|
|
65
|
-
//id is not nullable and should be part of the pk
|
|
66
|
-
| "id" => {...field, fieldName: id, isPrimaryKey: true}->Field->Some
|
|
67
|
-
| _ =>
|
|
68
|
-
{
|
|
69
|
-
...field,
|
|
70
|
-
isNullable: true, //All entity fields are nullable in the case
|
|
71
|
-
isIndex: false, //No need to index any additional entity data fields in entity history
|
|
72
|
-
}
|
|
73
|
-
->Field
|
|
74
|
-
->Some
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
| DerivedFrom(_) => None
|
|
78
|
-
}
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
let actionField = mkField(changeFieldName, Custom(RowAction.name), ~fieldSchema=S.never)
|
|
82
|
-
|
|
83
|
-
let checkpointIdField = mkField(
|
|
84
|
-
checkpointIdFieldName,
|
|
85
|
-
Integer,
|
|
86
|
-
~fieldSchema=S.int,
|
|
87
|
-
~isPrimaryKey=true,
|
|
88
|
-
)
|
|
89
|
-
|
|
90
|
-
let entityTableName = table.tableName
|
|
91
|
-
let historyTableName = historyTableName(~entityName=entityTableName, ~entityIndex)
|
|
92
|
-
//ignore composite indices
|
|
93
|
-
let table = mkTable(
|
|
94
|
-
historyTableName,
|
|
95
|
-
~fields=dataFields->Belt.Array.concat([checkpointIdField, actionField]),
|
|
96
|
-
)
|
|
97
|
-
|
|
98
|
-
let setUpdateSchema = makeSetUpdateSchema(schema)
|
|
99
|
-
|
|
100
|
-
let makeInsertDeleteUpdatesQuery = {
|
|
101
|
-
// Get all field names for the INSERT statement
|
|
102
|
-
let allFieldNames = table.fields->Belt.Array.map(field => field->getFieldName)
|
|
103
|
-
let allFieldNamesStr =
|
|
104
|
-
allFieldNames->Belt.Array.map(name => `"${name}"`)->Js.Array2.joinWith(", ")
|
|
105
|
-
|
|
106
|
-
// Build the SELECT part: id from unnest, checkpoint_id from unnest, 'DELETE' for action, NULL for all other fields
|
|
107
|
-
let selectParts = allFieldNames->Belt.Array.map(fieldName => {
|
|
108
|
-
switch fieldName {
|
|
109
|
-
| "id" => "u.id"
|
|
110
|
-
| field if field == checkpointIdFieldName => "u.checkpoint_id"
|
|
111
|
-
| field if field == changeFieldName => "'DELETE'"
|
|
112
|
-
| _ => "NULL"
|
|
113
|
-
}
|
|
114
|
-
})
|
|
115
|
-
let selectPartsStr = selectParts->Js.Array2.joinWith(", ")
|
|
116
|
-
(~pgSchema) => {
|
|
117
|
-
`INSERT INTO "${pgSchema}"."${historyTableName}" (${allFieldNamesStr})
|
|
118
|
-
SELECT ${selectPartsStr}
|
|
119
|
-
FROM UNNEST($1::text[], $2::int[]) AS u(id, checkpoint_id)`
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
// Get data field names for rollback queries (exclude changeFieldName and checkpointIdFieldName)
|
|
124
|
-
let dataFieldNames =
|
|
125
|
-
table.fields
|
|
126
|
-
->Belt.Array.map(field => field->getFieldName)
|
|
127
|
-
->Belt.Array.keep(fieldName =>
|
|
128
|
-
fieldName != changeFieldName && fieldName != checkpointIdFieldName
|
|
129
|
-
)
|
|
130
|
-
let dataFieldsCommaSeparated =
|
|
131
|
-
dataFieldNames->Belt.Array.map(name => `"${name}"`)->Js.Array2.joinWith(", ")
|
|
132
|
-
|
|
133
|
-
// Returns entity IDs that were created after the rollback target and have no history before it.
|
|
134
|
-
// These entities should be deleted during rollback.
|
|
135
|
-
let makeGetRollbackRemovedIdsQuery = (~pgSchema) => {
|
|
136
|
-
`SELECT DISTINCT id
|
|
137
|
-
FROM "${pgSchema}"."${historyTableName}"
|
|
138
|
-
WHERE "${checkpointIdFieldName}" > $1
|
|
139
|
-
AND NOT EXISTS (
|
|
140
|
-
SELECT 1
|
|
141
|
-
FROM "${pgSchema}"."${historyTableName}" h
|
|
142
|
-
WHERE h.id = "${historyTableName}".id
|
|
143
|
-
AND h."${checkpointIdFieldName}" <= $1
|
|
144
|
-
)`
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
// Returns the most recent entity state for IDs that need to be restored during rollback.
|
|
148
|
-
// For each ID modified after the rollback target, retrieves its latest state at or before the target.
|
|
149
|
-
let makeGetRollbackRestoredEntitiesQuery = (~pgSchema) => {
|
|
150
|
-
`SELECT DISTINCT ON (id) ${dataFieldsCommaSeparated}
|
|
151
|
-
FROM "${pgSchema}"."${historyTableName}"
|
|
152
|
-
WHERE "${checkpointIdFieldName}" <= $1
|
|
153
|
-
AND EXISTS (
|
|
154
|
-
SELECT 1
|
|
155
|
-
FROM "${pgSchema}"."${historyTableName}" h
|
|
156
|
-
WHERE h.id = "${historyTableName}".id
|
|
157
|
-
AND h."${checkpointIdFieldName}" > $1
|
|
158
|
-
)
|
|
159
|
-
ORDER BY id, "${checkpointIdFieldName}" DESC`
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
{
|
|
163
|
-
table,
|
|
164
|
-
setUpdateSchema,
|
|
165
|
-
setUpdateSchemaRows: S.array(setUpdateSchema),
|
|
166
|
-
makeInsertDeleteUpdatesQuery,
|
|
167
|
-
makeGetRollbackRemovedIdsQuery,
|
|
168
|
-
makeGetRollbackRestoredEntitiesQuery,
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
64
|
type safeReorgBlocks = {
|
|
173
65
|
chainIds: array<int>,
|
|
174
66
|
blockNumbers: array<int>,
|
|
@@ -225,11 +117,11 @@ let pruneStaleEntityHistory = (
|
|
|
225
117
|
}
|
|
226
118
|
|
|
227
119
|
// If an entity doesn't have a history before the update
|
|
228
|
-
// we create it automatically with
|
|
120
|
+
// we create it automatically with envio_checkpoint_id 0
|
|
229
121
|
let makeBackfillHistoryQuery = (~pgSchema, ~entityName, ~entityIndex) => {
|
|
230
122
|
let historyTableRef = `"${pgSchema}"."${historyTableName(~entityName, ~entityIndex)}"`
|
|
231
123
|
`WITH target_ids AS (
|
|
232
|
-
SELECT UNNEST($1::${(Text:
|
|
124
|
+
SELECT UNNEST($1::${(Text: Postgres.columnType :> string)}[]) AS id
|
|
233
125
|
),
|
|
234
126
|
missing_history AS (
|
|
235
127
|
SELECT e.*
|
|
@@ -252,22 +144,7 @@ let backfillHistory = (sql, ~pgSchema, ~entityName, ~entityIndex, ~ids: array<st
|
|
|
252
144
|
->Promise.ignoreValue
|
|
253
145
|
}
|
|
254
146
|
|
|
255
|
-
let
|
|
256
|
-
sql,
|
|
257
|
-
~pgSchema,
|
|
258
|
-
~entityHistory,
|
|
259
|
-
~batchDeleteEntityIds,
|
|
260
|
-
~batchDeleteCheckpointIds,
|
|
261
|
-
) => {
|
|
262
|
-
sql
|
|
263
|
-
->Postgres.preparedUnsafe(
|
|
264
|
-
entityHistory.makeInsertDeleteUpdatesQuery(~pgSchema),
|
|
265
|
-
(batchDeleteEntityIds, batchDeleteCheckpointIds)->Obj.magic,
|
|
266
|
-
)
|
|
267
|
-
->Promise.ignoreValue
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
let rollback = (sql, ~pgSchema, ~entityName, ~entityIndex, ~rollbackTargetCheckpointId: int) => {
|
|
147
|
+
let rollback = (sql, ~pgSchema, ~entityName, ~entityIndex, ~rollbackTargetCheckpointId: float) => {
|
|
271
148
|
sql
|
|
272
149
|
->Postgres.preparedUnsafe(
|
|
273
150
|
`DELETE FROM "${pgSchema}"."${historyTableName(
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Table from "./Table.res.mjs";
|
|
4
|
+
import * as Js_string from "rescript/lib/es6/js_string.js";
|
|
5
|
+
import * as Belt_Float from "rescript/lib/es6/belt_Float.js";
|
|
6
|
+
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
7
|
+
|
|
8
|
+
var variants = [
|
|
9
|
+
"SET",
|
|
10
|
+
"DELETE"
|
|
11
|
+
];
|
|
12
|
+
|
|
13
|
+
var name = "ENVIO_HISTORY_CHANGE";
|
|
14
|
+
|
|
15
|
+
var schema = S$RescriptSchema.$$enum(variants);
|
|
16
|
+
|
|
17
|
+
var config = {
|
|
18
|
+
name: name,
|
|
19
|
+
variants: variants,
|
|
20
|
+
schema: schema
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
var RowAction = {
|
|
24
|
+
variants: variants,
|
|
25
|
+
name: name,
|
|
26
|
+
schema: schema,
|
|
27
|
+
config: config
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
var changeFieldName = "envio_change";
|
|
31
|
+
|
|
32
|
+
var checkpointIdFieldName = "envio_checkpoint_id";
|
|
33
|
+
|
|
34
|
+
var changeFieldType = {
|
|
35
|
+
type: "Enum",
|
|
36
|
+
config: config
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
var unsafeCheckpointIdSchema = S$RescriptSchema.transform(S$RescriptSchema.setName(S$RescriptSchema.string, "CheckpointId"), (function (s) {
|
|
40
|
+
return {
|
|
41
|
+
p: (function (string) {
|
|
42
|
+
var $$float = Belt_Float.fromString(string);
|
|
43
|
+
if ($$float !== undefined) {
|
|
44
|
+
return $$float;
|
|
45
|
+
} else {
|
|
46
|
+
return s.fail("The string is not valid CheckpointId", undefined);
|
|
47
|
+
}
|
|
48
|
+
}),
|
|
49
|
+
s: (function ($$float) {
|
|
50
|
+
return String($$float);
|
|
51
|
+
})
|
|
52
|
+
};
|
|
53
|
+
}));
|
|
54
|
+
|
|
55
|
+
function makeSetUpdateSchema(entitySchema) {
|
|
56
|
+
return S$RescriptSchema.object(function (s) {
|
|
57
|
+
s.tag(changeFieldName, "SET");
|
|
58
|
+
return {
|
|
59
|
+
type: "SET",
|
|
60
|
+
entityId: s.f(Table.idFieldName, S$RescriptSchema.string),
|
|
61
|
+
entity: s.flatten(entitySchema),
|
|
62
|
+
checkpointId: s.f(checkpointIdFieldName, unsafeCheckpointIdSchema)
|
|
63
|
+
};
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
var historyTablePrefix = "envio_history_";
|
|
68
|
+
|
|
69
|
+
function historyTableName(entityName, entityIndex) {
|
|
70
|
+
var fullName = historyTablePrefix + entityName;
|
|
71
|
+
if (fullName.length <= 63) {
|
|
72
|
+
return fullName;
|
|
73
|
+
}
|
|
74
|
+
var entityIndexStr = String(entityIndex);
|
|
75
|
+
return Js_string.slice(0, 63 - entityIndexStr.length | 0, fullName) + entityIndexStr;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema) {
|
|
79
|
+
var historyTableRef = "\"" + pgSchema + "\".\"" + historyTableName(entityName, entityIndex) + "\"";
|
|
80
|
+
return "WITH anchors AS (\n SELECT t.id, MAX(t." + checkpointIdFieldName + ") AS keep_checkpoint_id\n FROM " + historyTableRef + " t WHERE t." + checkpointIdFieldName + " <= $1\n GROUP BY t.id\n)\nDELETE FROM " + historyTableRef + " d\nUSING anchors a\nWHERE d.id = a.id\n AND (\n d." + checkpointIdFieldName + " < a.keep_checkpoint_id\n OR (\n d." + checkpointIdFieldName + " = a.keep_checkpoint_id AND\n NOT EXISTS (\n SELECT 1 FROM " + historyTableRef + " ps \n WHERE ps.id = d.id AND ps." + checkpointIdFieldName + " > $1\n ) \n )\n );";
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, safeCheckpointId) {
|
|
84
|
+
return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema), [safeCheckpointId], {prepare: true});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function makeBackfillHistoryQuery(pgSchema, entityName, entityIndex) {
|
|
88
|
+
var historyTableRef = "\"" + pgSchema + "\".\"" + historyTableName(entityName, entityIndex) + "\"";
|
|
89
|
+
return "WITH target_ids AS (\n SELECT UNNEST($1::TEXT[]) AS id\n),\nmissing_history AS (\n SELECT e.*\n FROM \"" + pgSchema + "\".\"" + entityName + "\" e\n JOIN target_ids t ON e.id = t.id\n LEFT JOIN " + historyTableRef + " h ON h.id = e.id\n WHERE h.id IS NULL\n)\nINSERT INTO " + historyTableRef + "\nSELECT *, 0 AS " + checkpointIdFieldName + ", '" + "SET" + "' as " + changeFieldName + "\nFROM missing_history;";
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function backfillHistory(sql, pgSchema, entityName, entityIndex, ids) {
|
|
93
|
+
return sql.unsafe(makeBackfillHistoryQuery(pgSchema, entityName, entityIndex), [ids], {prepare: true});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function rollback(sql, pgSchema, entityName, entityIndex, rollbackTargetCheckpointId) {
|
|
97
|
+
return sql.unsafe("DELETE FROM \"" + pgSchema + "\".\"" + historyTableName(entityName, entityIndex) + "\" WHERE \"" + checkpointIdFieldName + "\" > $1;", [rollbackTargetCheckpointId], {prepare: true});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
var checkpointIdFieldType = "Uint32";
|
|
101
|
+
|
|
102
|
+
var maxPgTableNameLength = 63;
|
|
103
|
+
|
|
104
|
+
export {
|
|
105
|
+
RowAction ,
|
|
106
|
+
changeFieldName ,
|
|
107
|
+
checkpointIdFieldName ,
|
|
108
|
+
checkpointIdFieldType ,
|
|
109
|
+
changeFieldType ,
|
|
110
|
+
unsafeCheckpointIdSchema ,
|
|
111
|
+
makeSetUpdateSchema ,
|
|
112
|
+
maxPgTableNameLength ,
|
|
113
|
+
historyTablePrefix ,
|
|
114
|
+
historyTableName ,
|
|
115
|
+
makePruneStaleEntityHistoryQuery ,
|
|
116
|
+
pruneStaleEntityHistory ,
|
|
117
|
+
makeBackfillHistoryQuery ,
|
|
118
|
+
backfillHistory ,
|
|
119
|
+
rollback ,
|
|
120
|
+
}
|
|
121
|
+
/* schema Not a pure module */
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
/* eslint-disable */
|
|
4
4
|
/* tslint:disable */
|
|
5
5
|
|
|
6
|
-
import type {Json_t as Js_Json_t} from '../../src/Js.shim';
|
|
6
|
+
import type {Json_t as Js_Json_t} from '../../src/Js.shim.js';
|
|
7
7
|
|
|
8
|
-
import type {t as Address_t} from '../../src/Address.gen';
|
|
8
|
+
import type {t as Address_t} from '../../src/Address.gen.js';
|
|
9
9
|
|
|
10
10
|
export type DynamicContractRegistry_t = {
|
|
11
11
|
readonly id: string;
|
package/src/db/InternalTable.res
CHANGED
|
@@ -46,21 +46,19 @@ module DynamicContractRegistry = {
|
|
|
46
46
|
let table = mkTable(
|
|
47
47
|
name,
|
|
48
48
|
~fields=[
|
|
49
|
-
mkField("id",
|
|
50
|
-
mkField("chain_id",
|
|
51
|
-
mkField("registering_event_block_number",
|
|
52
|
-
mkField("registering_event_log_index",
|
|
53
|
-
mkField("registering_event_block_timestamp",
|
|
54
|
-
mkField("registering_event_contract_name",
|
|
55
|
-
mkField("registering_event_name",
|
|
56
|
-
mkField("registering_event_src_address",
|
|
57
|
-
mkField("contract_address",
|
|
58
|
-
mkField("contract_name",
|
|
49
|
+
mkField("id", String, ~isPrimaryKey, ~fieldSchema=S.string),
|
|
50
|
+
mkField("chain_id", Int32, ~fieldSchema=S.int),
|
|
51
|
+
mkField("registering_event_block_number", Int32, ~fieldSchema=S.int),
|
|
52
|
+
mkField("registering_event_log_index", Int32, ~fieldSchema=S.int),
|
|
53
|
+
mkField("registering_event_block_timestamp", Int32, ~fieldSchema=S.int),
|
|
54
|
+
mkField("registering_event_contract_name", String, ~fieldSchema=S.string),
|
|
55
|
+
mkField("registering_event_name", String, ~fieldSchema=S.string),
|
|
56
|
+
mkField("registering_event_src_address", String, ~fieldSchema=Address.schema),
|
|
57
|
+
mkField("contract_address", String, ~fieldSchema=Address.schema),
|
|
58
|
+
mkField("contract_name", String, ~fieldSchema=S.string),
|
|
59
59
|
],
|
|
60
60
|
)
|
|
61
61
|
|
|
62
|
-
let entityHistory = table->EntityHistory.fromTable(~schema, ~entityIndex=index)
|
|
63
|
-
|
|
64
62
|
external castToInternal: t => Internal.entity = "%identity"
|
|
65
63
|
|
|
66
64
|
let config = {
|
|
@@ -69,7 +67,6 @@ module DynamicContractRegistry = {
|
|
|
69
67
|
schema,
|
|
70
68
|
rowsSchema,
|
|
71
69
|
table,
|
|
72
|
-
entityHistory,
|
|
73
70
|
}->Internal.fromGenericEntityConfig
|
|
74
71
|
}
|
|
75
72
|
|
|
@@ -131,19 +128,19 @@ module Chains = {
|
|
|
131
128
|
let table = mkTable(
|
|
132
129
|
"envio_chains",
|
|
133
130
|
~fields=[
|
|
134
|
-
mkField((#id: field :> string),
|
|
131
|
+
mkField((#id: field :> string), Int32, ~fieldSchema=S.int, ~isPrimaryKey),
|
|
135
132
|
// Values populated from config
|
|
136
|
-
mkField((#start_block: field :> string),
|
|
137
|
-
mkField((#end_block: field :> string),
|
|
138
|
-
mkField((#max_reorg_depth: field :> string),
|
|
133
|
+
mkField((#start_block: field :> string), Int32, ~fieldSchema=S.int),
|
|
134
|
+
mkField((#end_block: field :> string), Int32, ~fieldSchema=S.null(S.int), ~isNullable),
|
|
135
|
+
mkField((#max_reorg_depth: field :> string), Int32, ~fieldSchema=S.int),
|
|
139
136
|
// Block number of the latest block that was fetched from the source
|
|
140
|
-
mkField((#buffer_block: field :> string),
|
|
137
|
+
mkField((#buffer_block: field :> string), Int32, ~fieldSchema=S.int),
|
|
141
138
|
// Block number of the currently active source
|
|
142
|
-
mkField((#source_block: field :> string),
|
|
139
|
+
mkField((#source_block: field :> string), Int32, ~fieldSchema=S.int),
|
|
143
140
|
// Block number of the first event that was processed for this chain
|
|
144
141
|
mkField(
|
|
145
142
|
(#first_event_block: field :> string),
|
|
146
|
-
|
|
143
|
+
Int32,
|
|
147
144
|
~fieldSchema=S.null(S.int),
|
|
148
145
|
~isNullable,
|
|
149
146
|
),
|
|
@@ -151,17 +148,17 @@ module Chains = {
|
|
|
151
148
|
// null during historical sync, set to current time when sync is complete
|
|
152
149
|
mkField(
|
|
153
150
|
(#ready_at: field :> string),
|
|
154
|
-
|
|
151
|
+
Date,
|
|
155
152
|
~fieldSchema=S.null(Utils.Schema.dbDate),
|
|
156
153
|
~isNullable,
|
|
157
154
|
),
|
|
158
|
-
mkField((#events_processed: field :> string),
|
|
155
|
+
mkField((#events_processed: field :> string), Int32, ~fieldSchema=S.int),
|
|
159
156
|
// TODO: In the future it should reference a table with sources
|
|
160
157
|
mkField((#_is_hyper_sync: field :> string), Boolean, ~fieldSchema=S.bool),
|
|
161
158
|
// Fully processed block number
|
|
162
|
-
mkField((#progress_block: field :> string),
|
|
159
|
+
mkField((#progress_block: field :> string), Int32, ~fieldSchema=S.int),
|
|
163
160
|
// TODO: Should deprecate after changing the ETA calculation logic
|
|
164
|
-
mkField((#_num_batches_fetched: field :> string),
|
|
161
|
+
mkField((#_num_batches_fetched: field :> string), Int32, ~fieldSchema=S.int),
|
|
165
162
|
],
|
|
166
163
|
)
|
|
167
164
|
|
|
@@ -360,7 +357,6 @@ module PersistedState = {
|
|
|
360
357
|
envio_version: string,
|
|
361
358
|
config_hash: string,
|
|
362
359
|
schema_hash: string,
|
|
363
|
-
handler_files_hash: string,
|
|
364
360
|
abi_files_hash: string,
|
|
365
361
|
}
|
|
366
362
|
|
|
@@ -368,11 +364,10 @@ module PersistedState = {
|
|
|
368
364
|
"persisted_state",
|
|
369
365
|
~fields=[
|
|
370
366
|
mkField("id", Serial, ~fieldSchema=S.int, ~isPrimaryKey),
|
|
371
|
-
mkField("envio_version",
|
|
372
|
-
mkField("config_hash",
|
|
373
|
-
mkField("schema_hash",
|
|
374
|
-
mkField("
|
|
375
|
-
mkField("abi_files_hash", Text, ~fieldSchema=S.string),
|
|
367
|
+
mkField("envio_version", String, ~fieldSchema=S.string),
|
|
368
|
+
mkField("config_hash", String, ~fieldSchema=S.string),
|
|
369
|
+
mkField("schema_hash", String, ~fieldSchema=S.string),
|
|
370
|
+
mkField("abi_files_hash", String, ~fieldSchema=S.string),
|
|
376
371
|
],
|
|
377
372
|
)
|
|
378
373
|
}
|
|
@@ -387,7 +382,7 @@ module Checkpoints = {
|
|
|
387
382
|
]
|
|
388
383
|
|
|
389
384
|
type t = {
|
|
390
|
-
id:
|
|
385
|
+
id: float,
|
|
391
386
|
@as("chain_id")
|
|
392
387
|
chainId: int,
|
|
393
388
|
@as("block_number")
|
|
@@ -398,16 +393,16 @@ module Checkpoints = {
|
|
|
398
393
|
eventsProcessed: int,
|
|
399
394
|
}
|
|
400
395
|
|
|
401
|
-
let initialCheckpointId = 0
|
|
396
|
+
let initialCheckpointId = 0.
|
|
402
397
|
|
|
403
398
|
let table = mkTable(
|
|
404
399
|
"envio_checkpoints",
|
|
405
400
|
~fields=[
|
|
406
|
-
mkField((#id: field :> string),
|
|
407
|
-
mkField((#chain_id: field :> string),
|
|
408
|
-
mkField((#block_number: field :> string),
|
|
409
|
-
mkField((#block_hash: field :> string),
|
|
410
|
-
mkField((#events_processed: field :> string),
|
|
401
|
+
mkField((#id: field :> string), Int32, ~fieldSchema=S.int, ~isPrimaryKey),
|
|
402
|
+
mkField((#chain_id: field :> string), Int32, ~fieldSchema=S.int),
|
|
403
|
+
mkField((#block_number: field :> string), Int32, ~fieldSchema=S.int),
|
|
404
|
+
mkField((#block_hash: field :> string), String, ~fieldSchema=S.null(S.string), ~isNullable),
|
|
405
|
+
mkField((#events_processed: field :> string), Int32, ~fieldSchema=S.int),
|
|
411
406
|
],
|
|
412
407
|
)
|
|
413
408
|
|
|
@@ -438,12 +433,12 @@ WHERE cp."${(#block_hash: field :> string)}" IS NOT NULL
|
|
|
438
433
|
}
|
|
439
434
|
|
|
440
435
|
let makeCommitedCheckpointIdQuery = (~pgSchema) => {
|
|
441
|
-
`SELECT COALESCE(MAX(${(#id: field :> string)}), ${initialCheckpointId->Belt.
|
|
436
|
+
`SELECT COALESCE(MAX(${(#id: field :> string)}), ${initialCheckpointId->Belt.Float.toString}) AS id FROM "${pgSchema}"."${table.tableName}";`
|
|
442
437
|
}
|
|
443
438
|
|
|
444
439
|
let makeInsertCheckpointQuery = (~pgSchema) => {
|
|
445
440
|
`INSERT INTO "${pgSchema}"."${table.tableName}" ("${(#id: field :> string)}", "${(#chain_id: field :> string)}", "${(#block_number: field :> string)}", "${(#block_hash: field :> string)}", "${(#events_processed: field :> string)}")
|
|
446
|
-
SELECT * FROM unnest($1::${(Integer :> string)}[],$2::${(Integer :> string)}[],$3::${(Integer :> string)}[],$4::${(Text :> string)}[],$5::${(Integer :> string)}[]);`
|
|
441
|
+
SELECT * FROM unnest($1::${(Integer: Postgres.columnType :> string)}[],$2::${(Integer: Postgres.columnType :> string)}[],$3::${(Integer: Postgres.columnType :> string)}[],$4::${(Text: Postgres.columnType :> string)}[],$5::${(Integer: Postgres.columnType :> string)}[]);`
|
|
447
442
|
}
|
|
448
443
|
|
|
449
444
|
let insert = (
|
|
@@ -468,14 +463,14 @@ SELECT * FROM unnest($1::${(Integer :> string)}[],$2::${(Integer :> string)}[],$
|
|
|
468
463
|
checkpointEventsProcessed,
|
|
469
464
|
)->(
|
|
470
465
|
Utils.magic: (
|
|
471
|
-
(array<
|
|
466
|
+
(array<float>, array<int>, array<int>, array<Js.Null.t<string>>, array<int>)
|
|
472
467
|
) => unknown
|
|
473
468
|
),
|
|
474
469
|
)
|
|
475
470
|
->Promise.ignoreValue
|
|
476
471
|
}
|
|
477
472
|
|
|
478
|
-
let rollback = (sql, ~pgSchema, ~rollbackTargetCheckpointId:
|
|
473
|
+
let rollback = (sql, ~pgSchema, ~rollbackTargetCheckpointId: Internal.checkpointId) => {
|
|
479
474
|
sql
|
|
480
475
|
->Postgres.preparedUnsafe(
|
|
481
476
|
`DELETE FROM "${pgSchema}"."${table.tableName}" WHERE "${(#id: field :> string)}" > $1;`,
|
|
@@ -488,7 +483,7 @@ SELECT * FROM unnest($1::${(Integer :> string)}[],$2::${(Integer :> string)}[],$
|
|
|
488
483
|
`DELETE FROM "${pgSchema}"."${table.tableName}" WHERE "${(#id: field :> string)}" < $1;`
|
|
489
484
|
}
|
|
490
485
|
|
|
491
|
-
let pruneStaleCheckpoints = (sql, ~pgSchema, ~safeCheckpointId:
|
|
486
|
+
let pruneStaleCheckpoints = (sql, ~pgSchema, ~safeCheckpointId: float) => {
|
|
492
487
|
sql
|
|
493
488
|
->Postgres.preparedUnsafe(
|
|
494
489
|
makePruneStaleCheckpointsQuery(~pgSchema),
|
|
@@ -517,7 +512,7 @@ LIMIT 1;`
|
|
|
517
512
|
makeGetRollbackTargetCheckpointQuery(~pgSchema),
|
|
518
513
|
(reorgChainId, lastKnownValidBlockNumber)->Obj.magic,
|
|
519
514
|
)
|
|
520
|
-
->(Utils.magic: promise<unknown> => promise<array<{"id":
|
|
515
|
+
->(Utils.magic: promise<unknown> => promise<array<{"id": Internal.checkpointId}>>)
|
|
521
516
|
}
|
|
522
517
|
|
|
523
518
|
let makeGetRollbackProgressDiffQuery = (~pgSchema) => {
|
|
@@ -530,7 +525,11 @@ WHERE "${(#id: field :> string)}" > $1
|
|
|
530
525
|
GROUP BY "${(#chain_id: field :> string)}";`
|
|
531
526
|
}
|
|
532
527
|
|
|
533
|
-
let getRollbackProgressDiff = (
|
|
528
|
+
let getRollbackProgressDiff = (
|
|
529
|
+
sql,
|
|
530
|
+
~pgSchema,
|
|
531
|
+
~rollbackTargetCheckpointId: Internal.checkpointId,
|
|
532
|
+
) => {
|
|
534
533
|
sql
|
|
535
534
|
->Postgres.preparedUnsafe(
|
|
536
535
|
makeGetRollbackProgressDiffQuery(~pgSchema),
|
|
@@ -584,18 +583,18 @@ module RawEvents = {
|
|
|
584
583
|
let table = mkTable(
|
|
585
584
|
"raw_events",
|
|
586
585
|
~fields=[
|
|
587
|
-
mkField("chain_id",
|
|
588
|
-
mkField("event_id",
|
|
589
|
-
mkField("event_name",
|
|
590
|
-
mkField("contract_name",
|
|
591
|
-
mkField("block_number",
|
|
592
|
-
mkField("log_index",
|
|
593
|
-
mkField("src_address",
|
|
594
|
-
mkField("block_hash",
|
|
595
|
-
mkField("block_timestamp",
|
|
596
|
-
mkField("block_fields",
|
|
597
|
-
mkField("transaction_fields",
|
|
598
|
-
mkField("params",
|
|
586
|
+
mkField("chain_id", Int32, ~fieldSchema=S.int),
|
|
587
|
+
mkField("event_id", BigInt({}), ~fieldSchema=S.bigint),
|
|
588
|
+
mkField("event_name", String, ~fieldSchema=S.string),
|
|
589
|
+
mkField("contract_name", String, ~fieldSchema=S.string),
|
|
590
|
+
mkField("block_number", Int32, ~fieldSchema=S.int),
|
|
591
|
+
mkField("log_index", Int32, ~fieldSchema=S.int),
|
|
592
|
+
mkField("src_address", String, ~fieldSchema=Address.schema),
|
|
593
|
+
mkField("block_hash", String, ~fieldSchema=S.string),
|
|
594
|
+
mkField("block_timestamp", Int32, ~fieldSchema=S.int),
|
|
595
|
+
mkField("block_fields", Json, ~fieldSchema=S.json(~validate=false)),
|
|
596
|
+
mkField("transaction_fields", Json, ~fieldSchema=S.json(~validate=false)),
|
|
597
|
+
mkField("params", Json, ~fieldSchema=S.json(~validate=false)),
|
|
599
598
|
mkField("serial", Serial, ~isNullable, ~isPrimaryKey, ~fieldSchema=S.null(S.int)),
|
|
600
599
|
],
|
|
601
600
|
)
|