envio 3.9.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +26 -0
- package/fuel.schema.json +26 -0
- package/index.d.ts +35 -3
- package/package.json +6 -7
- package/src/Batch.res +9 -0
- package/src/ChainState.res +35 -39
- package/src/ChainState.res.mjs +14 -11
- package/src/ChainState.resi +15 -7
- package/src/Config.res +52 -44
- package/src/Config.res.mjs +51 -17
- package/src/Core.res +4 -0
- package/src/CrossChainState.res +15 -23
- package/src/CrossChainState.res.mjs +10 -18
- package/src/CrossChainState.resi +4 -1
- package/src/Ecosystem.res +7 -2
- package/src/Envio.res +6 -4
- package/src/EventConfigBuilder.res +169 -26
- package/src/EventConfigBuilder.res.mjs +124 -15
- package/src/HandlerRegister.res +21 -4
- package/src/HandlerRegister.res.mjs +24 -5
- package/src/InMemoryStore.res +9 -7
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/IndexerState.res +43 -6
- package/src/IndexerState.res.mjs +36 -7
- package/src/IndexerState.resi +4 -4
- package/src/Internal.res +16 -17
- package/src/Metrics.res +72 -6
- package/src/Metrics.res.mjs +30 -2
- package/src/Persistence.res +36 -30
- package/src/Persistence.res.mjs +1 -20
- package/src/PgStorage.res +104 -56
- package/src/PgStorage.res.mjs +57 -35
- package/src/PruneStaleHistory.res +85 -52
- package/src/PruneStaleHistory.res.mjs +63 -37
- package/src/Rollback.res +68 -45
- package/src/Rollback.res.mjs +38 -19
- package/src/RollbackCommit.res +7 -9
- package/src/RollbackCommit.res.mjs +5 -6
- package/src/SafeCheckpointTracking.res +11 -11
- package/src/SafeCheckpointTracking.res.mjs +7 -6
- package/src/SimulateItems.res +33 -18
- package/src/SimulateItems.res.mjs +26 -22
- package/src/Sink.res +38 -26
- package/src/Sink.res.mjs +21 -16
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +1 -1
- package/src/Utils.res +170 -0
- package/src/Utils.res.mjs +153 -0
- package/src/Writing.res +2 -2
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +366 -698
- package/src/bindings/ClickHouse.res.mjs +371 -421
- package/src/bindings/ClickHouseSink.res +337 -0
- package/src/bindings/ClickHouseSink.res.mjs +246 -0
- package/src/bindings/EventSource.res +8 -2
- package/src/bindings/NodeJs.res +6 -0
- package/src/bindings/Vitest.res +23 -0
- package/src/bindings/Vitest.res.mjs +3 -0
- package/src/bindings/WebSocket.res +9 -10
- package/src/db/CheckpointBounds.res +53 -0
- package/src/db/CheckpointBounds.res.mjs +44 -0
- package/src/db/EntityFilter.res +36 -17
- package/src/db/EntityFilter.res.mjs +25 -14
- package/src/db/EntityHistory.res +47 -22
- package/src/db/EntityHistory.res.mjs +19 -12
- package/src/db/InternalTable.res +84 -35
- package/src/db/InternalTable.res.mjs +56 -23
- package/src/db/RollbackFloors.res +55 -0
- package/src/db/RollbackFloors.res.mjs +92 -0
- package/src/db/Table.res +22 -4
- package/src/db/Table.res.mjs +30 -9
- package/src/sources/Evm.res +2 -0
- package/src/sources/Evm.res.mjs +2 -0
- package/src/sources/EvmHyperSyncSource.res +3 -3
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcWs.res +124 -0
- package/src/sources/EvmRpcWs.res.mjs +117 -0
- package/src/sources/Fuel.res +2 -0
- package/src/sources/Fuel.res.mjs +2 -0
- package/src/sources/FuelHyperSyncSource.res +1 -1
- package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
- package/src/sources/HeightFeed.res +568 -0
- package/src/sources/HeightFeed.res.mjs +452 -0
- package/src/sources/HeightStream.res +257 -0
- package/src/sources/HeightStream.res.mjs +194 -0
- package/src/sources/HyperSync.res +5 -0
- package/src/sources/HyperSync.res.mjs +3 -0
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncSSE.res +49 -0
- package/src/sources/HyperSyncSSE.res.mjs +79 -0
- package/src/sources/RpcSource.res +1 -1
- package/src/sources/RpcSource.res.mjs +2 -2
- package/src/sources/Source.res +54 -1
- package/src/sources/Source.res.mjs +25 -0
- package/src/sources/SourceManager.res +220 -227
- package/src/sources/SourceManager.res.mjs +139 -164
- package/src/sources/SourceManager.resi +8 -0
- package/src/sources/Svm.res +2 -0
- package/src/sources/Svm.res.mjs +2 -0
- package/src/sources/SvmHyperSyncClient.res +70 -127
- package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
- package/src/sources/SvmHyperSyncSource.res +18 -15
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/svm.schema.json +1 -75
- package/src/MemoryStorage.res +0 -729
- package/src/MemoryStorage.res.mjs +0 -586
- package/src/sources/HyperSyncHeightStream.res +0 -129
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
- package/src/sources/RpcWebSocketHeightStream.res +0 -175
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
|
@@ -1,586 +0,0 @@
|
|
|
1
|
-
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
-
|
|
3
|
-
import * as Table from "./db/Table.res.mjs";
|
|
4
|
-
import * as Utils from "./Utils.res.mjs";
|
|
5
|
-
import * as Config from "./Config.res.mjs";
|
|
6
|
-
import * as ChainId from "./ChainId.res.mjs";
|
|
7
|
-
import * as EntityId from "./EntityId.res.mjs";
|
|
8
|
-
import * as Internal from "./Internal.res.mjs";
|
|
9
|
-
import * as ChainState from "./ChainState.res.mjs";
|
|
10
|
-
import * as AddressRows from "./AddressRows.res.mjs";
|
|
11
|
-
import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
|
|
12
|
-
import * as EntityFilter from "./db/EntityFilter.res.mjs";
|
|
13
|
-
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
14
|
-
import * as InternalTable from "./db/InternalTable.res.mjs";
|
|
15
|
-
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
16
|
-
import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
|
|
17
|
-
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
18
|
-
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
19
|
-
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
20
|
-
|
|
21
|
-
function make() {
|
|
22
|
-
return {
|
|
23
|
-
entities: {},
|
|
24
|
-
entityConfigs: {},
|
|
25
|
-
history: {},
|
|
26
|
-
checkpoints: [],
|
|
27
|
-
chains: {},
|
|
28
|
-
cache: {},
|
|
29
|
-
effectCache: {},
|
|
30
|
-
envioInfo: undefined,
|
|
31
|
-
addresses: AddressRows.Table.make(),
|
|
32
|
-
isInitialized: false
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function rowKey(scope, entityId) {
|
|
37
|
-
if (scope === "crossChain") {
|
|
38
|
-
return EntityId.toKey(entityId);
|
|
39
|
-
} else {
|
|
40
|
-
return ChainId.toString(scope) + `|` + EntityId.toKey(entityId);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
let historyKey = rowKey;
|
|
45
|
-
|
|
46
|
-
function readChainId(entity, field) {
|
|
47
|
-
return entity[field.fieldName];
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function copyEntity(entity) {
|
|
51
|
-
return Utils.Dict.shallowCopy(entity);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function getEntityDict(state, name) {
|
|
55
|
-
let dict = state.entities[name];
|
|
56
|
-
if (dict !== undefined) {
|
|
57
|
-
return dict;
|
|
58
|
-
}
|
|
59
|
-
let dict$1 = {};
|
|
60
|
-
state.entities[name] = dict$1;
|
|
61
|
-
return dict$1;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
function getHistory(state, name) {
|
|
65
|
-
let rows = state.history[name];
|
|
66
|
-
if (rows !== undefined) {
|
|
67
|
-
return rows;
|
|
68
|
-
}
|
|
69
|
-
let rows$1 = [];
|
|
70
|
-
state.history[name] = rows$1;
|
|
71
|
-
return rows$1;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function registerEntities(state, entities) {
|
|
75
|
-
entities.forEach(entityConfig => {
|
|
76
|
-
state.entityConfigs[entityConfig.name] = entityConfig;
|
|
77
|
-
state.entityConfigs[entityConfig.table.tableName] = entityConfig;
|
|
78
|
-
getEntityDict(state, entityConfig.name);
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function seedConfigAddresses(state, chainConfigs, contractMapping, ecosystem) {
|
|
83
|
-
AddressRows.Table.clear(state.addresses);
|
|
84
|
-
ChainState.seedConfigAddresses(state.addresses, chainConfigs, ecosystem, contractMapping);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function addressRowsByChain(state) {
|
|
88
|
-
return AddressRows.Table.groupByChain(state.addresses);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function toInitialChainStates(state) {
|
|
92
|
-
let addressesByChain = AddressRows.Table.groupByChain(state.addresses);
|
|
93
|
-
return Object.values(state.chains).map(chain => ({
|
|
94
|
-
id: chain.id,
|
|
95
|
-
startBlock: chain.startBlock,
|
|
96
|
-
endBlock: chain.endBlock,
|
|
97
|
-
maxReorgDepth: chain.maxReorgDepth,
|
|
98
|
-
progressBlockNumber: chain.progressBlockNumber,
|
|
99
|
-
numEventsProcessed: chain.numEventsProcessed,
|
|
100
|
-
firstEventBlockNumber: chain.firstEventBlockNumber,
|
|
101
|
-
timestampCaughtUpToHeadOrEndblock: chain.timestampCaughtUpToHeadOrEndblock,
|
|
102
|
-
addressRows: Stdlib_Option.getOr(addressesByChain[ChainId.toString(chain.id)], AddressRows.emptySeedRows()),
|
|
103
|
-
sourceBlockNumber: chain.sourceBlockNumber
|
|
104
|
-
}));
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
function committedCheckpointId(state) {
|
|
108
|
-
return Stdlib_Array.reduce(state.checkpoints, InternalTable.Checkpoints.initialCheckpointId, (max, cp) => {
|
|
109
|
-
if (cp.id > max) {
|
|
110
|
-
return cp.id;
|
|
111
|
-
} else {
|
|
112
|
-
return max;
|
|
113
|
-
}
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
function reorgCheckpoints(state) {
|
|
118
|
-
return Stdlib_Array.filterMap(state.checkpoints, cp => {
|
|
119
|
-
let match = cp.blockHash;
|
|
120
|
-
let match$1 = state.chains[ChainId.toString(cp.chainId)];
|
|
121
|
-
if (match === undefined) {
|
|
122
|
-
return;
|
|
123
|
-
}
|
|
124
|
-
if (match$1 === undefined) {
|
|
125
|
-
return;
|
|
126
|
-
}
|
|
127
|
-
let safeBlock = match$1.sourceBlockNumber - match$1.maxReorgDepth | 0;
|
|
128
|
-
if (match$1.maxReorgDepth > 0 && match$1.progressBlockNumber > safeBlock && cp.blockNumber >= safeBlock) {
|
|
129
|
-
return {
|
|
130
|
-
id: cp.id,
|
|
131
|
-
chain_id: cp.chainId,
|
|
132
|
-
block_number: cp.blockNumber,
|
|
133
|
-
block_hash: match
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
});
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
function toInitialState(state, cleanRun, contractMapping) {
|
|
140
|
-
return {
|
|
141
|
-
cleanRun: cleanRun,
|
|
142
|
-
contractMapping: contractMapping,
|
|
143
|
-
envioInfo: state.envioInfo,
|
|
144
|
-
cache: state.cache,
|
|
145
|
-
chains: toInitialChainStates(state),
|
|
146
|
-
checkpointId: committedCheckpointId(state),
|
|
147
|
-
reorgCheckpoints: reorgCheckpoints(state)
|
|
148
|
-
};
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function handleLoad(state, tableName, filter) {
|
|
152
|
-
let entityConfig = state.entityConfigs[tableName];
|
|
153
|
-
if (entityConfig !== undefined) {
|
|
154
|
-
let entityDict = Stdlib_Option.getOr(state.entities[entityConfig.name], {});
|
|
155
|
-
let matched = Object.values(entityDict).filter(entity => EntityFilter.matches(filter, entity));
|
|
156
|
-
let field = Table.getChainIdField(entityConfig.table);
|
|
157
|
-
if (field !== undefined) {
|
|
158
|
-
return matched.map(entity => {
|
|
159
|
-
let copy = Utils.Dict.shallowCopy(entity);
|
|
160
|
-
Utils.Dict.deleteInPlace(copy, field.fieldName);
|
|
161
|
-
return copy;
|
|
162
|
-
});
|
|
163
|
-
} else {
|
|
164
|
-
return matched;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
let cacheDict = state.effectCache[tableName];
|
|
168
|
-
if (cacheDict !== undefined) {
|
|
169
|
-
return Object.values(cacheDict).filter(item => EntityFilter.matches(filter, item));
|
|
170
|
-
} else {
|
|
171
|
-
return [];
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
function backfillHistory(state, entityConfig, scope, entityId, rows) {
|
|
176
|
-
let key = rowKey(scope, entityId);
|
|
177
|
-
if (rows.some(row => rowKey(row.scope, row.entityId) === key)) {
|
|
178
|
-
return;
|
|
179
|
-
}
|
|
180
|
-
let entity = Stdlib_Option.flatMap(state.entities[entityConfig.name], dict => dict[rowKey(scope, entityId)]);
|
|
181
|
-
if (entity !== undefined) {
|
|
182
|
-
rows.push({
|
|
183
|
-
entityId: entityId,
|
|
184
|
-
scope: scope,
|
|
185
|
-
checkpointId: InternalTable.Checkpoints.initialCheckpointId,
|
|
186
|
-
action: "SET",
|
|
187
|
-
entity: entity
|
|
188
|
-
});
|
|
189
|
-
return;
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
function applyRollback(state, targetCheckpointId, rolledBackAddresses) {
|
|
194
|
-
state.checkpoints = state.checkpoints.filter(cp => cp.id <= targetCheckpointId);
|
|
195
|
-
rolledBackAddresses.forEach(key => AddressRows.Table.$$delete(state.addresses, key));
|
|
196
|
-
Object.entries(state.history).forEach(param => {
|
|
197
|
-
state.history[param[0]] = param[1].filter(row => row.checkpointId <= targetCheckpointId);
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
function writeBatch(state, batch, rollback, isInReorgThreshold, config, updatedEntities, registeredAddresses, updatedEffectsCache, chainMetaData) {
|
|
202
|
-
let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold);
|
|
203
|
-
if (rollback !== undefined) {
|
|
204
|
-
applyRollback(state, rollback.targetCheckpointId, rollback.rolledBackAddresses);
|
|
205
|
-
}
|
|
206
|
-
registeredAddresses.forEach(param => AddressRows.Table.insert(state.addresses, param.row));
|
|
207
|
-
let diffCheckpointId = Stdlib_Option.map(rollback, param => param.diffCheckpointId);
|
|
208
|
-
let isDiff = change => {
|
|
209
|
-
if (diffCheckpointId !== undefined) {
|
|
210
|
-
return change.checkpointId === diffCheckpointId;
|
|
211
|
-
} else {
|
|
212
|
-
return false;
|
|
213
|
-
}
|
|
214
|
-
};
|
|
215
|
-
updatedEntities.forEach(param => {
|
|
216
|
-
let changes = param.changes;
|
|
217
|
-
let scope = param.scope;
|
|
218
|
-
let entityConfig = param.entityConfig;
|
|
219
|
-
let entityDict = getEntityDict(state, entityConfig.name);
|
|
220
|
-
let historyRows = getHistory(state, entityConfig.name);
|
|
221
|
-
let chainIdField = Table.getChainIdField(entityConfig.table);
|
|
222
|
-
let idsWithDiff = new Set();
|
|
223
|
-
changes.forEach(change => {
|
|
224
|
-
if (isDiff(change)) {
|
|
225
|
-
idsWithDiff.add(EntityId.toKey(change.entityId));
|
|
226
|
-
return;
|
|
227
|
-
}
|
|
228
|
-
});
|
|
229
|
-
changes.forEach(change => {
|
|
230
|
-
let entityId = change.entityId;
|
|
231
|
-
let shouldSaveChangeHistory = shouldSaveHistory && !isDiff(change);
|
|
232
|
-
if (shouldSaveHistory && !idsWithDiff.has(EntityId.toKey(entityId))) {
|
|
233
|
-
backfillHistory(state, entityConfig, scope, entityId, historyRows);
|
|
234
|
-
}
|
|
235
|
-
if (change.type === "SET") {
|
|
236
|
-
let entity = change.entity;
|
|
237
|
-
let match = Internal.chainScopeChainId(scope);
|
|
238
|
-
let storedEntity = chainIdField !== undefined && match !== undefined ? Internal.stampChainId(entity, chainIdField.fieldName, Primitive_option.valFromOption(match)) : entity;
|
|
239
|
-
entityDict[rowKey(scope, entityId)] = storedEntity;
|
|
240
|
-
if (shouldSaveChangeHistory) {
|
|
241
|
-
historyRows.push({
|
|
242
|
-
entityId: entityId,
|
|
243
|
-
scope: scope,
|
|
244
|
-
checkpointId: change.checkpointId,
|
|
245
|
-
action: "SET",
|
|
246
|
-
entity: storedEntity
|
|
247
|
-
});
|
|
248
|
-
return;
|
|
249
|
-
} else {
|
|
250
|
-
return;
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
Utils.Dict.deleteInPlace(entityDict, rowKey(scope, entityId));
|
|
254
|
-
if (shouldSaveChangeHistory) {
|
|
255
|
-
historyRows.push({
|
|
256
|
-
entityId: entityId,
|
|
257
|
-
scope: scope,
|
|
258
|
-
checkpointId: change.checkpointId,
|
|
259
|
-
action: "DELETE",
|
|
260
|
-
entity: undefined
|
|
261
|
-
});
|
|
262
|
-
return;
|
|
263
|
-
}
|
|
264
|
-
});
|
|
265
|
-
});
|
|
266
|
-
Object.values(batch.progressedChainsById).forEach(chainAfterBatch => {
|
|
267
|
-
let key = ChainId.toString(chainAfterBatch.fetchState.chainId);
|
|
268
|
-
let chain = state.chains[key];
|
|
269
|
-
if (chain !== undefined) {
|
|
270
|
-
chain.progressBlockNumber = chainAfterBatch.progressBlockNumber;
|
|
271
|
-
chain.sourceBlockNumber = chainAfterBatch.sourceBlockNumber;
|
|
272
|
-
chain.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
|
|
273
|
-
return;
|
|
274
|
-
}
|
|
275
|
-
});
|
|
276
|
-
if (chainMetaData !== undefined) {
|
|
277
|
-
Object.entries(chainMetaData).forEach(param => {
|
|
278
|
-
let meta = param[1];
|
|
279
|
-
let chain = state.chains[param[0]];
|
|
280
|
-
if (chain !== undefined) {
|
|
281
|
-
chain.firstEventBlockNumber = Primitive_option.fromNull(meta.first_event_block);
|
|
282
|
-
chain.latestFetchedBlockNumber = meta.buffer_block;
|
|
283
|
-
chain.timestampCaughtUpToHeadOrEndblock = Primitive_option.fromNull(meta.ready_at);
|
|
284
|
-
return;
|
|
285
|
-
}
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
if (shouldSaveHistory) {
|
|
289
|
-
for (let i = 0, i_finish = batch.checkpointIds.length; i < i_finish; ++i) {
|
|
290
|
-
state.checkpoints.push({
|
|
291
|
-
id: batch.checkpointIds[i],
|
|
292
|
-
chainId: batch.checkpointChainIds[i],
|
|
293
|
-
blockNumber: batch.checkpointBlockNumbers[i],
|
|
294
|
-
blockHash: Primitive_option.fromNull(batch.checkpointBlockHashes[i]),
|
|
295
|
-
eventsProcessed: batch.checkpointEventsProcessed[i]
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
updatedEffectsCache.forEach(param => {
|
|
300
|
-
let table = param.table;
|
|
301
|
-
let dict = state.effectCache[table.tableName];
|
|
302
|
-
let cacheDict;
|
|
303
|
-
if (dict !== undefined) {
|
|
304
|
-
cacheDict = dict;
|
|
305
|
-
} else {
|
|
306
|
-
let dict$1 = {};
|
|
307
|
-
state.effectCache[table.tableName] = dict$1;
|
|
308
|
-
cacheDict = dict$1;
|
|
309
|
-
}
|
|
310
|
-
param.items.forEach(item => {
|
|
311
|
-
cacheDict[item.id] = item;
|
|
312
|
-
});
|
|
313
|
-
let record = state.cache[table.tableName];
|
|
314
|
-
if (record !== undefined) {
|
|
315
|
-
record.count = Object.keys(cacheDict).length;
|
|
316
|
-
return;
|
|
317
|
-
}
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
function getRollbackData(state, entityConfig, rollbackTargetCheckpointId) {
|
|
322
|
-
let rows = Stdlib_Option.getOr(state.history[entityConfig.name], []);
|
|
323
|
-
let changedKeys = {};
|
|
324
|
-
rows.forEach(row => {
|
|
325
|
-
if (row.checkpointId > rollbackTargetCheckpointId) {
|
|
326
|
-
changedKeys[rowKey(row.scope, row.entityId)] = row;
|
|
327
|
-
return;
|
|
328
|
-
}
|
|
329
|
-
});
|
|
330
|
-
let removals = [];
|
|
331
|
-
let restored = [];
|
|
332
|
-
Object.entries(changedKeys).forEach(param => {
|
|
333
|
-
let changedRow = param[1];
|
|
334
|
-
let key = param[0];
|
|
335
|
-
let preTarget = Stdlib_Array.reduce(rows, undefined, (latest, row) => {
|
|
336
|
-
if (rowKey(row.scope, row.entityId) === key && row.checkpointId <= rollbackTargetCheckpointId && !(latest !== undefined && latest.checkpointId >= row.checkpointId)) {
|
|
337
|
-
return row;
|
|
338
|
-
} else {
|
|
339
|
-
return latest;
|
|
340
|
-
}
|
|
341
|
-
});
|
|
342
|
-
if (preTarget !== undefined) {
|
|
343
|
-
if (preTarget.action === "SET") {
|
|
344
|
-
let entity = preTarget.entity;
|
|
345
|
-
if (entity !== undefined) {
|
|
346
|
-
restored.push(entity);
|
|
347
|
-
return;
|
|
348
|
-
} else {
|
|
349
|
-
return;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
removals.push({
|
|
353
|
-
entityId: preTarget.entityId,
|
|
354
|
-
scope: preTarget.scope
|
|
355
|
-
});
|
|
356
|
-
return;
|
|
357
|
-
}
|
|
358
|
-
removals.push({
|
|
359
|
-
entityId: changedRow.entityId,
|
|
360
|
-
scope: changedRow.scope
|
|
361
|
-
});
|
|
362
|
-
});
|
|
363
|
-
return [
|
|
364
|
-
removals,
|
|
365
|
-
restored
|
|
366
|
-
];
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
function getRollbackProgressDiff(state, rollbackTargetCheckpointId) {
|
|
370
|
-
let byChain = {};
|
|
371
|
-
state.checkpoints.forEach(cp => {
|
|
372
|
-
if (cp.id <= rollbackTargetCheckpointId) {
|
|
373
|
-
return;
|
|
374
|
-
}
|
|
375
|
-
let key = ChainId.toString(cp.chainId);
|
|
376
|
-
let acc = byChain[key];
|
|
377
|
-
if (acc !== undefined) {
|
|
378
|
-
acc.eventsProcessed = acc.eventsProcessed + cp.eventsProcessed | 0;
|
|
379
|
-
acc.newProgressBlockNumber = Math.min(acc.newProgressBlockNumber, cp.blockNumber - 1 | 0);
|
|
380
|
-
} else {
|
|
381
|
-
byChain[key] = {
|
|
382
|
-
chainId: cp.chainId,
|
|
383
|
-
eventsProcessed: cp.eventsProcessed,
|
|
384
|
-
newProgressBlockNumber: cp.blockNumber - 1 | 0
|
|
385
|
-
};
|
|
386
|
-
}
|
|
387
|
-
});
|
|
388
|
-
return Object.values(byChain).map(acc => ({
|
|
389
|
-
chain_id: acc.chainId,
|
|
390
|
-
events_processed_diff: acc.eventsProcessed.toString(),
|
|
391
|
-
new_progress_block_number: acc.newProgressBlockNumber
|
|
392
|
-
}));
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
function toStorage(state, config) {
|
|
396
|
-
return {
|
|
397
|
-
name: "memory",
|
|
398
|
-
isInitialized: async () => state.isInitialized,
|
|
399
|
-
initialize: async (chainConfigsOpt, entitiesOpt, $staropt$star, contractMapping, envioInfo) => {
|
|
400
|
-
let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
|
|
401
|
-
let entities = entitiesOpt !== undefined ? entitiesOpt : [];
|
|
402
|
-
$staropt$star !== undefined;
|
|
403
|
-
registerEntities(state, entities);
|
|
404
|
-
seedConfigAddresses(state, chainConfigs, contractMapping, config.ecosystem.name);
|
|
405
|
-
chainConfigs.forEach(chainConfig => {
|
|
406
|
-
state.chains[ChainId.toString(chainConfig.id)] = {
|
|
407
|
-
id: chainConfig.id,
|
|
408
|
-
startBlock: chainConfig.startBlock,
|
|
409
|
-
endBlock: chainConfig.endBlock,
|
|
410
|
-
maxReorgDepth: chainConfig.maxReorgDepth,
|
|
411
|
-
progressBlockNumber: -1,
|
|
412
|
-
sourceBlockNumber: 0,
|
|
413
|
-
numEventsProcessed: 0,
|
|
414
|
-
firstEventBlockNumber: undefined,
|
|
415
|
-
latestFetchedBlockNumber: 0,
|
|
416
|
-
timestampCaughtUpToHeadOrEndblock: undefined
|
|
417
|
-
};
|
|
418
|
-
});
|
|
419
|
-
state.envioInfo = envioInfo;
|
|
420
|
-
state.isInitialized = true;
|
|
421
|
-
return toInitialState(state, true, contractMapping);
|
|
422
|
-
},
|
|
423
|
-
resumeInitialState: async () => toInitialState(state, false, config.contractMapping),
|
|
424
|
-
loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
|
|
425
|
-
ensureQueryIndexes: async (param, param$1) => {},
|
|
426
|
-
ensureSchemaIndexes: async param => {},
|
|
427
|
-
finalizeBackfill: async (param, chainIds, readyAt) => {
|
|
428
|
-
chainIds.forEach(chainId => {
|
|
429
|
-
let chain = state.chains[ChainId.toString(chainId)];
|
|
430
|
-
if (chain !== undefined) {
|
|
431
|
-
chain.timestampCaughtUpToHeadOrEndblock = readyAt;
|
|
432
|
-
return;
|
|
433
|
-
}
|
|
434
|
-
});
|
|
435
|
-
},
|
|
436
|
-
dumpEffectCache: async () => {},
|
|
437
|
-
reset: async () => {
|
|
438
|
-
let clear = Utils.Dict.clearInPlace;
|
|
439
|
-
clear(state.entities);
|
|
440
|
-
clear(state.history);
|
|
441
|
-
clear(state.chains);
|
|
442
|
-
clear(state.effectCache);
|
|
443
|
-
clear(state.cache);
|
|
444
|
-
state.checkpoints = [];
|
|
445
|
-
AddressRows.Table.clear(state.addresses);
|
|
446
|
-
state.envioInfo = undefined;
|
|
447
|
-
state.isInitialized = false;
|
|
448
|
-
},
|
|
449
|
-
setChainMeta: async chainsData => {
|
|
450
|
-
Object.entries(chainsData).forEach(param => {
|
|
451
|
-
let meta = param[1];
|
|
452
|
-
let chain = state.chains[param[0]];
|
|
453
|
-
if (chain !== undefined) {
|
|
454
|
-
chain.firstEventBlockNumber = Primitive_option.fromNull(meta.first_event_block);
|
|
455
|
-
chain.latestFetchedBlockNumber = meta.buffer_block;
|
|
456
|
-
chain.timestampCaughtUpToHeadOrEndblock = Primitive_option.fromNull(meta.ready_at);
|
|
457
|
-
return;
|
|
458
|
-
}
|
|
459
|
-
});
|
|
460
|
-
return undefined;
|
|
461
|
-
},
|
|
462
|
-
pruneStaleCheckpoints: async safeCheckpointId => {
|
|
463
|
-
state.checkpoints = state.checkpoints.filter(cp => cp.id >= safeCheckpointId);
|
|
464
|
-
},
|
|
465
|
-
pruneStaleEntityHistory: async (entityName, param, param$1, safeCheckpointId) => {
|
|
466
|
-
let rows = state.history[entityName];
|
|
467
|
-
if (rows === undefined) {
|
|
468
|
-
return;
|
|
469
|
-
}
|
|
470
|
-
let newestBelow = {};
|
|
471
|
-
rows.forEach(row => {
|
|
472
|
-
if (row.checkpointId >= safeCheckpointId) {
|
|
473
|
-
return;
|
|
474
|
-
}
|
|
475
|
-
let key = rowKey(row.scope, row.entityId);
|
|
476
|
-
let current = newestBelow[key];
|
|
477
|
-
if (current !== undefined && current.checkpointId >= row.checkpointId) {
|
|
478
|
-
return;
|
|
479
|
-
} else {
|
|
480
|
-
newestBelow[key] = row;
|
|
481
|
-
return;
|
|
482
|
-
}
|
|
483
|
-
});
|
|
484
|
-
state.history[entityName] = rows.filter(row => {
|
|
485
|
-
let key = rowKey(row.scope, row.entityId);
|
|
486
|
-
if (row.checkpointId >= safeCheckpointId) {
|
|
487
|
-
return true;
|
|
488
|
-
}
|
|
489
|
-
let kept = newestBelow[key];
|
|
490
|
-
if (kept !== undefined) {
|
|
491
|
-
return kept === row;
|
|
492
|
-
} else {
|
|
493
|
-
return false;
|
|
494
|
-
}
|
|
495
|
-
});
|
|
496
|
-
},
|
|
497
|
-
getRollbackTargetCheckpoint: async (reorgChainId, lastKnownValidBlockNumber) => Stdlib_Array.reduce(state.checkpoints, undefined, (target, cp) => {
|
|
498
|
-
if (Primitive_object.equal(cp.chainId, reorgChainId) && cp.blockNumber <= lastKnownValidBlockNumber && !(target !== undefined && target >= cp.id)) {
|
|
499
|
-
return cp.id;
|
|
500
|
-
} else {
|
|
501
|
-
return target;
|
|
502
|
-
}
|
|
503
|
-
}),
|
|
504
|
-
getRollbackProgressDiff: async rollbackTargetCheckpointId => getRollbackProgressDiff(state, rollbackTargetCheckpointId),
|
|
505
|
-
getRollbackData: async (entityConfig, rollbackTargetCheckpointId) => getRollbackData(state, entityConfig, rollbackTargetCheckpointId),
|
|
506
|
-
writeBatch: async (batch, rollback, isInReorgThreshold, param, param$1, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, param$2) => writeBatch(state, batch, rollback, isInReorgThreshold, config, updatedEntities, registeredAddresses, updatedEffectsCache, chainMetaData),
|
|
507
|
-
close: async () => {}
|
|
508
|
-
};
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
function currentRows(state, entityConfig) {
|
|
512
|
-
return Object.values(Stdlib_Option.getOr(state.entities[entityConfig.name], {}));
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
function historyChanges(state, entityConfig) {
|
|
516
|
-
return Stdlib_Option.getOr(state.history[entityConfig.name], []).map(row => {
|
|
517
|
-
let match = row.action;
|
|
518
|
-
if (match === "SET") {
|
|
519
|
-
return {
|
|
520
|
-
type: "SET",
|
|
521
|
-
entityId: row.entityId,
|
|
522
|
-
entity: Stdlib_Option.getOr(row.entity, {}),
|
|
523
|
-
checkpointId: row.checkpointId
|
|
524
|
-
};
|
|
525
|
-
} else {
|
|
526
|
-
return {
|
|
527
|
-
type: "DELETE",
|
|
528
|
-
entityId: row.entityId,
|
|
529
|
-
checkpointId: row.checkpointId
|
|
530
|
-
};
|
|
531
|
-
}
|
|
532
|
-
}).toSorted((a, b) => {
|
|
533
|
-
let order = Primitive_string.compare(EntityId.toKey(a.entityId), EntityId.toKey(b.entityId));
|
|
534
|
-
if (order !== 0) {
|
|
535
|
-
return order;
|
|
536
|
-
} else {
|
|
537
|
-
return Primitive_float.compare(Number(a.checkpointId), Number(b.checkpointId));
|
|
538
|
-
}
|
|
539
|
-
});
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
function checkpointRows(state) {
|
|
543
|
-
return state.checkpoints.map(cp => ({
|
|
544
|
-
id: cp.id,
|
|
545
|
-
chain_id: cp.chainId,
|
|
546
|
-
block_number: cp.blockNumber,
|
|
547
|
-
block_hash: Stdlib_Null.fromOption(cp.blockHash),
|
|
548
|
-
events_processed: cp.eventsProcessed
|
|
549
|
-
}));
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
function effectCacheRows(state, tableName) {
|
|
553
|
-
return Object.values(Stdlib_Option.getOr(state.effectCache[tableName], {})).map(item => ({
|
|
554
|
-
id: item.id,
|
|
555
|
-
output: item.output
|
|
556
|
-
}));
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
export {
|
|
560
|
-
make,
|
|
561
|
-
rowKey,
|
|
562
|
-
historyKey,
|
|
563
|
-
readChainId,
|
|
564
|
-
copyEntity,
|
|
565
|
-
getEntityDict,
|
|
566
|
-
getHistory,
|
|
567
|
-
registerEntities,
|
|
568
|
-
seedConfigAddresses,
|
|
569
|
-
addressRowsByChain,
|
|
570
|
-
toInitialChainStates,
|
|
571
|
-
committedCheckpointId,
|
|
572
|
-
reorgCheckpoints,
|
|
573
|
-
toInitialState,
|
|
574
|
-
handleLoad,
|
|
575
|
-
backfillHistory,
|
|
576
|
-
applyRollback,
|
|
577
|
-
writeBatch,
|
|
578
|
-
getRollbackData,
|
|
579
|
-
getRollbackProgressDiff,
|
|
580
|
-
toStorage,
|
|
581
|
-
currentRows,
|
|
582
|
-
historyChanges,
|
|
583
|
-
checkpointRows,
|
|
584
|
-
effectCacheRows,
|
|
585
|
-
}
|
|
586
|
-
/* Table Not a pure module */
|
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
Pure subscription-based implementation of the HyperSync height stream.
|
|
3
|
-
*/
|
|
4
|
-
|
|
5
|
-
let subscribe = (~hyperSyncUrl, ~apiToken, ~chainId, ~onHeight: int => unit): (unit => unit) => {
|
|
6
|
-
let eventsourceRef = ref(None)
|
|
7
|
-
let errorCount = ref(0)
|
|
8
|
-
let baseDuration = 50
|
|
9
|
-
// Timeout doesn't do anything for initialization
|
|
10
|
-
let timeoutIdRef = ref(setTimeout(() => (), 0))
|
|
11
|
-
|
|
12
|
-
// On every successful ping or height event, clear the timeout and set a new one.
|
|
13
|
-
// If the timeout lapses, close and reconnect the EventSource.
|
|
14
|
-
let rec updateTimeoutId = () => {
|
|
15
|
-
timeoutIdRef.contents->clearTimeout
|
|
16
|
-
|
|
17
|
-
// Should receive a ping at least every 5s, so 15s is a safe margin
|
|
18
|
-
// for staleness to restart the EventSource connection
|
|
19
|
-
let staleTimeMillis = 15_000
|
|
20
|
-
let newTimeoutId = setTimeout(() => {
|
|
21
|
-
Logging.trace({
|
|
22
|
-
"msg": "Timeout fired for height stream",
|
|
23
|
-
"chainId": chainId,
|
|
24
|
-
"url": hyperSyncUrl,
|
|
25
|
-
"staleTimeMillis": staleTimeMillis,
|
|
26
|
-
})
|
|
27
|
-
refreshEventSource()
|
|
28
|
-
}, staleTimeMillis)
|
|
29
|
-
|
|
30
|
-
timeoutIdRef := newTimeoutId
|
|
31
|
-
}
|
|
32
|
-
// Instantiate a new EventSource and set it to the shared refs.
|
|
33
|
-
// Add the necessary event listeners, handle errors
|
|
34
|
-
// and update the timeout.
|
|
35
|
-
and scheduleReconnect = () => {
|
|
36
|
-
// Clear any pending stale/reconnect timeout to avoid double reconnect
|
|
37
|
-
timeoutIdRef.contents->clearTimeout
|
|
38
|
-
let delay = baseDuration * Math.pow(2.0, ~exp=errorCount.contents->Int.toFloat)->Float.toInt
|
|
39
|
-
timeoutIdRef := setTimeout(() => refreshEventSource(), Pervasives.min(delay, 60_000))
|
|
40
|
-
}
|
|
41
|
-
and refreshEventSource = () => {
|
|
42
|
-
// Close the old EventSource if it exists (on a new connection after timeout)
|
|
43
|
-
switch eventsourceRef.contents {
|
|
44
|
-
| Some(es) => es->EventSource.close
|
|
45
|
-
| None => ()
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
let userAgent = `hyperindex/${Utils.EnvioPackage.value.version}`
|
|
49
|
-
let es = EventSource.create(
|
|
50
|
-
~url=`${hyperSyncUrl}/height/sse`,
|
|
51
|
-
~options={
|
|
52
|
-
fetch: (url, ~args) => {
|
|
53
|
-
EventSource.Fetch.fetch(
|
|
54
|
-
url,
|
|
55
|
-
~args={
|
|
56
|
-
...args,
|
|
57
|
-
headers: Dict.fromArray([
|
|
58
|
-
("Authorization", `Bearer ${apiToken}`),
|
|
59
|
-
("User-Agent", userAgent),
|
|
60
|
-
]),
|
|
61
|
-
},
|
|
62
|
-
)
|
|
63
|
-
},
|
|
64
|
-
},
|
|
65
|
-
)
|
|
66
|
-
|
|
67
|
-
// Set the new EventSource to the shared ref
|
|
68
|
-
eventsourceRef := Some(es)
|
|
69
|
-
// Update the timeout in case connection goes stale
|
|
70
|
-
updateTimeoutId()
|
|
71
|
-
|
|
72
|
-
es->EventSource.onopen(_ => {
|
|
73
|
-
errorCount := 0
|
|
74
|
-
Logging.trace({
|
|
75
|
-
"msg": "SSE connection opened for height stream",
|
|
76
|
-
"chainId": chainId,
|
|
77
|
-
"url": hyperSyncUrl,
|
|
78
|
-
})
|
|
79
|
-
})
|
|
80
|
-
|
|
81
|
-
es->EventSource.onerror(error => {
|
|
82
|
-
errorCount := errorCount.contents + 1
|
|
83
|
-
Logging.trace({
|
|
84
|
-
"msg": "EventSource error on height stream, reconnecting",
|
|
85
|
-
"chainId": chainId,
|
|
86
|
-
"url": hyperSyncUrl,
|
|
87
|
-
"status": error.status,
|
|
88
|
-
"error": error.message,
|
|
89
|
-
"errorCount": errorCount.contents,
|
|
90
|
-
})
|
|
91
|
-
scheduleReconnect()
|
|
92
|
-
})
|
|
93
|
-
|
|
94
|
-
es->EventSource.addEventListener("ping", _event => {
|
|
95
|
-
// ping lets us know from the server that the connection is still alive
|
|
96
|
-
// and that the height hasn't updated for 5 seconds
|
|
97
|
-
// update the timeout on each successful ping received
|
|
98
|
-
updateTimeoutId()
|
|
99
|
-
})
|
|
100
|
-
|
|
101
|
-
es->EventSource.addEventListener("height", event => {
|
|
102
|
-
switch event.data->Int.fromString {
|
|
103
|
-
| Some(height) =>
|
|
104
|
-
// On a successful height event, update the timeout
|
|
105
|
-
updateTimeoutId()
|
|
106
|
-
// Call the callback with the new height
|
|
107
|
-
onHeight(height)
|
|
108
|
-
| None =>
|
|
109
|
-
Logging.trace({
|
|
110
|
-
"msg": "Height was not a number in event.data",
|
|
111
|
-
"chainId": chainId,
|
|
112
|
-
"data": event.data,
|
|
113
|
-
})
|
|
114
|
-
}
|
|
115
|
-
})
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
// Start the EventSource connection
|
|
119
|
-
refreshEventSource()
|
|
120
|
-
|
|
121
|
-
// Return unsubscribe function
|
|
122
|
-
() => {
|
|
123
|
-
timeoutIdRef.contents->clearTimeout
|
|
124
|
-
switch eventsourceRef.contents {
|
|
125
|
-
| Some(es) => es->EventSource.close
|
|
126
|
-
| None => ()
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
}
|