envio 3.15.0 → 3.16.0-subgraph
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -34
- package/evm.schema.json +11 -2
- package/fuel.schema.json +10 -1
- package/index.d.ts +6 -0
- package/licenses/README.md +1 -1
- package/package.json +10 -9
- package/src/Batch.res.mjs +5 -5
- package/src/BatchProcessing.res +1 -0
- package/src/BatchProcessing.res.mjs +1 -0
- package/src/Bin.res +7 -0
- package/src/Bin.res.mjs +8 -0
- package/src/ChainFetching.res +450 -16
- package/src/ChainFetching.res.mjs +403 -32
- package/src/ChainState.res +124 -7
- package/src/ChainState.res.mjs +128 -67
- package/src/ChainState.resi +26 -0
- package/src/Clock.res +57 -0
- package/src/Clock.res.mjs +72 -0
- package/src/Config.res +31 -3
- package/src/Config.res.mjs +15 -4
- package/src/ContractRegisterContext.res +3 -6
- package/src/ContractRegisterContext.res.mjs +4 -2
- package/src/Core.res +95 -36
- package/src/Core.res.mjs +90 -1
- package/src/CrossChainState.res +12 -2
- package/src/CrossChainState.res.mjs +3 -3
- package/src/Ecosystem.res +7 -4
- package/src/Ecosystem.res.mjs +8 -8
- package/src/EffectCacheLookup.res +98 -0
- package/src/EffectCacheLookup.res.mjs +116 -0
- package/src/EffectState.res +10 -2
- package/src/EffectState.res.mjs +8 -4
- package/src/EffectState.resi +3 -1
- package/src/EntityId.res +2 -7
- package/src/EntityId.res.mjs +1 -8
- package/src/EntityTables.res +4 -1
- package/src/EntityTables.res.mjs +1 -1
- package/src/Env.res +20 -0
- package/src/Env.res.mjs +16 -0
- package/src/EventProcessing.res +38 -1
- package/src/EventProcessing.res.mjs +28 -1
- package/src/FetchState.res +408 -73
- package/src/FetchState.res.mjs +361 -91
- package/src/FinalizeBackfill.res +2 -4
- package/src/FinalizeBackfill.res.mjs +3 -4
- package/src/HandlerLoader.res +39 -15
- package/src/HandlerLoader.res.mjs +27 -8
- package/src/HandlerRegister.res +4 -0
- package/src/HandlerRegister.res.mjs +5 -2
- package/src/HandlerRegister.resi +2 -0
- package/src/InMemoryTable.res +17 -16
- package/src/InMemoryTable.res.mjs +14 -15
- package/src/IndexerState.res +28 -4
- package/src/IndexerState.res.mjs +19 -6
- package/src/IndexerState.resi +4 -0
- package/src/Internal.res +61 -49
- package/src/Internal.res.mjs +36 -80
- package/src/LoadLayer.res +54 -15
- package/src/LoadLayer.res.mjs +75 -45
- package/src/LoadLayer.resi +26 -0
- package/src/Main.res +14 -1
- package/src/Main.res.mjs +4 -3
- package/src/Persistence.res +3 -23
- package/src/Persistence.res.mjs +3 -3
- package/src/PreloadLookahead.res +202 -0
- package/src/PreloadLookahead.res.mjs +218 -0
- package/src/Rollback.res +0 -1
- package/src/Rollback.res.mjs +1 -1
- package/src/RunScope.res +3 -0
- package/src/SafeCheckpointTracking.res +4 -4
- package/src/SafeCheckpointTracking.res.mjs +1 -1
- package/src/SimulateItems.res +42 -26
- package/src/SimulateItems.res.mjs +40 -16
- package/src/StagedRows.res +106 -0
- package/src/StagedRows.res.mjs +102 -0
- package/src/Staging.res +32 -14
- package/src/Staging.res.mjs +17 -20
- package/src/Storage.res +303 -948
- package/src/Storage.res.mjs +229 -708
- package/src/SubgraphLog.res +19 -0
- package/src/SubgraphLog.res.mjs +26 -0
- package/src/Supervisor.res +75 -56
- package/src/Supervisor.res.mjs +55 -26
- package/src/SyncWrites.res +107 -0
- package/src/SyncWrites.res.mjs +98 -0
- package/src/TestIndexer.res +22 -27
- package/src/TestIndexer.res.mjs +20 -17
- package/src/Throttler.res +15 -8
- package/src/Throttler.res.mjs +9 -6
- package/src/Throttler.resi +1 -1
- package/src/UserContext.res +419 -46
- package/src/UserContext.res.mjs +318 -30
- package/src/Utils.res +17 -6
- package/src/Utils.res.mjs +12 -6
- package/src/Writing.res +0 -1
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +0 -322
- package/src/bindings/ClickHouse.res.mjs +0 -338
- package/src/bindings/StorageClient.res +108 -174
- package/src/bindings/StorageClient.res.mjs +19 -8
- package/src/bindings/Vitest.res +0 -26
- package/src/bindings/Vitest.res.mjs +0 -3
- package/src/db/Table.res +24 -150
- package/src/db/Table.res.mjs +38 -272
- package/src/sources/AddressSet.res +2 -0
- package/src/sources/AddressStore.res +3 -0
- package/src/sources/CallSource.res +274 -0
- package/src/sources/CallSource.res.mjs +196 -0
- package/src/sources/ChainSources.res +2 -0
- package/src/sources/ChainSources.res.mjs +2 -2
- package/src/sources/EvmChain.res +10 -1
- package/src/sources/EvmChain.res.mjs +6 -3
- package/src/sources/EvmHyperSyncSource.res +2 -2
- package/src/sources/EvmHyperSyncSource.res.mjs +1 -1
- package/src/sources/EvmRpcClient.res +13 -3
- package/src/sources/EvmRpcClient.res.mjs +4 -3
- package/src/sources/EvmRpcWs.res +4 -3
- package/src/sources/EvmRpcWs.res.mjs +2 -2
- package/src/sources/HeightFeed.res +31 -10
- package/src/sources/HeightFeed.res.mjs +10 -6
- package/src/sources/HeightStream.res +32 -17
- package/src/sources/HeightStream.res.mjs +10 -10
- package/src/sources/HyperSync.res +2 -1
- package/src/sources/HyperSync.res.mjs +2 -1
- package/src/sources/HyperSyncSSE.res +4 -6
- package/src/sources/HyperSyncSSE.res.mjs +2 -2
- package/src/sources/RpcSource.res +10 -1
- package/src/sources/RpcSource.res.mjs +4 -3
- package/src/sources/Source.res +3 -0
- package/src/sources/SourceManager.res +84 -77
- package/src/sources/SourceManager.res.mjs +17 -20
- package/src/sources/SourceManager.resi +3 -4
- package/src/sources/StartBlockResolver.res +1 -0
- package/src/sources/StartBlockResolver.res.mjs +3 -2
- package/src/sources/SvmHyperSyncSource.res +2 -2
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
- package/src/subgraph/abi-types.ts +84 -0
- package/src/subgraph/aggregations.ts +745 -0
- package/src/subgraph/as-package.ts +143 -0
- package/src/subgraph/assemblyscript-regex.ts +67 -0
- package/src/subgraph/assemblyscript.ts +731 -0
- package/src/subgraph/blocks.ts +398 -0
- package/src/subgraph/calls-source.ts +702 -0
- package/src/subgraph/calls.ts +258 -0
- package/src/subgraph/conformance.ts +106 -0
- package/src/subgraph/errors.ts +85 -0
- package/src/subgraph/graft.ts +73 -0
- package/src/subgraph/graph-cli-worker.mjs +35 -0
- package/src/subgraph/graph-cli.ts +900 -0
- package/src/subgraph/graph-node-decimal.ts +199 -0
- package/src/subgraph/graph-ts-enums.ts +28 -0
- package/src/subgraph/graph-ts-types/VERSION +2 -0
- package/src/subgraph/graph-ts-types/chain/arweave.d.ts +70 -0
- package/src/subgraph/graph-ts-types/chain/cosmos.d.ts +327 -0
- package/src/subgraph/graph-ts-types/chain/ethereum.d.ts +233 -0
- package/src/subgraph/graph-ts-types/chain/near.d.ts +253 -0
- package/src/subgraph/graph-ts-types/chain/starknet.d.ts +32 -0
- package/src/subgraph/graph-ts-types/common/collections.d.ts +136 -0
- package/src/subgraph/graph-ts-types/common/conversion.d.ts +11 -0
- package/src/subgraph/graph-ts-types/common/datasource.d.ts +30 -0
- package/src/subgraph/graph-ts-types/common/eager-offset.d.ts +0 -0
- package/src/subgraph/graph-ts-types/common/json.d.ts +17 -0
- package/src/subgraph/graph-ts-types/common/numbers.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/value.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/yaml.d.ts +90 -0
- package/src/subgraph/graph-ts-types/global/global.d.ts +194 -0
- package/src/subgraph/graph-ts-types/helper-functions.d.ts +22 -0
- package/src/subgraph/graph-ts-types/index.d.ts +102 -0
- package/src/subgraph/graph-ts.ts +4339 -0
- package/src/subgraph/hosts.ts +554 -0
- package/src/subgraph/log.ts +28 -0
- package/src/subgraph/receipts.ts +184 -0
- package/src/subgraph/register-slots.ts +93 -0
- package/src/subgraph/rollup.ts +109 -0
- package/src/subgraph/rpc.ts +337 -0
- package/src/subgraph/runtime.ts +2059 -0
- package/src/subgraph/scope.ts +176 -0
- package/src/subgraph/stats.ts +42 -0
- package/src/subgraph/template-instances.ts +66 -0
- package/src/subgraph/toolchain.ts +327 -0
- package/src/subgraph/transient.ts +176 -0
- package/svm.schema.json +11 -2
- package/src/PgWriting.res +0 -36
- package/src/PgWriting.res.mjs +0 -34
- package/src/db/IndexDefinition.res +0 -47
- package/src/db/IndexDefinition.res.mjs +0 -67
package/src/Persistence.res
CHANGED
|
@@ -117,8 +117,6 @@ type storage = {
|
|
|
117
117
|
// carries a matching row — storage doesn't interpret it.
|
|
118
118
|
initialize: (
|
|
119
119
|
~chainConfigs: array<Config.chain>=?,
|
|
120
|
-
~entities: array<Internal.entityConfig>=?,
|
|
121
|
-
~enums: array<Table.enumConfig<Table.enum>>=?,
|
|
122
120
|
~contractMapping: ContractMapping.t,
|
|
123
121
|
~envioInfo: JSON.t,
|
|
124
122
|
) => promise<initialState>,
|
|
@@ -129,16 +127,11 @@ type storage = {
|
|
|
129
127
|
// Brings an initialized storage a chain it doesn't have yet: its row, its
|
|
130
128
|
// partitions and its config addresses, as `initialize` would have created
|
|
131
129
|
// them. The chains already there are left untouched.
|
|
132
|
-
addChain: (
|
|
133
|
-
~chainConfig: Config.chain,
|
|
134
|
-
~entities: array<Internal.entityConfig>,
|
|
135
|
-
~contractMapping: ContractMapping.t,
|
|
136
|
-
) => promise<unit>,
|
|
130
|
+
addChain: (~chainConfig: Config.chain, ~contractMapping: ContractMapping.t) => promise<unit>,
|
|
137
131
|
// `chainIds` is what this run drives. An isolated run resumes a subset of
|
|
138
132
|
// the stored chains, and ClickHouse's resume trims past each resumed chain's
|
|
139
133
|
// checkpoint, so chains a sibling process is still writing must stay out.
|
|
140
134
|
resumeInitialState: (
|
|
141
|
-
~entities: array<Internal.entityConfig>,
|
|
142
135
|
~chainIds: array<ChainId.t>,
|
|
143
136
|
~contractMapping: ContractMapping.t,
|
|
144
137
|
) => promise<initialState>,
|
|
@@ -159,19 +152,12 @@ type storage = {
|
|
|
159
152
|
// never runs `finalizeBackfill`: an index dropped or invalidated while it was
|
|
160
153
|
// down would otherwise never be rebuilt. Best-effort, and safe to run with
|
|
161
154
|
// indexing live.
|
|
162
|
-
ensureSchemaIndexes: (
|
|
163
|
-
~entities: array<Internal.entityConfig>,
|
|
164
|
-
~chainIds: array<ChainId.t>,
|
|
165
|
-
) => promise<unit>,
|
|
155
|
+
ensureSchemaIndexes: (~chainIds: array<ChainId.t>) => promise<unit>,
|
|
166
156
|
// Creates every index the schema promises for the given chains, then stamps
|
|
167
157
|
// `ready_at` on them. Called once those chains finish backfill. The indexes
|
|
168
158
|
// are committed one at a time so a failure part way through doesn't undo the
|
|
169
159
|
// ones already built; `ready_at` is only written once they all verify.
|
|
170
|
-
finalizeBackfill: (
|
|
171
|
-
~entities: array<Internal.entityConfig>,
|
|
172
|
-
~chainIds: array<ChainId.t>,
|
|
173
|
-
~readyAt: Date.t,
|
|
174
|
-
) => promise<unit>,
|
|
160
|
+
finalizeBackfill: (~chainIds: array<ChainId.t>, ~readyAt: Date.t) => promise<unit>,
|
|
175
161
|
// This is to download cache from the database to .envio/cache
|
|
176
162
|
dumpEffectCache: unit => promise<unit>,
|
|
177
163
|
reset: unit => promise<unit>,
|
|
@@ -190,7 +176,6 @@ type storage = {
|
|
|
190
176
|
getRollbackTargetCheckpoint: (
|
|
191
177
|
~reorgChainId: ChainId.t,
|
|
192
178
|
~lastKnownValidBlockNumber: int,
|
|
193
|
-
~sequence: CheckpointSequence.t,
|
|
194
179
|
) => promise<option<Internal.checkpointId>>,
|
|
195
180
|
// Get rollback progress diff
|
|
196
181
|
getRollbackProgressDiff: (
|
|
@@ -214,7 +199,6 @@ type storage = {
|
|
|
214
199
|
~batch: Batch.t,
|
|
215
200
|
~rollback: option<rollback>,
|
|
216
201
|
~config: Config.t,
|
|
217
|
-
~allEntities: array<Internal.entityConfig>,
|
|
218
202
|
~updatedEffectsCache: array<updatedEffectCache>,
|
|
219
203
|
~updatedEntities: array<updatedEntity>,
|
|
220
204
|
// Addresses this batch registered, with the checkpoint that covers them.
|
|
@@ -339,8 +323,6 @@ let init = {
|
|
|
339
323
|
throw(Refused(Config.setUpMeanwhileMessage))
|
|
340
324
|
}
|
|
341
325
|
let initialState = await persistence.storage.initialize(
|
|
342
|
-
~entities=persistence.allEntities,
|
|
343
|
-
~enums=persistence.allEnums,
|
|
344
326
|
~chainConfigs,
|
|
345
327
|
~contractMapping,
|
|
346
328
|
~envioInfo,
|
|
@@ -397,13 +379,11 @@ let init = {
|
|
|
397
379
|
)->Array.getUnsafe(0)
|
|
398
380
|
await persistence.storage.addChain(
|
|
399
381
|
~chainConfig,
|
|
400
|
-
~entities=persistence.allEntities,
|
|
401
382
|
~contractMapping=stored.contractMapping,
|
|
402
383
|
)
|
|
403
384
|
}
|
|
404
385
|
if resume {
|
|
405
386
|
let initialState = await persistence.storage.resumeInitialState(
|
|
406
|
-
~entities=persistence.allEntities,
|
|
407
387
|
~chainIds=chainConfigs->Array.map(chain => chain.id),
|
|
408
388
|
~contractMapping=stored.contractMapping,
|
|
409
389
|
)
|
package/src/Persistence.res.mjs
CHANGED
|
@@ -94,7 +94,7 @@ async function init(persistence, logger, chainConfigs, contractMapping, envioInf
|
|
|
94
94
|
Error: new Error()
|
|
95
95
|
};
|
|
96
96
|
}
|
|
97
|
-
let initialState = await persistence.storage.initialize(chainConfigs$1,
|
|
97
|
+
let initialState = await persistence.storage.initialize(chainConfigs$1, contractMapping, envioInfo);
|
|
98
98
|
Logger.info(logger, `The indexer storage is ready. Starting indexing!`, undefined);
|
|
99
99
|
persistence.storageStatus = {
|
|
100
100
|
TAG: "Ready",
|
|
@@ -136,11 +136,11 @@ async function init(persistence, logger, chainConfigs, contractMapping, envioInf
|
|
|
136
136
|
chainId: chainConfig.id
|
|
137
137
|
});
|
|
138
138
|
let chainConfig$1 = (await StartBlockResolver.resolveAllOrThrow([chainConfig], logger, lowercaseAddresses, startBlockRetry, sourceTiming))[0];
|
|
139
|
-
await persistence.storage.addChain(chainConfig$1,
|
|
139
|
+
await persistence.storage.addChain(chainConfig$1, stored.contractMapping);
|
|
140
140
|
}
|
|
141
141
|
}
|
|
142
142
|
if (resume) {
|
|
143
|
-
let initialState$1 = await persistence.storage.resumeInitialState(
|
|
143
|
+
let initialState$1 = await persistence.storage.resumeInitialState(chainConfigs.map(chain => chain.id), stored.contractMapping);
|
|
144
144
|
persistence.storageStatus = {
|
|
145
145
|
TAG: "Ready",
|
|
146
146
|
_0: initialState$1
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
// Preload runs a batch's handlers against the store as it was before the batch,
|
|
2
|
+
// so a handler reading an entity an earlier event of the batch creates (a v3
|
|
3
|
+
// pool's Mint after its PoolCreated) reads it as absent and makes none of the
|
|
4
|
+
// contract calls after that read. The execute pass then asks them one event
|
|
5
|
+
// at a time, a round trip each.
|
|
6
|
+
//
|
|
7
|
+
// So preload reads that find an entity absent are remembered per item, and once
|
|
8
|
+
// execute creates that entity, the items that read it are preloaded again, as
|
|
9
|
+
// long as they are within `window` items of the one executing. Preload only
|
|
10
|
+
// warms the store and the effect cache, and every value it fetches is keyed by
|
|
11
|
+
// its full input, so what execute computes is the same with or without it.
|
|
12
|
+
//
|
|
13
|
+
// Preload also sees what earlier items of the batch wrote in their own preload
|
|
14
|
+
// (`overlay`): agent0's URIUpdated(n), the log right after the Registered(n)
|
|
15
|
+
// that creates agent n, would otherwise find no agent and return before its
|
|
16
|
+
// call, and preloading it again once agent n exists is too late — it executes
|
|
17
|
+
// right after. Those writes are a guess at the store execute will see; they
|
|
18
|
+
// are held here, never in the store, and only an item before the reader that
|
|
19
|
+
// execute hasn't run yet is taken: once it has, the store is the answer.
|
|
20
|
+
//
|
|
21
|
+
// Preloading again costs the handler's CPU time once more, and an item whose
|
|
22
|
+
// preloads have already spent `maxPreloadMs` (a v3 Mint computing 1.0001^tick
|
|
23
|
+
// a step at a time) isn't preloaded again: its calls would land later than the
|
|
24
|
+
// CPU it takes away from execute.
|
|
25
|
+
|
|
26
|
+
type t = {
|
|
27
|
+
items: array<Internal.item>,
|
|
28
|
+
indexOf: Utils.Map.t<Internal.item, int>,
|
|
29
|
+
absentReaders: Map.t<string, array<int>>,
|
|
30
|
+
dirty: array<bool>,
|
|
31
|
+
// Milliseconds the item's preload rounds have run for, all runs together.
|
|
32
|
+
preloadMs: array<float>,
|
|
33
|
+
// By entity key, the writes items' preloads made: (item, entity or deleted).
|
|
34
|
+
overlay: Map.t<string, array<(int, option<Internal.entity>)>>,
|
|
35
|
+
window: int,
|
|
36
|
+
// The last item execute finished; reads and writes at or before it are history.
|
|
37
|
+
mutable executed: int,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let active: ref<option<t>> = ref(None)
|
|
41
|
+
|
|
42
|
+
// Subgraph mappings make their contract calls in the middle of a handler, after
|
|
43
|
+
// the reads, so a read that misses is the difference between preloading the
|
|
44
|
+
// calls and not. Native handlers are left as they are.
|
|
45
|
+
let start = (~items: array<Internal.item>) => {
|
|
46
|
+
let window = Env.subgraphPreloadLookahead
|
|
47
|
+
if Env.subgraphMode.contents && window > 0 {
|
|
48
|
+
let indexOf = Utils.Map.make()
|
|
49
|
+
items->Array.forEachWithIndex((item, idx) => indexOf->Utils.Map.set(item, idx)->ignore)
|
|
50
|
+
active :=
|
|
51
|
+
Some({
|
|
52
|
+
items,
|
|
53
|
+
indexOf,
|
|
54
|
+
absentReaders: Map.make(),
|
|
55
|
+
dirty: Array.make(~length=items->Array.length, false),
|
|
56
|
+
preloadMs: Array.make(~length=items->Array.length, 0.),
|
|
57
|
+
overlay: Map.make(),
|
|
58
|
+
window,
|
|
59
|
+
executed: -1,
|
|
60
|
+
})
|
|
61
|
+
} else {
|
|
62
|
+
active := None
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let stop = () => active := None
|
|
67
|
+
|
|
68
|
+
let maxPreloadMs = 20.
|
|
69
|
+
|
|
70
|
+
let roundStarted = (~isPreload) =>
|
|
71
|
+
switch active.contents {
|
|
72
|
+
| Some(_) if isPreload => Performance.now()
|
|
73
|
+
| _ => 0.
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
let roundEnded = (~item: Internal.item, ~startedAt) =>
|
|
77
|
+
if startedAt > 0. {
|
|
78
|
+
switch active.contents {
|
|
79
|
+
| None => ()
|
|
80
|
+
| Some(t) =>
|
|
81
|
+
switch t.indexOf->Utils.Map.get(item) {
|
|
82
|
+
| Some(idx) =>
|
|
83
|
+
t.preloadMs->Array.setUnsafe(
|
|
84
|
+
idx,
|
|
85
|
+
t.preloadMs->Array.getUnsafe(idx) +. startedAt->Performance.secondsSince *. 1000.,
|
|
86
|
+
)
|
|
87
|
+
| None => ()
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
let entityKey = (~entityName, ~scope: Internal.chainScope, ~entityId) =>
|
|
93
|
+
switch scope {
|
|
94
|
+
| CrossChain => `${entityName}..${entityId}`
|
|
95
|
+
| Chain(chainId) => `${entityName}.${chainId->ChainId.toString}.${entityId}`
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let recordAbsent = (~item: Internal.item, ~entityName, ~scope, ~entityId) =>
|
|
99
|
+
switch active.contents {
|
|
100
|
+
| None => ()
|
|
101
|
+
| Some(t) =>
|
|
102
|
+
switch t.indexOf->Utils.Map.get(item) {
|
|
103
|
+
| Some(idx) if idx > t.executed =>
|
|
104
|
+
let key = entityKey(~entityName, ~scope, ~entityId)
|
|
105
|
+
switch t.absentReaders->Map.get(key) {
|
|
106
|
+
| Some(readers) => readers->Array.push(idx)
|
|
107
|
+
| None => t.absentReaders->Map.set(key, [idx])
|
|
108
|
+
}
|
|
109
|
+
| _ => ()
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
let onSet = (~entityName, ~scope, ~entityId) =>
|
|
114
|
+
switch active.contents {
|
|
115
|
+
| None => ()
|
|
116
|
+
// Every write execute makes lands here, and most batches have no absent read.
|
|
117
|
+
| Some(t) if t.absentReaders->Map.size > 0 =>
|
|
118
|
+
let key = entityKey(~entityName, ~scope, ~entityId)
|
|
119
|
+
switch t.absentReaders->Map.get(key) {
|
|
120
|
+
| None => ()
|
|
121
|
+
| Some(readers) =>
|
|
122
|
+
t.absentReaders->Map.delete(key)->ignore
|
|
123
|
+
readers->Array.forEach(idx =>
|
|
124
|
+
if idx > t.executed {
|
|
125
|
+
t.dirty->Array.setUnsafe(idx, true)
|
|
126
|
+
}
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
| Some(_) => ()
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
let isActive = () => active.contents->Option.isSome
|
|
133
|
+
|
|
134
|
+
// How many of `writes` (sorted by item) are by an item before `item`. An entity
|
|
135
|
+
// every event of a batch touches (a popular holder) gathers a write per event,
|
|
136
|
+
// and a scan of them per read was quadratic in the batch.
|
|
137
|
+
let countWritesBeforeItem = (writes: array<(int, option<Internal.entity>)>, item) => {
|
|
138
|
+
let low = ref(0)
|
|
139
|
+
let high = ref(writes->Array.length)
|
|
140
|
+
while low.contents < high.contents {
|
|
141
|
+
let mid = (low.contents + high.contents) / 2
|
|
142
|
+
let (writer, _) = writes->Array.getUnsafe(mid)
|
|
143
|
+
if writer < item {
|
|
144
|
+
low := mid + 1
|
|
145
|
+
} else {
|
|
146
|
+
high := mid
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
low.contents
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
let overlaySet = (~item: Internal.item, ~entityName, ~scope, ~entityId, ~entity) =>
|
|
153
|
+
switch active.contents {
|
|
154
|
+
| None => ()
|
|
155
|
+
| Some(t) =>
|
|
156
|
+
switch t.indexOf->Utils.Map.get(item) {
|
|
157
|
+
| Some(writer) =>
|
|
158
|
+
let key = entityKey(~entityName, ~scope, ~entityId)
|
|
159
|
+
switch t.overlay->Map.get(key) {
|
|
160
|
+
| Some(writes) =>
|
|
161
|
+
let at = writes->countWritesBeforeItem(writer)
|
|
162
|
+
switch writes->Array.get(at) {
|
|
163
|
+
| Some((idx, _)) if idx === writer => writes->Array.setUnsafe(at, (writer, entity))
|
|
164
|
+
| _ => writes->Array.splice(~start=at, ~remove=0, ~insert=[(writer, entity)])
|
|
165
|
+
}
|
|
166
|
+
| None => t.overlay->Map.set(key, [(writer, entity)])
|
|
167
|
+
}
|
|
168
|
+
| None => ()
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// `Some(entity or None for deleted)` when an item before the reader that execute
|
|
173
|
+
// hasn't run yet wrote the entity in its preload: the latest such write.
|
|
174
|
+
let overlayGet = (~item: Internal.item, ~entityName, ~scope, ~entityId) =>
|
|
175
|
+
switch active.contents {
|
|
176
|
+
| None => None
|
|
177
|
+
| Some(t) =>
|
|
178
|
+
switch (
|
|
179
|
+
t.indexOf->Utils.Map.get(item),
|
|
180
|
+
t.overlay->Map.get(entityKey(~entityName, ~scope, ~entityId)),
|
|
181
|
+
) {
|
|
182
|
+
| (Some(reader), Some(writes)) =>
|
|
183
|
+
switch writes->Array.get(writes->countWritesBeforeItem(reader) - 1) {
|
|
184
|
+
| Some((writer, entity)) if writer > t.executed => Some(entity)
|
|
185
|
+
| _ => None
|
|
186
|
+
}
|
|
187
|
+
| _ => None
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Called once execute has finished the item at `index`: the dirty items in the
|
|
192
|
+
// window ahead of it are preloaded again against the store as it is now.
|
|
193
|
+
let afterItem = (t: t, ~index, ~preload: (Internal.item, int) => unit) => {
|
|
194
|
+
t.executed = index
|
|
195
|
+
let last = Math.Int.min(t.items->Array.length - 1, index + t.window)
|
|
196
|
+
for idx in index + 1 to last {
|
|
197
|
+
if t.dirty->Array.getUnsafe(idx) && t.preloadMs->Array.getUnsafe(idx) < maxPreloadMs {
|
|
198
|
+
t.dirty->Array.setUnsafe(idx, false)
|
|
199
|
+
preload(t.items->Array.getUnsafe(idx), idx)
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Env from "./Env.res.mjs";
|
|
4
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
5
|
+
import * as Performance from "./bindings/Performance.res.mjs";
|
|
6
|
+
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
7
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
8
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
9
|
+
|
|
10
|
+
let active = {
|
|
11
|
+
contents: undefined
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
function start(items) {
|
|
15
|
+
if (!(Env.subgraphMode.contents && Env.subgraphPreloadLookahead > 0)) {
|
|
16
|
+
active.contents = undefined;
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
let indexOf = new Map();
|
|
20
|
+
items.forEach((item, idx) => {
|
|
21
|
+
indexOf.set(item, idx);
|
|
22
|
+
});
|
|
23
|
+
active.contents = {
|
|
24
|
+
items: items,
|
|
25
|
+
indexOf: indexOf,
|
|
26
|
+
absentReaders: new Map(),
|
|
27
|
+
dirty: Stdlib_Array.make(items.length, false),
|
|
28
|
+
preloadMs: Stdlib_Array.make(items.length, 0),
|
|
29
|
+
overlay: new Map(),
|
|
30
|
+
window: Env.subgraphPreloadLookahead,
|
|
31
|
+
executed: -1
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function stop() {
|
|
36
|
+
active.contents = undefined;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function roundStarted(isPreload) {
|
|
40
|
+
let match = active.contents;
|
|
41
|
+
if (match !== undefined) {
|
|
42
|
+
if (isPreload) {
|
|
43
|
+
return Performance.now();
|
|
44
|
+
} else {
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
} else {
|
|
48
|
+
return 0;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function roundEnded(item, startedAt) {
|
|
53
|
+
if (startedAt <= 0) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
let t = active.contents;
|
|
57
|
+
if (t === undefined) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
let idx = t.indexOf.get(item);
|
|
61
|
+
if (idx !== undefined) {
|
|
62
|
+
t.preloadMs[idx] = t.preloadMs[idx] + Performance.secondsSince(startedAt) * 1000;
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function entityKey(entityName, scope, entityId) {
|
|
68
|
+
if (scope === "crossChain") {
|
|
69
|
+
return entityName + `..` + entityId;
|
|
70
|
+
} else {
|
|
71
|
+
return entityName + `.` + ChainId.toString(scope) + `.` + entityId;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function recordAbsent(item, entityName, scope, entityId) {
|
|
76
|
+
let t = active.contents;
|
|
77
|
+
if (t === undefined) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
let idx = t.indexOf.get(item);
|
|
81
|
+
if (idx === undefined) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
if (idx <= t.executed) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
let key = entityKey(entityName, scope, entityId);
|
|
88
|
+
let readers = t.absentReaders.get(key);
|
|
89
|
+
if (readers !== undefined) {
|
|
90
|
+
readers.push(idx);
|
|
91
|
+
} else {
|
|
92
|
+
t.absentReaders.set(key, [idx]);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function onSet(entityName, scope, entityId) {
|
|
97
|
+
let t = active.contents;
|
|
98
|
+
if (t === undefined) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (t.absentReaders.size <= 0) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
let key = entityKey(entityName, scope, entityId);
|
|
105
|
+
let readers = t.absentReaders.get(key);
|
|
106
|
+
if (readers !== undefined) {
|
|
107
|
+
t.absentReaders.delete(key);
|
|
108
|
+
readers.forEach(idx => {
|
|
109
|
+
if (idx > t.executed) {
|
|
110
|
+
t.dirty[idx] = true;
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function isActive() {
|
|
119
|
+
return Stdlib_Option.isSome(active.contents);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function countWritesBeforeItem(writes, item) {
|
|
123
|
+
let low = 0;
|
|
124
|
+
let high = writes.length;
|
|
125
|
+
while (low < high) {
|
|
126
|
+
let mid = (low + high | 0) / 2 | 0;
|
|
127
|
+
let match = writes[mid];
|
|
128
|
+
if (match[0] < item) {
|
|
129
|
+
low = mid + 1 | 0;
|
|
130
|
+
} else {
|
|
131
|
+
high = mid;
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
return low;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function overlaySet(item, entityName, scope, entityId, entity) {
|
|
138
|
+
let t = active.contents;
|
|
139
|
+
if (t === undefined) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
let writer = t.indexOf.get(item);
|
|
143
|
+
if (writer === undefined) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
let key = entityKey(entityName, scope, entityId);
|
|
147
|
+
let writes = t.overlay.get(key);
|
|
148
|
+
if (writes !== undefined) {
|
|
149
|
+
let at = countWritesBeforeItem(writes, writer);
|
|
150
|
+
let match = writes[at];
|
|
151
|
+
if (match !== undefined && match[0] === writer) {
|
|
152
|
+
writes[at] = [
|
|
153
|
+
writer,
|
|
154
|
+
entity
|
|
155
|
+
];
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
writes.splice(at, 0, [
|
|
159
|
+
writer,
|
|
160
|
+
entity
|
|
161
|
+
]);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
t.overlay.set(key, [[
|
|
165
|
+
writer,
|
|
166
|
+
entity
|
|
167
|
+
]]);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function overlayGet(item, entityName, scope, entityId) {
|
|
171
|
+
let t = active.contents;
|
|
172
|
+
if (t === undefined) {
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
let match = t.indexOf.get(item);
|
|
176
|
+
let match$1 = t.overlay.get(entityKey(entityName, scope, entityId));
|
|
177
|
+
if (match === undefined) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
if (match$1 === undefined) {
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
let match$2 = match$1[countWritesBeforeItem(match$1, match) - 1 | 0];
|
|
184
|
+
if (match$2 !== undefined && match$2[0] > t.executed) {
|
|
185
|
+
return Primitive_option.some(match$2[1]);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function afterItem(t, index, preload) {
|
|
190
|
+
t.executed = index;
|
|
191
|
+
let last = Math.min(t.items.length - 1 | 0, index + t.window | 0);
|
|
192
|
+
for (let idx = index + 1 | 0; idx <= last; ++idx) {
|
|
193
|
+
if (t.dirty[idx] && t.preloadMs[idx] < 20) {
|
|
194
|
+
t.dirty[idx] = false;
|
|
195
|
+
preload(t.items[idx], idx);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
let maxPreloadMs = 20;
|
|
201
|
+
|
|
202
|
+
export {
|
|
203
|
+
active,
|
|
204
|
+
start,
|
|
205
|
+
stop,
|
|
206
|
+
maxPreloadMs,
|
|
207
|
+
roundStarted,
|
|
208
|
+
roundEnded,
|
|
209
|
+
entityKey,
|
|
210
|
+
recordAbsent,
|
|
211
|
+
onSet,
|
|
212
|
+
isActive,
|
|
213
|
+
countWritesBeforeItem,
|
|
214
|
+
overlaySet,
|
|
215
|
+
overlayGet,
|
|
216
|
+
afterItem,
|
|
217
|
+
}
|
|
218
|
+
/* Env Not a pure module */
|
package/src/Rollback.res
CHANGED
|
@@ -130,7 +130,6 @@ and executeRollback = async (
|
|
|
130
130
|
switch await (state->IndexerState.persistence).storage.getRollbackTargetCheckpoint(
|
|
131
131
|
~reorgChainId=reorgChain,
|
|
132
132
|
~lastKnownValidBlockNumber=rollbackTargetBlockNumber,
|
|
133
|
-
~sequence=(state->IndexerState.config).checkpointSequence,
|
|
134
133
|
) {
|
|
135
134
|
| Some(checkpointId) => checkpointId
|
|
136
135
|
| None => 0n
|
package/src/Rollback.res.mjs
CHANGED
|
@@ -42,7 +42,7 @@ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, sch
|
|
|
42
42
|
Logger.info(logger, "Started rollback on reorg", undefined);
|
|
43
43
|
ChainState.setRollbackTargetBlock(IndexerState.getChainState(state, reorgChain), rollbackTargetBlockNumber);
|
|
44
44
|
await Writing.flush(state);
|
|
45
|
-
let checkpointId = await IndexerState.persistence(state).storage.getRollbackTargetCheckpoint(reorgChain, rollbackTargetBlockNumber
|
|
45
|
+
let checkpointId = await IndexerState.persistence(state).storage.getRollbackTargetCheckpoint(reorgChain, rollbackTargetBlockNumber);
|
|
46
46
|
let rollbackTargetCheckpointId = checkpointId !== undefined ? checkpointId : 0n;
|
|
47
47
|
let config = IndexerState.config(state);
|
|
48
48
|
let next = RollbackFloors.make(config.checkpointSequence, ChainMap.keys(config.chainMap), reorgChain, rollbackTargetCheckpointId, rollbackTargetBlockNumber);
|
package/src/RunScope.res
CHANGED
|
@@ -22,6 +22,9 @@ type contractRegisterRun = {
|
|
|
22
22
|
chainLogger: Logger.t,
|
|
23
23
|
ecosystem: Ecosystem.t,
|
|
24
24
|
registrations: array<AddressStore.registration>,
|
|
25
|
+
// Subgraph mode: the creation round the addresses it adds belong to (see
|
|
26
|
+
// `AddressStore.registration.creationRound`).
|
|
27
|
+
creationRound?: int,
|
|
25
28
|
}
|
|
26
29
|
|
|
27
30
|
// An effect call is shared by the runs that asked for it, so it is told by the
|
|
@@ -16,9 +16,7 @@ let make = (
|
|
|
16
16
|
) => {
|
|
17
17
|
if maxReorgDepth > 0 && shouldRollbackOnReorg {
|
|
18
18
|
let checkpointIds = Utils.Array.jsArrayCreate(chainReorgCheckpoints->Array.length)
|
|
19
|
-
let checkpointBlockNumbers = Utils.Array.jsArrayCreate(
|
|
20
|
-
chainReorgCheckpoints->Array.length,
|
|
21
|
-
)
|
|
19
|
+
let checkpointBlockNumbers = Utils.Array.jsArrayCreate(chainReorgCheckpoints->Array.length)
|
|
22
20
|
chainReorgCheckpoints->Array.forEachWithIndex((checkpoint, idx) => {
|
|
23
21
|
checkpointIds->Array.setUnsafe(idx, checkpoint.checkpointId)
|
|
24
22
|
checkpointBlockNumbers->Array.setUnsafe(idx, checkpoint.blockNumber)
|
|
@@ -36,7 +34,9 @@ let make = (
|
|
|
36
34
|
let getSafeCheckpointId = (safeCheckpointTracking: t, ~sourceBlockNumber: int): option<
|
|
37
35
|
Internal.checkpointId,
|
|
38
36
|
> => {
|
|
39
|
-
|
|
37
|
+
// The first block reorg detection compares is itself reorgable, so the safe
|
|
38
|
+
// block is the one below it.
|
|
39
|
+
let safeBlockNumber = sourceBlockNumber - safeCheckpointTracking.maxReorgDepth - 1
|
|
40
40
|
|
|
41
41
|
switch safeCheckpointTracking.checkpointIds {
|
|
42
42
|
| [] => None
|
|
@@ -20,7 +20,7 @@ function make(maxReorgDepth, shouldRollbackOnReorg, chainReorgCheckpoints) {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
function getSafeCheckpointId(safeCheckpointTracking, sourceBlockNumber) {
|
|
23
|
-
let safeBlockNumber = sourceBlockNumber - safeCheckpointTracking.maxReorgDepth | 0;
|
|
23
|
+
let safeBlockNumber = (sourceBlockNumber - safeCheckpointTracking.maxReorgDepth | 0) - 1 | 0;
|
|
24
24
|
let match = safeCheckpointTracking.checkpointIds;
|
|
25
25
|
if (match.length === 0) {
|
|
26
26
|
return;
|
package/src/SimulateItems.res
CHANGED
|
@@ -638,7 +638,16 @@ let parse = (
|
|
|
638
638
|
// A simulate item must land on a distinct (block, logIndex): event ordering
|
|
639
639
|
// and the dead-input tracker both key on it. Catch explicit duplicates and
|
|
640
640
|
// explicit-vs-auto-increment collisions here, naming both offending indices.
|
|
641
|
-
|
|
641
|
+
// A call is ordered by its position in its transaction's trace, so two
|
|
642
|
+
// transactions' calls share an index.
|
|
643
|
+
let coordinate = switch Internal.callRankOfEventName(eventConfig.name) {
|
|
644
|
+
| Some(_) =>
|
|
645
|
+
let transactionIndex = (
|
|
646
|
+
transaction->(Utils.magic: Internal.eventTransaction => {"transactionIndex": int})
|
|
647
|
+
)["transactionIndex"]
|
|
648
|
+
`${blockNumber->Int.toString}:tx${transactionIndex->Int.toString}:${logIndex->Int.toString}`
|
|
649
|
+
| None => `${blockNumber->Int.toString}:${logIndex->Int.toString}`
|
|
650
|
+
}
|
|
642
651
|
switch seenCoordinates->Dict.get(coordinate) {
|
|
643
652
|
| Some(firstIndex) =>
|
|
644
653
|
JsError.throwWithMessage(
|
|
@@ -673,31 +682,38 @@ let parse = (
|
|
|
673
682
|
let onEventRegistration = {...reg, index: onEventRegistrationIndex}
|
|
674
683
|
onEventRegistrations->Array.push(onEventRegistration)->ignore
|
|
675
684
|
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
685
|
+
let item = Internal.Event({
|
|
686
|
+
onEventRegistration,
|
|
687
|
+
chainId,
|
|
688
|
+
blockNumber,
|
|
689
|
+
logIndex,
|
|
690
|
+
// Simulate keeps the transaction inline on the payload, so the store
|
|
691
|
+
// key is unused; a subgraph's call triggers order by it among logs.
|
|
692
|
+
transactionIndex: switch config.ecosystem.name {
|
|
693
|
+
| Evm =>
|
|
694
|
+
(
|
|
695
|
+
transaction->(Utils.magic: Internal.eventTransaction => {"transactionIndex": int})
|
|
696
|
+
)["transactionIndex"]
|
|
697
|
+
| _ => 0
|
|
698
|
+
},
|
|
699
|
+
payload: (
|
|
700
|
+
{
|
|
701
|
+
contractName: eventConfig.contractName,
|
|
702
|
+
eventName: eventConfig.name,
|
|
703
|
+
params,
|
|
704
|
+
chainId,
|
|
705
|
+
srcAddress,
|
|
706
|
+
logIndex,
|
|
707
|
+
transaction,
|
|
708
|
+
block,
|
|
709
|
+
}: Evm.payload
|
|
710
|
+
)->Evm.fromPayload,
|
|
711
|
+
})
|
|
712
|
+
switch Internal.callRankOfEventName(eventConfig.name) {
|
|
713
|
+
| Some(rank) => item->Internal.markItemCall(~rank)
|
|
714
|
+
| None => ()
|
|
715
|
+
}
|
|
716
|
+
items->Array.push(item)->ignore
|
|
701
717
|
})
|
|
702
718
|
}
|
|
703
719
|
})
|