@aztec/aztec-node 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/aztec-node/block_response_helpers.d.ts +25 -0
- package/dest/aztec-node/block_response_helpers.d.ts.map +1 -0
- package/dest/aztec-node/block_response_helpers.js +112 -0
- package/dest/aztec-node/config.d.ts +14 -8
- package/dest/aztec-node/config.d.ts.map +1 -1
- package/dest/aztec-node/config.js +16 -8
- package/dest/aztec-node/node_metrics.d.ts +1 -1
- package/dest/aztec-node/node_metrics.d.ts.map +1 -1
- package/dest/aztec-node/node_metrics.js +9 -16
- package/dest/aztec-node/public_data_overrides.d.ts +13 -0
- package/dest/aztec-node/public_data_overrides.d.ts.map +1 -0
- package/dest/aztec-node/public_data_overrides.js +21 -0
- package/dest/aztec-node/server.d.ts +101 -187
- package/dest/aztec-node/server.d.ts.map +1 -1
- package/dest/aztec-node/server.js +1528 -513
- package/dest/sentinel/config.d.ts +3 -2
- package/dest/sentinel/config.d.ts.map +1 -1
- package/dest/sentinel/config.js +15 -5
- package/dest/sentinel/factory.d.ts +5 -3
- package/dest/sentinel/factory.d.ts.map +1 -1
- package/dest/sentinel/factory.js +5 -5
- package/dest/sentinel/sentinel.d.ts +137 -12
- package/dest/sentinel/sentinel.d.ts.map +1 -1
- package/dest/sentinel/sentinel.js +273 -100
- package/dest/sentinel/store.d.ts +8 -8
- package/dest/sentinel/store.d.ts.map +1 -1
- package/dest/sentinel/store.js +33 -21
- package/dest/test/index.d.ts +3 -3
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +28 -26
- package/src/aztec-node/block_response_helpers.ts +161 -0
- package/src/aztec-node/config.ts +36 -16
- package/src/aztec-node/node_metrics.ts +6 -17
- package/src/aztec-node/public_data_overrides.ts +35 -0
- package/src/aztec-node/server.ts +1282 -663
- package/src/sentinel/README.md +103 -0
- package/src/sentinel/config.ts +18 -6
- package/src/sentinel/factory.ts +9 -11
- package/src/sentinel/sentinel.ts +336 -109
- package/src/sentinel/store.ts +35 -27
- package/src/test/index.ts +2 -2
|
@@ -1,18 +1,98 @@
|
|
|
1
|
-
import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
1
|
+
import { BlockNumber, CheckpointProposalHash, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { countWhile, filterAsync, fromEntries, getEntries, mapValues } from '@aztec/foundation/collection';
|
|
3
3
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
4
|
import { createLogger } from '@aztec/foundation/log';
|
|
5
5
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
6
6
|
import { L2TipsMemoryStore } from '@aztec/kv-store/stores';
|
|
7
|
-
import { OffenseType, WANT_TO_SLASH_EVENT } from '@aztec/slasher';
|
|
8
|
-
import { L2BlockStream,
|
|
7
|
+
import { OffenseType, WANT_TO_SLASH_EVENT, getOffenseTypeName } from '@aztec/slasher';
|
|
8
|
+
import { L2BlockStream, getAttestationInfoFromPublishedCheckpoint } from '@aztec/stdlib/block';
|
|
9
9
|
import { getEpochAtSlot, getSlotRangeForEpoch, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
10
|
+
import { ConsensusPayload } from '@aztec/stdlib/p2p';
|
|
10
11
|
import EventEmitter from 'node:events';
|
|
11
|
-
|
|
12
|
+
/** Maps a validator status to its category: proposer or attestation. */ function statusToCategory(status) {
|
|
13
|
+
switch(status){
|
|
14
|
+
case 'attestation-sent':
|
|
15
|
+
case 'attestation-missed':
|
|
16
|
+
return 'attestation';
|
|
17
|
+
default:
|
|
18
|
+
return 'proposer';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The Sentinel observes validator behaviour every L2 slot, classifies it into a per-slot status,
|
|
23
|
+
* aggregates those statuses into per-epoch performance once each epoch is fully observed, and
|
|
24
|
+
* emits inactivity slash payloads when a validator has been inactive for the configured number
|
|
25
|
+
* of consecutive epochs.
|
|
26
|
+
*
|
|
27
|
+
* ## Two cadences
|
|
28
|
+
*
|
|
29
|
+
* The sentinel runs `work()` every quarter L2 slot and drives two independent pipelines:
|
|
30
|
+
*
|
|
31
|
+
* 1. **Per-slot activity recording.** `processSlot(currentSlot - 2)` runs once per slot, with a
|
|
32
|
+
* two-slot lag to let P2P attestations settle and the archiver catch up. It classifies each
|
|
33
|
+
* committee member's behaviour for that slot via `getSlotActivity` and persists the result to
|
|
34
|
+
* `SentinelStore.historyMap` (sliding window of `sentinelHistoryLengthInEpochs * epochDuration`
|
|
35
|
+
* slots, default 24 epochs).
|
|
36
|
+
*
|
|
37
|
+
* 2. **Per-epoch evaluation.** `processEpochEnds(currentSlot)` runs every tick too. Once
|
|
38
|
+
* `sentinelEpochEndBufferSlots` (default 2) has elapsed past an epoch's last slot AND the
|
|
39
|
+
* per-slot recorder has covered that last slot, the sentinel calls `handleEpochEnd(epoch)`.
|
|
40
|
+
* That aggregates the slot-level statuses for the epoch into per-validator `{missed, total}`,
|
|
41
|
+
* persists it to `SentinelStore.epochMap` (default 2000-epoch window), and runs the slashing
|
|
42
|
+
* decision.
|
|
43
|
+
*
|
|
44
|
+
* Triggering per-epoch evaluation off local L2 state — rather than waiting for L1 proof
|
|
45
|
+
* publication — decouples slashing from prover availability.
|
|
46
|
+
*
|
|
47
|
+
* ## Six-case taxonomy in `getSlotActivity`
|
|
48
|
+
*
|
|
49
|
+
* For each slot, the sentinel assigns the proposer one of six statuses, ranked highest-confidence
|
|
50
|
+
* first:
|
|
51
|
+
*
|
|
52
|
+
* - `checkpoint-mined` — a checkpoint covering this slot has landed on L1
|
|
53
|
+
* (`slotNumberToCheckpoint` populated from `chain-checkpointed`).
|
|
54
|
+
* - `checkpoint-valid` — the local node re-executed a checkpoint proposal for this slot
|
|
55
|
+
* successfully (consulted via `CheckpointReexecutionTracker`).
|
|
56
|
+
* - `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this slot
|
|
57
|
+
* and rejected it (e.g. header/archive/out-hash mismatch, limit
|
|
58
|
+
* breach). Proposer-fault.
|
|
59
|
+
* - `checkpoint-unvalidated` — the local node observed a checkpoint proposal but could not
|
|
60
|
+
* validate it (missing blocks/txs, timeouts). Treated as
|
|
61
|
+
* proposer-fault for slashing.
|
|
62
|
+
* - `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal at all.
|
|
63
|
+
* - `blocks-missed` — no block proposals seen for this slot.
|
|
64
|
+
*
|
|
65
|
+
* Missing-attestor faults are recorded only in `checkpoint-mined` and `checkpoint-valid`, where
|
|
66
|
+
* the local node has positive evidence the checkpoint was canonical or valid. In the other four
|
|
67
|
+
* cases the proposer is at fault and no attestor penalty applies.
|
|
68
|
+
*
|
|
69
|
+
* ## Re-execution tracker
|
|
70
|
+
*
|
|
71
|
+
* `CheckpointReexecutionTracker` is populated by the validator client's checkpoint proposal
|
|
72
|
+
* handler. Every early return in `validateCheckpointProposal` records an outcome
|
|
73
|
+
* (`valid` / `invalid` / `unvalidated`) keyed by slot.
|
|
74
|
+
*
|
|
75
|
+
* ## Inactivity slashing
|
|
76
|
+
*
|
|
77
|
+
* `handleEpochPerformance` filters the epoch's per-validator stats by
|
|
78
|
+
* `slashInactivityTargetPercentage` and then calls `checkPastInactivity` to require
|
|
79
|
+
* `slashInactivityConsecutiveEpochThreshold` consecutive past epochs over the same threshold
|
|
80
|
+
* (read from `SentinelStore.epochMap`). Only validators meeting both conditions are emitted as
|
|
81
|
+
* `WANT_TO_SLASH_EVENT` with `OffenseType.INACTIVITY`. The slot-level counters that feed this —
|
|
82
|
+
* `missedProposals` and `missedAttestations` — include the four proposer-fault statuses plus
|
|
83
|
+
* `attestation-missed`.
|
|
84
|
+
*
|
|
85
|
+
* ## Escape hatch
|
|
86
|
+
*
|
|
87
|
+
* If `epochCache.getCommittee(slot)` reports `isEscapeHatchOpen`, per-slot recording is skipped
|
|
88
|
+
* (no history entries for that slot) and per-epoch evaluation writes an empty performance map
|
|
89
|
+
* (no slashing).
|
|
90
|
+
*/ export class Sentinel extends EventEmitter {
|
|
12
91
|
epochCache;
|
|
13
92
|
archiver;
|
|
14
93
|
p2p;
|
|
15
94
|
store;
|
|
95
|
+
reexecutionTracker;
|
|
16
96
|
config;
|
|
17
97
|
logger;
|
|
18
98
|
runningPromise;
|
|
@@ -20,14 +100,20 @@ export class Sentinel extends EventEmitter {
|
|
|
20
100
|
l2TipsStore;
|
|
21
101
|
initialSlot;
|
|
22
102
|
lastProcessedSlot;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
constructor(epochCache, archiver, p2p, store, config, logger = createLogger('node:sentinel')){
|
|
26
|
-
super(), this.epochCache = epochCache, this.archiver = archiver, this.p2p = p2p, this.store = store, this.config = config, this.logger = logger, this.
|
|
27
|
-
this.l2TipsStore = new L2TipsMemoryStore();
|
|
103
|
+
/** Largest epoch number for which the end-of-epoch aggregator has run. */ lastEvaluatedEpoch;
|
|
104
|
+
slotNumberToCheckpoint;
|
|
105
|
+
constructor(epochCache, archiver, p2p, store, reexecutionTracker, config, logger = createLogger('node:sentinel')){
|
|
106
|
+
super(), this.epochCache = epochCache, this.archiver = archiver, this.p2p = p2p, this.store = store, this.reexecutionTracker = reexecutionTracker, this.config = config, this.logger = logger, this.slotNumberToCheckpoint = new Map();
|
|
107
|
+
this.l2TipsStore = new L2TipsMemoryStore(archiver.getGenesisBlockHash());
|
|
28
108
|
const interval = epochCache.getL1Constants().ethereumSlotDuration * 1000 / 4;
|
|
29
109
|
this.runningPromise = new RunningPromise(this.work.bind(this), logger, interval);
|
|
30
110
|
}
|
|
111
|
+
getSignatureContext() {
|
|
112
|
+
return {
|
|
113
|
+
chainId: this.config.l1ChainId,
|
|
114
|
+
rollupAddress: this.config.rollupAddress
|
|
115
|
+
};
|
|
116
|
+
}
|
|
31
117
|
updateConfig(config) {
|
|
32
118
|
this.config = {
|
|
33
119
|
...this.config,
|
|
@@ -38,64 +124,80 @@ export class Sentinel extends EventEmitter {
|
|
|
38
124
|
await this.init();
|
|
39
125
|
this.runningPromise.start();
|
|
40
126
|
}
|
|
41
|
-
/**
|
|
42
|
-
|
|
127
|
+
/**
|
|
128
|
+
* Loads initial slot and initializes blockstream. We will not process anything at or before
|
|
129
|
+
* the initial slot. Floors at the archiver's synced L2 slot so the sentinel keeps making
|
|
130
|
+
* forward progress when L1 is advancing but L2 has no activity (the synced slot is driven by
|
|
131
|
+
* L1 sync, not by L2 blocks). Falls back to the wallclock if the archiver isn't ready yet
|
|
132
|
+
* (cold start).
|
|
133
|
+
*/ async init() {
|
|
134
|
+
this.initialSlot = await this.getCurrentSlot();
|
|
43
135
|
const startingBlock = BlockNumber(await this.archiver.getBlockNumber());
|
|
44
136
|
this.logger.info(`Starting validator sentinel with initial slot ${this.initialSlot} and block ${startingBlock}`);
|
|
45
137
|
this.blockStream = new L2BlockStream(this.archiver, this.l2TipsStore, this, this.logger, {
|
|
46
138
|
startingBlock
|
|
47
139
|
});
|
|
48
140
|
}
|
|
141
|
+
/**
|
|
142
|
+
* Returns the L2 slot the sentinel should treat as "current": the archiver's last fully
|
|
143
|
+
* synced L2 slot, falling back to the wallclock slot when the archiver isn't ready yet
|
|
144
|
+
* (cold start). Anchoring to the synced slot keeps timing arithmetic (initial floor,
|
|
145
|
+
* per-slot lag, end-of-epoch buffer, stats-range fallback) from speculating ahead of where
|
|
146
|
+
* L1 actually is.
|
|
147
|
+
*/ async getCurrentSlot() {
|
|
148
|
+
return await this.archiver.getSyncedL2SlotNumber() ?? this.epochCache.getSlotNow();
|
|
149
|
+
}
|
|
49
150
|
stop() {
|
|
50
151
|
return this.runningPromise.stop();
|
|
51
152
|
}
|
|
52
153
|
async handleBlockStreamEvent(event) {
|
|
53
154
|
await this.l2TipsStore.handleBlockStreamEvent(event);
|
|
54
|
-
if (event.type === '
|
|
55
|
-
|
|
56
|
-
for (const block of event.blocks){
|
|
57
|
-
this.slotNumberToBlock.set(block.block.header.getSlot(), {
|
|
58
|
-
blockNumber: BlockNumber(block.block.number),
|
|
59
|
-
archive: block.block.archive.root.toString(),
|
|
60
|
-
attestors: getAttestationInfoFromPublishedL2Block(block).filter((a)=>a.status === 'recovered-from-signature').map((a)=>a.address)
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
// Prune the archive map to only keep at most N entries
|
|
64
|
-
const historyLength = this.store.getHistoryLength();
|
|
65
|
-
if (this.slotNumberToBlock.size > historyLength) {
|
|
66
|
-
const toDelete = Array.from(this.slotNumberToBlock.keys()).sort((a, b)=>Number(a - b)).slice(0, this.slotNumberToBlock.size - historyLength);
|
|
67
|
-
for (const key of toDelete){
|
|
68
|
-
this.slotNumberToBlock.delete(key);
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
} else if (event.type === 'chain-proven') {
|
|
72
|
-
await this.handleChainProven(event);
|
|
155
|
+
if (event.type === 'chain-checkpointed') {
|
|
156
|
+
this.handleCheckpoint(event);
|
|
73
157
|
}
|
|
74
158
|
}
|
|
75
|
-
|
|
76
|
-
if (event.type !== 'chain-
|
|
159
|
+
handleCheckpoint(event) {
|
|
160
|
+
if (event.type !== 'chain-checkpointed') {
|
|
77
161
|
return;
|
|
78
162
|
}
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
163
|
+
const checkpoint = event.checkpoint;
|
|
164
|
+
// Store mapping from slot to archive, checkpoint number, attestors, and the consensus payload
|
|
165
|
+
// hash (used to query matching p2p attestations regardless of feeAssetPriceModifier variants).
|
|
166
|
+
const signatureContext = this.getSignatureContext();
|
|
167
|
+
const proposalPayloadHash = CheckpointProposalHash.fromBuffer(ConsensusPayload.fromCheckpoint(checkpoint.checkpoint, signatureContext).getPayloadHash());
|
|
168
|
+
this.slotNumberToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, {
|
|
169
|
+
checkpointNumber: checkpoint.checkpoint.number,
|
|
170
|
+
archive: checkpoint.checkpoint.archive.root.toString(),
|
|
171
|
+
proposalPayloadHash,
|
|
172
|
+
attestors: getAttestationInfoFromPublishedCheckpoint(checkpoint, signatureContext).filter((a)=>a.status === 'recovered-from-signature').map((a)=>a.address)
|
|
173
|
+
});
|
|
174
|
+
// Prune the archive map to only keep at most N entries
|
|
175
|
+
const historyLength = this.store.getHistoryLength();
|
|
176
|
+
if (this.slotNumberToCheckpoint.size > historyLength) {
|
|
177
|
+
const toDelete = Array.from(this.slotNumberToCheckpoint.keys()).sort((a, b)=>Number(a - b)).slice(0, this.slotNumberToCheckpoint.size - historyLength);
|
|
178
|
+
for (const key of toDelete){
|
|
179
|
+
this.slotNumberToCheckpoint.delete(key);
|
|
180
|
+
}
|
|
86
181
|
}
|
|
87
|
-
// TODO(palla/slash): We should only be computing proven performance if this is
|
|
88
|
-
// a full proof epoch and not a partial one, otherwise we'll end up with skewed stats.
|
|
89
|
-
const epoch = getEpochAtSlot(block.header.getSlot(), this.epochCache.getL1Constants());
|
|
90
|
-
this.logger.debug(`Computing proven performance for epoch ${epoch}`);
|
|
91
|
-
const performance = await this.computeProvenPerformance(epoch);
|
|
92
|
-
this.logger.info(`Computed proven performance for epoch ${epoch}`, performance);
|
|
93
|
-
await this.store.updateProvenPerformance(epoch, performance);
|
|
94
|
-
await this.handleProvenPerformance(epoch, performance);
|
|
95
182
|
}
|
|
96
|
-
|
|
183
|
+
/**
|
|
184
|
+
* Called once per epoch, after the configured end-of-epoch buffer has elapsed beyond the
|
|
185
|
+
* epoch's last slot. Computes per-epoch performance from the slot-level history collected
|
|
186
|
+
* by `processSlot` and emits any inactivity slash payloads.
|
|
187
|
+
*/ async handleEpochEnd(epoch) {
|
|
188
|
+
this.logger.debug(`Computing epoch performance for epoch ${epoch}`);
|
|
189
|
+
const performance = await this.computeEpochPerformance(epoch);
|
|
190
|
+
this.logger.info(`Computed epoch performance for epoch ${epoch}`, performance);
|
|
191
|
+
await this.store.updateEpochPerformance(epoch, performance);
|
|
192
|
+
await this.handleEpochPerformance(epoch, performance);
|
|
193
|
+
}
|
|
194
|
+
async computeEpochPerformance(epoch) {
|
|
97
195
|
const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, this.epochCache.getL1Constants());
|
|
98
|
-
const { committee } = await this.epochCache.getCommittee(fromSlot);
|
|
196
|
+
const { committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(fromSlot);
|
|
197
|
+
if (isEscapeHatchOpen) {
|
|
198
|
+
this.logger.info(`Skipping epoch performance for epoch ${epoch} - escape hatch is open`);
|
|
199
|
+
return {};
|
|
200
|
+
}
|
|
99
201
|
if (!committee) {
|
|
100
202
|
this.logger.trace(`No committee found for slot ${fromSlot}`);
|
|
101
203
|
return {};
|
|
@@ -131,8 +233,8 @@ export class Sentinel extends EventEmitter {
|
|
|
131
233
|
if (requiredConsecutiveEpochs === 0) {
|
|
132
234
|
return true;
|
|
133
235
|
}
|
|
134
|
-
// Get all historical performance for this validator
|
|
135
|
-
const allPerformance = await this.store.
|
|
236
|
+
// Get all historical per-epoch performance for this validator
|
|
237
|
+
const allPerformance = await this.store.getEpochPerformance(validator);
|
|
136
238
|
// Sort by epoch descending to get most recent first, keep only epochs strictly before the current one, and get the first N
|
|
137
239
|
const pastEpochs = allPerformance.sort((a, b)=>Number(b.epoch - a.epoch)).filter((p)=>p.epoch < currentEpoch);
|
|
138
240
|
// If we don't have enough historical data, don't slash
|
|
@@ -141,13 +243,10 @@ export class Sentinel extends EventEmitter {
|
|
|
141
243
|
return false;
|
|
142
244
|
}
|
|
143
245
|
// Check that we have at least requiredConsecutiveEpochs and that all of them are above the inactivity threshold
|
|
144
|
-
return pastEpochs.slice(0, requiredConsecutiveEpochs).every((p)=>p.missed / p.total >= this.config.slashInactivityTargetPercentage);
|
|
246
|
+
return pastEpochs.slice(0, requiredConsecutiveEpochs).every((p)=>p.total === 0 ? false : p.missed / p.total >= this.config.slashInactivityTargetPercentage);
|
|
145
247
|
}
|
|
146
|
-
async
|
|
147
|
-
|
|
148
|
-
return;
|
|
149
|
-
}
|
|
150
|
-
const inactiveValidators = getEntries(performance).filter(([_, { missed, total }])=>missed / total >= this.config.slashInactivityTargetPercentage).map(([address])=>address);
|
|
248
|
+
async handleEpochPerformance(epoch, performance) {
|
|
249
|
+
const inactiveValidators = getEntries(performance).filter(([_, { missed, total }])=>total > 0 && missed / total >= this.config.slashInactivityTargetPercentage).map(([address])=>address);
|
|
151
250
|
this.logger.debug(`Found ${inactiveValidators.length} inactive validators in epoch ${epoch}`, {
|
|
152
251
|
inactiveValidators,
|
|
153
252
|
epoch,
|
|
@@ -162,8 +261,13 @@ export class Sentinel extends EventEmitter {
|
|
|
162
261
|
epochOrSlot: BigInt(epoch)
|
|
163
262
|
}));
|
|
164
263
|
if (criminals.length > 0) {
|
|
165
|
-
this.logger.
|
|
166
|
-
|
|
264
|
+
this.logger.info(`Identified ${criminals.length} inactivity offenses in at least ${epochThreshold} consecutive epochs`, {
|
|
265
|
+
offenses: args.map((arg)=>({
|
|
266
|
+
validator: arg.validator.toString(),
|
|
267
|
+
amount: arg.amount,
|
|
268
|
+
offenseType: getOffenseTypeName(arg.offenseType),
|
|
269
|
+
epochOrSlot: arg.epochOrSlot
|
|
270
|
+
})),
|
|
167
271
|
epochThreshold
|
|
168
272
|
});
|
|
169
273
|
this.emit(WANT_TO_SLASH_EVENT, args);
|
|
@@ -173,23 +277,61 @@ export class Sentinel extends EventEmitter {
|
|
|
173
277
|
* Process data for two L2 slots ago.
|
|
174
278
|
* Note that we do not process historical data, since we rely on p2p data for processing,
|
|
175
279
|
* and we don't have that data if we were offline during the period.
|
|
280
|
+
*
|
|
281
|
+
* `currentSlot` is anchored to the archiver's last synced L2 slot rather than the wallclock,
|
|
282
|
+
* so the per-slot lag (`isReadyToProcess`) and the end-of-epoch buffer (`processEpochEnds`)
|
|
283
|
+
* advance with archiver.
|
|
176
284
|
*/ async work() {
|
|
177
|
-
const
|
|
285
|
+
const currentSlot = await this.getCurrentSlot();
|
|
178
286
|
try {
|
|
179
287
|
// Manually sync the block stream to ensure we have the latest data.
|
|
180
288
|
// Note we never `start` the blockstream, so it loops at the same pace as we do.
|
|
181
289
|
await this.blockStream.sync();
|
|
182
|
-
//
|
|
290
|
+
// Per-slot activity recording (lag = 2 slots for P2P attestation settlement).
|
|
183
291
|
const targetSlot = await this.isReadyToProcess(currentSlot);
|
|
184
|
-
// And process it if we are.
|
|
185
292
|
if (targetSlot !== false) {
|
|
186
293
|
await this.processSlot(targetSlot);
|
|
187
294
|
}
|
|
295
|
+
// End-of-epoch evaluation (lag = sentinelEpochEndBufferSlots beyond the epoch's last slot).
|
|
296
|
+
await this.processEpochEnds(currentSlot);
|
|
188
297
|
} catch (err) {
|
|
189
298
|
this.logger.error(`Failed to process slot ${currentSlot}`, err);
|
|
190
299
|
}
|
|
191
300
|
}
|
|
192
301
|
/**
|
|
302
|
+
* After the configured buffer has elapsed past an epoch's last slot, runs the end-of-epoch
|
|
303
|
+
* aggregator for that epoch. Catches up if multiple epochs become eligible at once.
|
|
304
|
+
*/ async processEpochEnds(currentSlot) {
|
|
305
|
+
const constants = this.epochCache.getL1Constants();
|
|
306
|
+
const buffer = this.config.sentinelEpochEndBufferSlots;
|
|
307
|
+
if (currentSlot < buffer) {
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
if (this.initialSlot === undefined) {
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
// We can close epoch E iff:
|
|
314
|
+
// - the per-slot recorder has covered the epoch's last slot (lastProcessedSlot ≥ toSlot(E))
|
|
315
|
+
// - the buffer has elapsed past the epoch's last slot (currentSlot − buffer ≥ toSlot(E))
|
|
316
|
+
// - the epoch is not in the past relative to when the sentinel started (toSlot(E) > initialSlot)
|
|
317
|
+
if (this.lastProcessedSlot === undefined) {
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
const slotForBuffer = SlotNumber(currentSlot - buffer);
|
|
321
|
+
// First eligible epoch to close is the one after lastEvaluatedEpoch, or the epoch containing
|
|
322
|
+
// the initial slot if we haven't evaluated any yet (the initialSlot epoch may be partial — we
|
|
323
|
+
// don't try to evaluate it, we start from initialSlot's epoch + 1).
|
|
324
|
+
const startEpoch = this.lastEvaluatedEpoch !== undefined ? EpochNumber(this.lastEvaluatedEpoch + 1) : EpochNumber(getEpochAtSlot(this.initialSlot, constants) + 1);
|
|
325
|
+
for(let epoch = startEpoch;; epoch = EpochNumber(epoch + 1)){
|
|
326
|
+
const [, toSlot] = getSlotRangeForEpoch(epoch, constants);
|
|
327
|
+
if (toSlot > this.lastProcessedSlot || toSlot > slotForBuffer) {
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
await this.handleEpochEnd(epoch);
|
|
331
|
+
this.lastEvaluatedEpoch = epoch;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
193
335
|
* Check if we are ready to process data for two L2 slots ago, so we allow plenty of time for p2p to process all in-flight attestations.
|
|
194
336
|
* We also don't move past the archiver last synced L2 slot, as we don't want to process data that is not yet available.
|
|
195
337
|
* Last, we check the p2p is synced with the archiver, so it has pulled all attestations from it.
|
|
@@ -213,16 +355,16 @@ export class Sentinel extends EventEmitter {
|
|
|
213
355
|
this.logger.trace(`Refusing to process slot ${targetSlot} given initial slot ${this.initialSlot}`);
|
|
214
356
|
return false;
|
|
215
357
|
}
|
|
216
|
-
const
|
|
217
|
-
if (
|
|
358
|
+
const syncedSlot = await this.archiver.getSyncedL2SlotNumber();
|
|
359
|
+
if (syncedSlot === undefined || syncedSlot < targetSlot) {
|
|
218
360
|
this.logger.debug(`Waiting for archiver to sync with L2 slot ${targetSlot}`, {
|
|
219
|
-
|
|
361
|
+
syncedSlot,
|
|
220
362
|
targetSlot
|
|
221
363
|
});
|
|
222
364
|
return false;
|
|
223
365
|
}
|
|
224
|
-
const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then((tip)=>tip.
|
|
225
|
-
const p2pLastBlockHash = await this.p2p.getL2Tips().then((tips)=>tips.
|
|
366
|
+
const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then((tip)=>tip.proposed.hash);
|
|
367
|
+
const p2pLastBlockHash = await this.p2p.getL2Tips().then((tips)=>tips.proposed.hash);
|
|
226
368
|
const isP2pSynced = archiverLastBlockHash === p2pLastBlockHash;
|
|
227
369
|
if (!isP2pSynced) {
|
|
228
370
|
this.logger.debug(`Waiting for P2P client to sync with archiver`, {
|
|
@@ -237,7 +379,12 @@ export class Sentinel extends EventEmitter {
|
|
|
237
379
|
* Gathers committee and proposer data for a given slot, computes slot stats,
|
|
238
380
|
* and updates overall stats.
|
|
239
381
|
*/ async processSlot(slot) {
|
|
240
|
-
const { epoch, seed, committee } = await this.epochCache.getCommittee(slot);
|
|
382
|
+
const { epoch, seed, committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(slot);
|
|
383
|
+
if (isEscapeHatchOpen) {
|
|
384
|
+
this.logger.info(`Skipping slot ${slot} at epoch ${epoch} - escape hatch is open`);
|
|
385
|
+
this.lastProcessedSlot = slot;
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
241
388
|
if (!committee || committee.length === 0) {
|
|
242
389
|
this.logger.trace(`No committee found for slot ${slot} at epoch ${epoch}`);
|
|
243
390
|
this.lastProcessedSlot = slot;
|
|
@@ -250,43 +397,66 @@ export class Sentinel extends EventEmitter {
|
|
|
250
397
|
await this.updateValidators(slot, stats);
|
|
251
398
|
this.lastProcessedSlot = slot;
|
|
252
399
|
}
|
|
253
|
-
/**
|
|
400
|
+
/**
|
|
401
|
+
* Computes activity for a given slot using the six-case taxonomy.
|
|
402
|
+
*
|
|
403
|
+
* Proposer status:
|
|
404
|
+
* - case 6 `checkpoint-mined` — a checkpoint covering this slot has landed on L1.
|
|
405
|
+
* - case 5 `checkpoint-valid` — the local node re-executed a checkpoint proposal for this
|
|
406
|
+
* slot successfully.
|
|
407
|
+
* - case 4 `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this
|
|
408
|
+
* slot and rejected it.
|
|
409
|
+
* - case 3 `checkpoint-unvalidated` — the local node observed a checkpoint proposal for this
|
|
410
|
+
* slot but could not validate it (missing data, timeouts).
|
|
411
|
+
* - case 2 `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal.
|
|
412
|
+
* - case 1 `blocks-missed` — no block proposals seen for this slot.
|
|
413
|
+
*
|
|
414
|
+
* Missing-attestor penalties apply only in cases 5 and 6, where the local node has positive
|
|
415
|
+
* evidence the checkpoint was valid or has been canonicalised on L1.
|
|
416
|
+
*/ async getSlotActivity(slot, epoch, proposer, committee) {
|
|
254
417
|
this.logger.debug(`Computing stats for slot ${slot} at epoch ${epoch}`, {
|
|
255
418
|
slot,
|
|
256
419
|
epoch,
|
|
257
420
|
proposer,
|
|
258
421
|
committee
|
|
259
422
|
});
|
|
260
|
-
//
|
|
261
|
-
//
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
// (contains the ones synced from mined blocks, which we may have missed from p2p).
|
|
265
|
-
const block = this.slotNumberToBlock.get(slot);
|
|
266
|
-
const p2pAttested = await this.p2p.getAttestationsForSlot(slot, block?.archive);
|
|
267
|
-
// Filter out attestations with invalid signatures
|
|
423
|
+
// Gather attestors from both p2p (live attestations) and the archiver (signers on the
|
|
424
|
+
// checkpoint if one has landed on L1). Used regardless of which case applies.
|
|
425
|
+
const checkpoint = this.slotNumberToCheckpoint.get(slot);
|
|
426
|
+
const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, checkpoint?.proposalPayloadHash);
|
|
268
427
|
const p2pAttestors = p2pAttested.map((a)=>a.getSender()).filter((s)=>s !== undefined);
|
|
269
428
|
const attestors = new Set([
|
|
270
429
|
...p2pAttestors.map((a)=>a.toString()),
|
|
271
|
-
...
|
|
430
|
+
...checkpoint?.attestors.map((a)=>a.toString()) ?? []
|
|
272
431
|
].filter((addr)=>proposer.toString() !== addr));
|
|
273
|
-
//
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
432
|
+
// Determine the proposer status from the six-case taxonomy.
|
|
433
|
+
const reexecutionOutcome = this.reexecutionTracker.getOutcomeForSlot(slot);
|
|
434
|
+
let status;
|
|
435
|
+
if (checkpoint) {
|
|
436
|
+
status = 'checkpoint-mined';
|
|
437
|
+
} else if (reexecutionOutcome === 'valid') {
|
|
438
|
+
status = 'checkpoint-valid';
|
|
439
|
+
} else if (reexecutionOutcome === 'invalid') {
|
|
440
|
+
status = 'checkpoint-invalid';
|
|
441
|
+
} else if (reexecutionOutcome === 'unvalidated') {
|
|
442
|
+
status = 'checkpoint-unvalidated';
|
|
443
|
+
} else {
|
|
444
|
+
// No L1 checkpoint, no local re-execution outcome for this slot. Distinguish "proposer
|
|
445
|
+
// sent block proposals but never made a checkpoint" from "proposer sent nothing".
|
|
446
|
+
const hasBlockProposals = await this.p2p.hasBlockProposalsForSlot(slot);
|
|
447
|
+
status = hasBlockProposals ? 'checkpoint-missed' : 'blocks-missed';
|
|
448
|
+
}
|
|
449
|
+
this.logger.debug(`Checkpoint status for slot ${slot}: ${status}`, {
|
|
450
|
+
...checkpoint,
|
|
282
451
|
slot
|
|
283
452
|
});
|
|
284
|
-
//
|
|
285
|
-
const
|
|
453
|
+
// Missing-attestor faults only apply when we have positive evidence the proposal was valid.
|
|
454
|
+
const attestorsExpected = status === 'checkpoint-mined' || status === 'checkpoint-valid';
|
|
455
|
+
const missedAttestors = new Set(attestorsExpected ? committee.filter((v)=>!attestors.has(v.toString()) && !proposer.equals(v)).map((v)=>v.toString()) : []);
|
|
286
456
|
this.logger.debug(`Retrieved ${attestors.size} attestors out of ${committee.length} for slot ${slot}`, {
|
|
287
|
-
|
|
457
|
+
status,
|
|
288
458
|
proposer: proposer.toString(),
|
|
289
|
-
...
|
|
459
|
+
...checkpoint,
|
|
290
460
|
slot,
|
|
291
461
|
attestors: [
|
|
292
462
|
...attestors
|
|
@@ -299,7 +469,7 @@ export class Sentinel extends EventEmitter {
|
|
|
299
469
|
// Compute the status for each validator in the committee
|
|
300
470
|
const statusFor = (who)=>{
|
|
301
471
|
if (who === proposer.toString()) {
|
|
302
|
-
return
|
|
472
|
+
return status;
|
|
303
473
|
} else if (attestors.has(who)) {
|
|
304
474
|
return 'attestation-sent';
|
|
305
475
|
} else if (missedAttestors.has(who)) {
|
|
@@ -321,7 +491,7 @@ export class Sentinel extends EventEmitter {
|
|
|
321
491
|
v.toString(),
|
|
322
492
|
await this.store.getHistory(v)
|
|
323
493
|
]))) : await this.store.getHistories();
|
|
324
|
-
const slotNow = this.
|
|
494
|
+
const slotNow = await this.getCurrentSlot();
|
|
325
495
|
fromSlot ??= SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
|
|
326
496
|
toSlot ??= this.lastProcessedSlot ?? slotNow;
|
|
327
497
|
const stats = mapValues(histories, (history, address)=>this.computeStatsForValidator(address, history ?? [], fromSlot, toSlot));
|
|
@@ -337,7 +507,7 @@ export class Sentinel extends EventEmitter {
|
|
|
337
507
|
if (!history || history.length === 0) {
|
|
338
508
|
return undefined;
|
|
339
509
|
}
|
|
340
|
-
const slotNow = this.
|
|
510
|
+
const slotNow = await this.getCurrentSlot();
|
|
341
511
|
const effectiveFromSlot = fromSlot ?? SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
|
|
342
512
|
const effectiveToSlot = toSlot ?? this.lastProcessedSlot ?? slotNow;
|
|
343
513
|
const historyLength = BigInt(this.store.getHistoryLength());
|
|
@@ -347,7 +517,7 @@ export class Sentinel extends EventEmitter {
|
|
|
347
517
|
const validator = this.computeStatsForValidator(validatorAddress.toString(), history, effectiveFromSlot, effectiveToSlot);
|
|
348
518
|
return {
|
|
349
519
|
validator,
|
|
350
|
-
|
|
520
|
+
allTimeEpochPerformance: await this.store.getEpochPerformance(validatorAddress),
|
|
351
521
|
lastProcessedSlot: this.lastProcessedSlot,
|
|
352
522
|
initialSlot: this.initialSlot,
|
|
353
523
|
slotWindow: this.store.getHistoryLength()
|
|
@@ -356,15 +526,18 @@ export class Sentinel extends EventEmitter {
|
|
|
356
526
|
computeStatsForValidator(address, allHistory, fromSlot, toSlot) {
|
|
357
527
|
let history = fromSlot ? allHistory.filter((h)=>BigInt(h.slot) >= fromSlot) : allHistory;
|
|
358
528
|
history = toSlot ? history.filter((h)=>BigInt(h.slot) <= toSlot) : history;
|
|
359
|
-
const lastProposal = history.filter((h)=>h.status === '
|
|
529
|
+
const lastProposal = history.filter((h)=>h.status === 'checkpoint-valid' || h.status === 'checkpoint-mined').at(-1);
|
|
360
530
|
const lastAttestation = history.filter((h)=>h.status === 'attestation-sent').at(-1);
|
|
361
531
|
return {
|
|
362
532
|
address: EthAddress.fromString(address),
|
|
363
533
|
lastProposal: this.computeFromSlot(lastProposal?.slot),
|
|
364
534
|
lastAttestation: this.computeFromSlot(lastAttestation?.slot),
|
|
365
535
|
totalSlots: history.length,
|
|
366
|
-
missedProposals: this.computeMissed(history, '
|
|
367
|
-
'
|
|
536
|
+
missedProposals: this.computeMissed(history, 'proposer', [
|
|
537
|
+
'checkpoint-missed',
|
|
538
|
+
'blocks-missed',
|
|
539
|
+
'checkpoint-invalid',
|
|
540
|
+
'checkpoint-unvalidated'
|
|
368
541
|
]),
|
|
369
542
|
missedAttestations: this.computeMissed(history, 'attestation', [
|
|
370
543
|
'attestation-missed'
|
|
@@ -372,8 +545,8 @@ export class Sentinel extends EventEmitter {
|
|
|
372
545
|
history
|
|
373
546
|
};
|
|
374
547
|
}
|
|
375
|
-
computeMissed(history,
|
|
376
|
-
const relevantHistory = history.filter((h)=>!
|
|
548
|
+
computeMissed(history, computeOverCategory, filter) {
|
|
549
|
+
const relevantHistory = history.filter((h)=>!computeOverCategory || statusToCategory(h.status) === computeOverCategory);
|
|
377
550
|
const filteredHistory = relevantHistory.filter((h)=>filter.includes(h.status));
|
|
378
551
|
return {
|
|
379
552
|
currentStreak: countWhile([
|
package/dest/sentinel/store.d.ts
CHANGED
|
@@ -5,22 +5,22 @@ import type { ValidatorStatusHistory, ValidatorStatusInSlot, ValidatorsEpochPerf
|
|
|
5
5
|
export declare class SentinelStore {
|
|
6
6
|
private store;
|
|
7
7
|
private config;
|
|
8
|
-
static readonly SCHEMA_VERSION
|
|
8
|
+
static readonly SCHEMA_VERSION = 4;
|
|
9
9
|
private readonly historyMap;
|
|
10
|
-
private readonly
|
|
10
|
+
private readonly epochMap;
|
|
11
11
|
constructor(store: AztecAsyncKVStore, config: {
|
|
12
12
|
historyLength: number;
|
|
13
|
-
|
|
13
|
+
historicEpochPerformanceLength: number;
|
|
14
14
|
});
|
|
15
15
|
getHistoryLength(): number;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
getHistoricEpochPerformanceLength(): number;
|
|
17
|
+
updateEpochPerformance(epoch: EpochNumber, performance: ValidatorsEpochPerformance): Promise<void>;
|
|
18
|
+
getEpochPerformance(who: EthAddress): Promise<{
|
|
19
19
|
missed: number;
|
|
20
20
|
total: number;
|
|
21
21
|
epoch: EpochNumber;
|
|
22
22
|
}[]>;
|
|
23
|
-
private
|
|
23
|
+
private pushValidatorEpochPerformance;
|
|
24
24
|
updateValidators(slot: SlotNumber, statuses: Record<`0x${string}`, ValidatorStatusInSlot | undefined>): Promise<void>;
|
|
25
25
|
private pushValidatorStatusForSlot;
|
|
26
26
|
getHistories(): Promise<Record<`0x${string}`, ValidatorStatusHistory>>;
|
|
@@ -32,4 +32,4 @@ export declare class SentinelStore {
|
|
|
32
32
|
private statusToNumber;
|
|
33
33
|
private statusFromNumber;
|
|
34
34
|
}
|
|
35
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
35
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RvcmUuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9zZW50aW5lbC9zdG9yZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsV0FBVyxFQUFFLFVBQVUsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBQzFFLE9BQU8sRUFBRSxVQUFVLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUUzRCxPQUFPLEtBQUssRUFBRSxpQkFBaUIsRUFBaUIsTUFBTSxpQkFBaUIsQ0FBQztBQUN4RSxPQUFPLEtBQUssRUFDVixzQkFBc0IsRUFDdEIscUJBQXFCLEVBQ3JCLDBCQUEwQixFQUMzQixNQUFNLDBCQUEwQixDQUFDO0FBRWxDLHFCQUFhLGFBQWE7SUFXdEIsT0FBTyxDQUFDLEtBQUs7SUFDYixPQUFPLENBQUMsTUFBTTtJQVhoQixnQkFBdUIsY0FBYyxLQUFLO0lBRzFDLE9BQU8sQ0FBQyxRQUFRLENBQUMsVUFBVSxDQUF1QztJQUlsRSxPQUFPLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBdUM7SUFFaEUsWUFDVSxLQUFLLEVBQUUsaUJBQWlCLEVBQ3hCLE1BQU0sRUFBRTtRQUFFLGFBQWEsRUFBRSxNQUFNLENBQUM7UUFBQyw4QkFBOEIsRUFBRSxNQUFNLENBQUE7S0FBRSxFQUlsRjtJQUVNLGdCQUFnQixXQUV0QjtJQUVNLGlDQUFpQyxXQUV2QztJQUVZLHNCQUFzQixDQUFDLEtBQUssRUFBRSxXQUFXLEVBQUUsV0FBVyxFQUFFLDBCQUEwQixpQkFNOUY7SUFFWSxtQkFBbUIsQ0FBQyxHQUFHLEVBQUUsVUFBVSxHQUFHLE9BQU8sQ0FBQztRQUFFLE1BQU0sRUFBRSxNQUFNLENBQUM7UUFBQyxLQUFLLEVBQUUsTUFBTSxDQUFDO1FBQUMsS0FBSyxFQUFFLFdBQVcsQ0FBQTtLQUFFLEVBQUUsQ0FBQyxDQUdsSDtZQUVhLDZCQUE2QjtJQTZCOUIsZ0JBQWdCLENBQUMsSUFBSSxFQUFFLFVBQVUsRUFBRSxRQUFRLEVBQUUsTUFBTSxDQUFDLEtBQUssTUFBTSxFQUFFLEVBQUUscUJBQXFCLEdBQUcsU0FBUyxDQUFDLGlCQVFqSDtZQUVhLDBCQUEwQjtJQVEzQixZQUFZLElBQUksT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLE1BQU0sRUFBRSxFQUFFLHNCQUFzQixDQUFDLENBQUMsQ0FNbEY7SUFFWSxVQUFVLENBQUMsT0FBTyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsc0JBQXNCLEdBQUcsU0FBUyxDQUFDLENBR3hGO0lBRUQsT0FBTyxDQUFDLG9CQUFvQjtJQU01QixPQUFPLENBQUMsc0JBQXNCO0lBYTlCLE9BQU8sQ0FBQyxnQkFBZ0I7SUFNeEIsT0FBTyxDQUFDLGtCQUFrQjtJQVcxQixPQUFPLENBQUMsY0FBYztJQXlCdEIsT0FBTyxDQUFDLGdCQUFnQjtDQXNCekIifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/sentinel/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAE3D,OAAO,KAAK,EAAE,iBAAiB,EAAiB,MAAM,iBAAiB,CAAC;AACxE,OAAO,KAAK,EACV,sBAAsB,EACtB,qBAAqB,EACrB,0BAA0B,EAC3B,MAAM,0BAA0B,CAAC;AAElC,qBAAa,aAAa;IAWtB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IAXhB,gBAAuB,cAAc,
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/sentinel/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAE3D,OAAO,KAAK,EAAE,iBAAiB,EAAiB,MAAM,iBAAiB,CAAC;AACxE,OAAO,KAAK,EACV,sBAAsB,EACtB,qBAAqB,EACrB,0BAA0B,EAC3B,MAAM,0BAA0B,CAAC;AAElC,qBAAa,aAAa;IAWtB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IAXhB,gBAAuB,cAAc,KAAK;IAG1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuC;IAIlE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,YACU,KAAK,EAAE,iBAAiB,EACxB,MAAM,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,8BAA8B,EAAE,MAAM,CAAA;KAAE,EAIlF;IAEM,gBAAgB,WAEtB;IAEM,iCAAiC,WAEvC;IAEY,sBAAsB,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,0BAA0B,iBAM9F;IAEY,mBAAmB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,WAAW,CAAA;KAAE,EAAE,CAAC,CAGlH;YAEa,6BAA6B;IA6B9B,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,MAAM,EAAE,EAAE,qBAAqB,GAAG,SAAS,CAAC,iBAQjH;YAEa,0BAA0B;IAQ3B,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAMlF;IAEY,UAAU,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAGxF;IAED,OAAO,CAAC,oBAAoB;IAM5B,OAAO,CAAC,sBAAsB;IAa9B,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,kBAAkB;IAW1B,OAAO,CAAC,cAAc;IAyBtB,OAAO,CAAC,gBAAgB;CAsBzB"}
|