@lodestar/beacon-node 1.35.0-dev.47470aab65 → 1.35.0-dev.47c570ab76

package/src/chain/archiveStore/historicalState/metrics.ts

@@ -1,214 +0,0 @@
-import {
-  BeaconStateTransitionMetrics,
-  EpochTransitionStep,
-  ProposerRewardType,
-  StateCloneSource,
-  StateHashTreeRootSource,
-} from "@lodestar/state-transition";
-import {Gauge, Histogram} from "@lodestar/utils";
-import {RegistryMetricCreator} from "../../../metrics/index.js";
-import {QueueMetrics} from "../../../util/queue/options.js";
-import {RegenErrorType} from "./types.js";
-
-export type HistoricalStateTransitionMetrics = BeaconStateTransitionMetrics;
-
-export type HistoricalStateRegenMetrics = HistoricalStateTransitionMetrics & {
-  regenTime: Histogram;
-  loadStateTime: Histogram;
-  stateTransitionTime: Histogram;
-  stateTransitionBlocks: Histogram;
-  stateSerializationTime: Histogram;
-  regenRequestCount: Gauge;
-  regenSuccessCount: Gauge;
-  regenErrorCount: Gauge<{reason: RegenErrorType}>;
-};
-
-export function createHistoricalStateTransitionMetrics(
-  metricsRegister: RegistryMetricCreator
-): HistoricalStateTransitionMetrics {
-  return {
-    // state transition metrics
-    epochTransitionTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_stfn_epoch_transition_seconds",
-      help: "Time to process a single epoch transition in seconds",
-      // Epoch transitions are 100ms on very fast clients, and average 800ms on heavy networks
-      buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1, 1.25, 1.5, 3, 10],
-    }),
-    epochTransitionCommitTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_stfn_epoch_transition_commit_seconds",
-      help: "Time to call commit after process a single epoch transition in seconds",
-      buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
-    }),
-    epochTransitionStepTime: metricsRegister.histogram<{step: EpochTransitionStep}>({
-      name: "lodestar_historical_state_stfn_epoch_transition_step_seconds",
-      help: "Time to call each step of epoch transition in seconds",
-      labelNames: ["step"],
-      buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
-    }),
-    processBlockTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_stfn_process_block_seconds",
-      help: "Time to process a single block in seconds",
-      // TODO: Add metrics for each step
-      // Block processing can take 5-40ms, 100ms max
-      buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1],
-    }),
-    processBlockCommitTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_stfn_process_block_commit_seconds",
-      help: "Time to call commit after process a single block in seconds",
-      buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1],
-    }),
-    stateHashTreeRootTime: metricsRegister.histogram<{source: StateHashTreeRootSource}>({
-      name: "lodestar_historical_state_stfn_hash_tree_root_seconds",
-      help: "Time to compute the hash tree root of a post state in seconds",
-      buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5],
-      labelNames: ["source"],
-    }),
-    numEffectiveBalanceUpdates: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_num_effective_balance_updates_count",
-      help: "Count of effective balance updates in epoch transition",
-    }),
-    validatorsInActivationQueue: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_validators_in_activation_queue",
-      help: "Current number of validators in the activation queue",
-    }),
-    validatorsInExitQueue: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_validators_in_exit_queue",
-      help: "Current number of validators in the exit queue",
-    }),
-    preStateBalancesNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({
-      name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total",
-      help: "Total count state.balances nodesPopulated is false on stfn",
-      labelNames: ["source"],
-    }),
-    preStateBalancesNodesPopulatedHit: metricsRegister.gauge<{source: StateCloneSource}>({
-      name: "lodestar_historical_state_stfn_balances_nodes_populated_hit_total",
-      help: "Total count state.balances nodesPopulated is true on stfn",
-      labelNames: ["source"],
-    }),
-    preStateValidatorsNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({
-      name: "lodestar_historical_state_stfn_validators_nodes_populated_miss_total",
-      help: "Total count state.validators nodesPopulated is false on stfn",
-      labelNames: ["source"],
-    }),
-    preStateValidatorsNodesPopulatedHit: metricsRegister.gauge<{source: StateCloneSource}>({
-      name: "lodestar_historical_state_stfn_validators_nodes_populated_hit_total",
-      help: "Total count state.validators nodesPopulated is true on stfn",
-      labelNames: ["source"],
-    }),
-    preStateClonedCount: metricsRegister.histogram({
-      name: "lodestar_historical_state_stfn_state_cloned_count",
-      help: "Histogram of cloned count per state every time state.clone() is called",
-      buckets: [1, 2, 5, 10, 50, 250],
-    }),
-    postStateBalancesNodesPopulatedHit: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_hit_total",
-      help: "Total count state.validators nodesPopulated is true on stfn for post state",
-    }),
-    postStateBalancesNodesPopulatedMiss: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_miss_total",
-      help: "Total count state.validators nodesPopulated is false on stfn for post state",
-    }),
-    postStateValidatorsNodesPopulatedHit: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_hit_total",
-      help: "Total count state.validators nodesPopulated is true on stfn for post state",
-    }),
-    postStateValidatorsNodesPopulatedMiss: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_miss_total",
-      help: "Total count state.validators nodesPopulated is false on stfn for post state",
-    }),
-    newSeenAttestersPerBlock: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_new_seen_attesters_per_block_total",
-      help: "Count of new seen attesters in epoch transition",
-    }),
-    newSeenAttestersEffectiveBalancePerBlock: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_new_seen_attesters_effective_balance_per_block_total",
-      help: "Total effective balance increment of new seen attesters per block",
-    }),
-    attestationsPerBlock: metricsRegister.gauge({
-      name: "lodestar_historical_state_stfn_attestations_per_block_total",
-      help: "Count of attestations per block",
-    }),
-    proposerRewards: metricsRegister.gauge<{type: ProposerRewardType}>({
-      name: "lodestar_historical_state_stfn_proposer_rewards_total",
-      help: "Proposer reward by type per block",
-      labelNames: ["type"],
-    }),
-  };
-}
-
-export function createHistoricalStateRegenMetrics(metricsRegister: RegistryMetricCreator): HistoricalStateRegenMetrics {
-  return {
-    ...createHistoricalStateTransitionMetrics(metricsRegister),
-    // historical state regen metrics
-    regenTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_regen_time_seconds",
-      help: "Time to regenerate a historical state in seconds",
-      // Historical state regen can take up to 3h as of Aug 2024
-      // 5m, 10m, 30m, 1h, 3h
-      buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60],
-    }),
-    loadStateTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_load_nearest_state_time_seconds",
-      help: "Time to load a nearest historical state from the database in seconds",
-      // 30s, 1m, 2m, 4m
-      buckets: [30, 60, 120, 240],
-    }),
-    stateTransitionTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_state_transition_time_seconds",
-      help: "Time to run state transition to regen historical state in seconds",
-      // 5m, 10m, 30m, 1h, 3h
-      buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60],
-    }),
-    stateTransitionBlocks: metricsRegister.histogram({
-      name: "lodestar_historical_state_state_transition_blocks",
-      help: "Count of blocks processed during state transition to regen historical state",
-      // given archiveStateEpochFrequency=1024, it could process up to 32768 blocks
-      buckets: [10, 100, 1000, 10000, 30000],
-    }),
-    stateSerializationTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_serialization_time_seconds",
-      help: "Time to serialize a historical state in seconds",
-      buckets: [0.25, 0.5, 1, 2],
-    }),
-    regenRequestCount: metricsRegister.gauge({
-      name: "lodestar_historical_state_request_count",
-      help: "Count of total historical state requests",
-    }),
-    regenSuccessCount: metricsRegister.gauge({
-      name: "lodestar_historical_state_success_count",
-      help: "Count of successful historical state regen",
-    }),
-    regenErrorCount: metricsRegister.gauge<{reason: RegenErrorType}>({
-      name: "lodestar_historical_state_error_count",
-      help: "Count of failed historical state regen",
-      labelNames: ["reason"],
-    }),
-  };
-}
-
-export function createHistoricalStateQueueMetrics(metricsRegister: RegistryMetricCreator): QueueMetrics {
-  return {
-    length: metricsRegister.gauge({
-      name: "lodestar_historical_state_queue_length",
-      help: "Count of total regen queue length",
-    }),
-    droppedJobs: metricsRegister.gauge({
-      name: "lodestar_historical_state_queue_dropped_jobs_total",
-      help: "Count of total regen queue dropped jobs",
-    }),
-    jobTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_queue_job_time_seconds",
-      help: "Time to process regen queue job in seconds",
-      buckets: [0.01, 0.1, 1, 10, 100],
-    }),
-    jobWaitTime: metricsRegister.histogram({
-      name: "lodestar_historical_state_queue_job_wait_time_seconds",
-      help: "Time from job added to the regen queue to starting in seconds",
-      buckets: [0.01, 0.1, 1, 10, 100],
-    }),
-    concurrency: metricsRegister.gauge({
-      name: "lodestar_historical_state_queue_concurrency",
-      help: "Current concurrency of regen queue",
-    }),
-  };
-}

package/src/chain/archiveStore/historicalState/types.ts

@@ -1,41 +0,0 @@
-import {ModuleThread} from "@chainsafe/threads";
-import {BeaconConfig, SpecJson} from "@lodestar/config";
-import {LoggerNode, LoggerNodeOpts} from "@lodestar/logger/node";
-import {Metrics} from "../../../metrics/index.js";
-
-export type HistoricalStateRegenInitModules = {
-  opts: {
-    genesisTime: number;
-    dbLocation: string;
-  };
-  config: BeaconConfig;
-  logger: LoggerNode;
-  metrics: Metrics | null;
-  signal?: AbortSignal;
-};
-export type HistoricalStateRegenModules = HistoricalStateRegenInitModules & {
-  api: ModuleThread<HistoricalStateWorkerApi>;
-};
-
-export type HistoricalStateWorkerData = {
-  chainConfigJson: SpecJson;
-  genesisValidatorsRoot: Uint8Array;
-  genesisTime: number;
-  maxConcurrency: number;
-  maxLength: number;
-  dbLocation: string;
-  metricsEnabled: boolean;
-  loggerOpts: LoggerNodeOpts;
-};
-
-export type HistoricalStateWorkerApi = {
-  close(): Promise<void>;
-  scrapeMetrics(): Promise<string>;
-  getHistoricalState(slot: number): Promise<Uint8Array>;
-};
-
-export enum RegenErrorType {
-  loadState = "load_state",
-  invalidStateRoot = "invalid_state_root",
-  blockProcessing = "block_processing",
-}

package/src/chain/archiveStore/historicalState/worker.ts

@@ -1,77 +0,0 @@
-import worker from "node:worker_threads";
-import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
-import {Transfer, expose} from "@chainsafe/threads/worker";
-import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config";
-import {LevelDbController} from "@lodestar/db/controller/level";
-import {getNodeLogger} from "@lodestar/logger/node";
-import {BeaconDb} from "../../../db/index.js";
-import {RegistryMetricCreator, collectNodeJSMetrics} from "../../../metrics/index.js";
-import {JobFnQueue} from "../../../util/queue/fnQueue.js";
-import {QueueMetrics} from "../../../util/queue/options.js";
-import {getHistoricalState} from "./getHistoricalState.js";
-import {
-  HistoricalStateRegenMetrics,
-  createHistoricalStateQueueMetrics,
-  createHistoricalStateRegenMetrics,
-} from "./metrics.js";
-import {HistoricalStateWorkerApi, HistoricalStateWorkerData} from "./types.js";
-
-// most of this setup copied from networkCoreWorker.ts
-
-const workerData = worker.workerData as HistoricalStateWorkerData;
-
-const logger = getNodeLogger(workerData.loggerOpts);
-
-logger.info("Historical state worker started");
-
-const config = createBeaconConfig(chainConfigFromJson(workerData.chainConfigJson), workerData.genesisValidatorsRoot);
-
-const db = new BeaconDb(config, await LevelDbController.create({name: workerData.dbLocation}, {logger}));
-
-const abortController = new AbortController();
-
-// Set up metrics, nodejs, state transition, queue
-const metricsRegister = workerData.metricsEnabled ? new RegistryMetricCreator() : null;
-let historicalStateRegenMetrics: HistoricalStateRegenMetrics | undefined;
-let queueMetrics: QueueMetrics | undefined;
-
-if (metricsRegister) {
-  const closeMetrics = collectNodeJSMetrics(metricsRegister, "lodestar_historical_state_worker_");
-  abortController.signal.addEventListener("abort", closeMetrics, {once: true});
-
-  historicalStateRegenMetrics = createHistoricalStateRegenMetrics(metricsRegister);
-  queueMetrics = createHistoricalStateQueueMetrics(metricsRegister);
-}
-
-const queue = new JobFnQueue(
-  {
-    maxConcurrency: workerData.maxConcurrency,
-    maxLength: workerData.maxLength,
-    signal: abortController.signal,
-  },
-  queueMetrics
-);
-
-const pubkey2index = new PubkeyIndexMap();
-
-const api: HistoricalStateWorkerApi = {
-  async close() {
-    abortController.abort();
-  },
-  async scrapeMetrics() {
-    return metricsRegister?.metrics() ?? "";
-  },
-  async getHistoricalState(slot) {
-    historicalStateRegenMetrics?.regenRequestCount.inc();
-
-    const stateBytes = await queue.push<Uint8Array>(() =>
-      getHistoricalState(slot, config, db, pubkey2index, historicalStateRegenMetrics)
-    );
-    const result = Transfer(stateBytes, [stateBytes.buffer]) as unknown as Uint8Array;
-
-    historicalStateRegenMetrics?.regenSuccessCount.inc();
-    return result;
-  },
-};
-
-expose(api);

package/src/chain/archiveStore/index.ts

@@ -1,3 +0,0 @@
-export * from "./archiveStore.js";
-export * from "./constants.js";
-export * from "./interface.js";

package/src/chain/archiveStore/interface.ts

@@ -1,75 +0,0 @@
-import {CheckpointWithHex} from "@lodestar/fork-choice";
-import {RootHex} from "@lodestar/types";
-import {Metrics} from "../../metrics/metrics.js";
-
-export enum ArchiveMode {
-  Frequency = "frequency",
-  // New strategy to be implemented
-  // WIP: https://github.com/ChainSafe/lodestar/pull/7005
-  // Differential = "diff",
-}
-
-export interface StatesArchiveOpts {
-  /**
-   * Minimum number of epochs between archived states
-   */
-  archiveStateEpochFrequency: number;
-  /**
-   * Strategy to store archive states
-   */
-  archiveMode: ArchiveMode;
-}
-
-export type ArchiveStoreOpts = StatesArchiveOpts & {
-  disableArchiveOnCheckpoint?: boolean;
-  archiveDataEpochs?: number;
-  pruneHistory?: boolean;
-  serveHistoricalState?: boolean;
-};
-
-export type ProposalStats = {
-  total: number;
-  finalized: number;
-  orphaned: number;
-  missed: number;
-};
-
-export type FinalizedStats = {
-  allValidators: ProposalStats;
-  attachedValidators: ProposalStats;
-  finalizedCanonicalCheckpointsCount: number;
-  finalizedFoundCheckpointsInStateCache: number;
-  finalizedAttachedValidatorsCount: number;
-};
-
-export interface StateArchiveStrategy {
-  onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise<void>;
-  onFinalizedCheckpoint(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
-  maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
-  archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
-}
-
-export interface IArchiveStore {
-  /**
-   * Initialize archive store and load any worker required
-   */
-  init(): Promise<void>;
-  /**
-   * Cleanup and close any worker
-   */
-  close(): Promise<void>;
-  /**
-   * Scrape metrics from the archive store
-   */
-  scrapeMetrics(): Promise<string>;
-  /**
-   * Get historical state by slot
-   */
-  getHistoricalStateBySlot(
-    slot: number
-  ): Promise<{state: Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>;
-  /**
-   * Archive latest finalized state
-   */
-  persistToDisk(): Promise<void>;
-}

package/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts

@@ -1,138 +0,0 @@
-import {CheckpointWithHex} from "@lodestar/fork-choice";
-import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {Epoch, RootHex, Slot} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
-import {IBeaconDb} from "../../../db/index.js";
-import {Metrics} from "../../../metrics/metrics.js";
-import {AllocSource, BufferPool} from "../../../util/bufferPool.js";
-import {getStateSlotFromBytes} from "../../../util/multifork.js";
-import {IStateRegenerator} from "../../regen/interface.js";
-import {serializeState} from "../../serializeState.js";
-import {StateArchiveStrategy, StatesArchiveOpts} from "../interface.js";
-
-/**
- * Minimum number of epochs between single temp archived states
- * These states will be pruned once a new state is persisted
- */
-export const PERSIST_TEMP_STATE_EVERY_EPOCHS = 32;
-
-/**
- * Archives finalized states from active bucket to archive bucket.
- *
- * Only the new finalized state is stored to disk
- */
-export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
-  constructor(
-    private readonly regen: IStateRegenerator,
-    private readonly db: IBeaconDb,
-    private readonly logger: Logger,
-    private readonly opts: StatesArchiveOpts,
-    private readonly bufferPool?: BufferPool | null
-  ) {}
-
-  async onFinalizedCheckpoint(_finalized: CheckpointWithHex, _metrics?: Metrics | null): Promise<void> {}
-  async onCheckpoint(_stateRoot: RootHex, _metrics?: Metrics | null): Promise<void> {}
-
-  /**
-   * Persist states every some epochs to
-   * - Minimize disk space, storing the least states possible
-   * - Minimize the sync progress lost on unexpected crash, storing temp state every few epochs
-   *
-   * At epoch `e` there will be states peristed at intervals of `PERSIST_STATE_EVERY_EPOCHS` = 32
-   * and one at `PERSIST_TEMP_STATE_EVERY_EPOCHS` = 1024
-   * ```
-   *        |                |             |           .
-   * epoch - 1024*2    epoch - 1024    epoch - 32    epoch
-   * ```
-   */
-  async maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
-    const lastStoredSlot = await this.db.stateArchive.lastKey();
-    const lastStoredEpoch = computeEpochAtSlot(lastStoredSlot ?? 0);
-    const {archiveStateEpochFrequency} = this.opts;
-
-    if (finalized.epoch - lastStoredEpoch >= Math.min(PERSIST_TEMP_STATE_EVERY_EPOCHS, archiveStateEpochFrequency)) {
-      await this.archiveState(finalized, metrics);
-
-      // Only check the current and previous intervals
-      const minEpoch = Math.max(
-        0,
-        (Math.floor(finalized.epoch / archiveStateEpochFrequency) - 1) * archiveStateEpochFrequency
-      );
-
-      const storedStateSlots = await this.db.stateArchive.keys({
-        lt: computeStartSlotAtEpoch(finalized.epoch),
-        gte: computeStartSlotAtEpoch(minEpoch),
-      });
-
-      const statesSlotsToDelete = computeStateSlotsToDelete(storedStateSlots, archiveStateEpochFrequency);
-      if (statesSlotsToDelete.length > 0) {
-        await this.db.stateArchive.batchDelete(statesSlotsToDelete);
-      }
-
-      // More logs to investigate the rss spike issue https://github.com/ChainSafe/lodestar/issues/5591
-      this.logger.verbose("Archived state completed", {
-        finalizedEpoch: finalized.epoch,
-        minEpoch,
-        storedStateSlots: storedStateSlots.join(","),
-        statesSlotsToDelete: statesSlotsToDelete.join(","),
-      });
-    }
-  }
-
-  /**
-   * Archives finalized states from active bucket to archive bucket.
-   * Only the new finalized state is stored to disk
-   */
-  async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
-    // starting from Mar 2024, the finalized state could be from disk or in memory
-    const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized);
-    const {rootHex} = finalized;
-    if (!finalizedStateOrBytes) {
-      throw Error(`No state in cache for finalized checkpoint state epoch #${finalized.epoch} root ${rootHex}`);
-    }
-    if (finalizedStateOrBytes instanceof Uint8Array) {
-      const slot = getStateSlotFromBytes(finalizedStateOrBytes);
-      await this.db.stateArchive.putBinary(slot, finalizedStateOrBytes);
-      this.logger.verbose("Archived finalized state bytes", {epoch: finalized.epoch, slot, root: rootHex});
-    } else {
-      // serialize state using BufferPool if provided
-      const timer = metrics?.stateSerializeDuration.startTimer({source: AllocSource.ARCHIVE_STATE});
-      await serializeState(
-        finalizedStateOrBytes,
-        AllocSource.ARCHIVE_STATE,
-        (stateBytes) => {
-          timer?.();
-          return this.db.stateArchive.putBinary(finalizedStateOrBytes.slot, stateBytes);
-        },
-        this.bufferPool
-      );
-      // don't delete states before the finalized state, auto-prune will take care of it
-      this.logger.verbose("Archived finalized state", {
-        epoch: finalized.epoch,
-        slot: finalizedStateOrBytes.slot,
-        root: rootHex,
-      });
-    }
-  }
-}
-
-/**
- * Keeps first epoch per interval of persistEveryEpochs, deletes the rest
- */
-export function computeStateSlotsToDelete(storedStateSlots: Slot[], persistEveryEpochs: Epoch): Slot[] {
-  const persistEverySlots = persistEveryEpochs * SLOTS_PER_EPOCH;
-  const intervalsWithStates = new Set<number>();
-  const stateSlotsToDelete = new Set<number>();
-
-  for (const slot of storedStateSlots) {
-    const interval = Math.floor(slot / persistEverySlots);
-    if (intervalsWithStates.has(interval)) {
-      stateSlotsToDelete.add(slot);
-    } else {
-      intervalsWithStates.add(interval);
-    }
-  }
-
-  return Array.from(stateSlotsToDelete.values());
-}