@lodestar/beacon-node 1.31.0-dev.fa41ea44de → 1.31.0-peerDAS.0a673a00bb

package/lib/chain/archiveStore/historicalState/metrics.d.ts

@@ -0,0 +1,22 @@
+import { BeaconStateTransitionMetrics } from "@lodestar/state-transition";
+import { Gauge, Histogram } from "@lodestar/utils";
+import { RegistryMetricCreator } from "../../../metrics/index.js";
+import { QueueMetrics } from "../../../util/queue/options.js";
+import { RegenErrorType } from "./types.js";
+export type HistoricalStateTransitionMetrics = BeaconStateTransitionMetrics;
+export type HistoricalStateRegenMetrics = HistoricalStateTransitionMetrics & {
+    regenTime: Histogram;
+    loadStateTime: Histogram;
+    stateTransitionTime: Histogram;
+    stateTransitionBlocks: Histogram;
+    stateSerializationTime: Histogram;
+    regenRequestCount: Gauge;
+    regenSuccessCount: Gauge;
+    regenErrorCount: Gauge<{
+        reason: RegenErrorType;
+    }>;
+};
+export declare function createHistoricalStateTransitionMetrics(metricsRegister: RegistryMetricCreator): HistoricalStateTransitionMetrics;
+export declare function createHistoricalStateRegenMetrics(metricsRegister: RegistryMetricCreator): HistoricalStateRegenMetrics;
+export declare function createHistoricalStateQueueMetrics(metricsRegister: RegistryMetricCreator): QueueMetrics;
+//# sourceMappingURL=metrics.d.ts.map

package/lib/chain/archiveStore/historicalState/metrics.js

@@ -0,0 +1,173 @@
+export function createHistoricalStateTransitionMetrics(metricsRegister) {
+    return {
+        // state transition metrics
+        epochTransitionTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_stfn_epoch_transition_seconds",
+            help: "Time to process a single epoch transition in seconds",
+            // Epoch transitions are 100ms on very fast clients, and average 800ms on heavy networks
+            buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1, 1.25, 1.5, 3, 10],
+        }),
+        epochTransitionCommitTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_stfn_epoch_transition_commit_seconds",
+            help: "Time to call commit after process a single epoch transition in seconds",
+            buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
+        }),
+        epochTransitionStepTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_stfn_epoch_transition_step_seconds",
+            help: "Time to call each step of epoch transition in seconds",
+            labelNames: ["step"],
+            buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
+        }),
+        processBlockTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_stfn_process_block_seconds",
+            help: "Time to process a single block in seconds",
+            // TODO: Add metrics for each step
+            // Block processing can take 5-40ms, 100ms max
+            buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1],
+        }),
+        processBlockCommitTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_stfn_process_block_commit_seconds",
+            help: "Time to call commit after process a single block in seconds",
+            buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1],
+        }),
+        stateHashTreeRootTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_stfn_hash_tree_root_seconds",
+            help: "Time to compute the hash tree root of a post state in seconds",
+            buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5],
+            labelNames: ["source"],
+        }),
+        numEffectiveBalanceUpdates: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_num_effective_balance_updates_count",
+            help: "Count of effective balance updates in epoch transition",
+        }),
+        preStateBalancesNodesPopulatedMiss: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total",
+            help: "Total count state.balances nodesPopulated is false on stfn",
+            labelNames: ["source"],
+        }),
+        preStateBalancesNodesPopulatedHit: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_balances_nodes_populated_hit_total",
+            help: "Total count state.balances nodesPopulated is true on stfn",
+            labelNames: ["source"],
+        }),
+        preStateValidatorsNodesPopulatedMiss: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_validators_nodes_populated_miss_total",
+            help: "Total count state.validators nodesPopulated is false on stfn",
+            labelNames: ["source"],
+        }),
+        preStateValidatorsNodesPopulatedHit: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_validators_nodes_populated_hit_total",
+            help: "Total count state.validators nodesPopulated is true on stfn",
+            labelNames: ["source"],
+        }),
+        preStateClonedCount: metricsRegister.histogram({
+            name: "lodestar_historical_state_stfn_state_cloned_count",
+            help: "Histogram of cloned count per state every time state.clone() is called",
+            buckets: [1, 2, 5, 10, 50, 250],
+        }),
+        postStateBalancesNodesPopulatedHit: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_hit_total",
+            help: "Total count state.validators nodesPopulated is true on stfn for post state",
+        }),
+        postStateBalancesNodesPopulatedMiss: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_miss_total",
+            help: "Total count state.validators nodesPopulated is false on stfn for post state",
+        }),
+        postStateValidatorsNodesPopulatedHit: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_hit_total",
+            help: "Total count state.validators nodesPopulated is true on stfn for post state",
+        }),
+        postStateValidatorsNodesPopulatedMiss: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_miss_total",
+            help: "Total count state.validators nodesPopulated is false on stfn for post state",
+        }),
+        newSeenAttestersPerBlock: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_new_seen_attesters_per_block_total",
+            help: "Count of new seen attesters in epoch transition",
+        }),
+        newSeenAttestersEffectiveBalancePerBlock: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_new_seen_attesters_effective_balance_per_block_total",
+            help: "Total effective balance increment of new seen attesters per block",
+        }),
+        attestationsPerBlock: metricsRegister.gauge({
+            name: "lodestar_historical_state_stfn_attestations_per_block_total",
+            help: "Count of attestations per block",
+        }),
+    };
+}
+export function createHistoricalStateRegenMetrics(metricsRegister) {
+    return {
+        ...createHistoricalStateTransitionMetrics(metricsRegister),
+        // historical state regen metrics
+        regenTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_regen_time_seconds",
+            help: "Time to regenerate a historical state in seconds",
+            // Historical state regen can take up to 3h as of Aug 2024
+            // 5m, 10m, 30m, 1h, 3h
+            buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60],
+        }),
+        loadStateTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_load_nearest_state_time_seconds",
+            help: "Time to load a nearest historical state from the database in seconds",
+            // 30s, 1m, 2m, 4m
+            buckets: [30, 60, 120, 240],
+        }),
+        stateTransitionTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_state_transition_time_seconds",
+            help: "Time to run state transition to regen historical state in seconds",
+            // 5m, 10m, 30m, 1h, 3h
+            buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60],
+        }),
+        stateTransitionBlocks: metricsRegister.histogram({
+            name: "lodestar_historical_state_state_transition_blocks",
+            help: "Count of blocks processed during state transition to regen historical state",
+            // given archiveStateEpochFrequency=1024, it could process up to 32768 blocks
+            buckets: [10, 100, 1000, 10000, 30000],
+        }),
+        stateSerializationTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_serialization_time_seconds",
+            help: "Time to serialize a historical state in seconds",
+            buckets: [0.25, 0.5, 1, 2],
+        }),
+        regenRequestCount: metricsRegister.gauge({
+            name: "lodestar_historical_state_request_count",
+            help: "Count of total historical state requests",
+        }),
+        regenSuccessCount: metricsRegister.gauge({
+            name: "lodestar_historical_state_success_count",
+            help: "Count of successful historical state regen",
+        }),
+        regenErrorCount: metricsRegister.gauge({
+            name: "lodestar_historical_state_error_count",
+            help: "Count of failed historical state regen",
+            labelNames: ["reason"],
+        }),
+    };
+}
+export function createHistoricalStateQueueMetrics(metricsRegister) {
+    return {
+        length: metricsRegister.gauge({
+            name: "lodestar_historical_state_queue_length",
+            help: "Count of total regen queue length",
+        }),
+        droppedJobs: metricsRegister.gauge({
+            name: "lodestar_historical_state_queue_dropped_jobs_total",
+            help: "Count of total regen queue dropped jobs",
+        }),
+        jobTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_queue_job_time_seconds",
+            help: "Time to process regen queue job in seconds",
+            buckets: [0.01, 0.1, 1, 10, 100],
+        }),
+        jobWaitTime: metricsRegister.histogram({
+            name: "lodestar_historical_state_queue_job_wait_time_seconds",
+            help: "Time from job added to the regen queue to starting in seconds",
+            buckets: [0.01, 0.1, 1, 10, 100],
+        }),
+        concurrency: metricsRegister.gauge({
+            name: "lodestar_historical_state_queue_concurrency",
+            help: "Current concurrency of regen queue",
+        }),
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/lib/chain/archiveStore/historicalState/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../../src/chain/archiveStore/historicalState/metrics.ts"],"names":[],"mappings":"AAwBA,MAAM,UAAU,sCAAsC,CACpD,eAAsC;IAEtC,OAAO;QACL,2BAA2B;QAC3B,mBAAmB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC7C,IAAI,EAAE,yDAAyD;YAC/D,IAAI,EAAE,sDAAsD;YAC5D,wFAAwF;YACxF,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;SAChE,CAAC;QACF,yBAAyB,EAAE,eAAe,CAAC,SAAS,CAAC;YACnD,IAAI,EAAE,gEAAgE;YACtE,IAAI,EAAE,wEAAwE;YAC9E,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC9C,CAAC;QACF,uBAAuB,EAAE,eAAe,CAAC,SAAS,CAA8B;YAC9E,IAAI,EAAE,8DAA8D;YACpE,IAAI,EAAE,uDAAuD;YAC7D,UAAU,EAAE,CAAC,MAAM,CAAC;YACpB,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC9C,CAAC;QACF,gBAAgB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC1C,IAAI,EAAE,sDAAsD;YAC5D,IAAI,EAAE,2CAA2C;YACjD,kCAAkC;YAClC,8CAA8C;YAC9C,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;SAC3C,CAAC;QACF,sBAAsB,EAAE,eAAe,CAAC,SAAS,CAAC;YAChD,IAAI,EAAE,6DAA6D;YACnE,IAAI,EAAE,6DAA6D;YACnE,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;SAC3C,CAAC;QACF,qBAAqB,EAAE,eAAe,CAAC,SAAS,CAAoC;YAClF,IAAI,EAAE,uDAAuD;YAC7D,IAAI,EAAE,+DAA+D;YACrE,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;YACtC,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,0BAA0B,EAAE,eAAe,CAAC,KAAK,CAAC;YAChD,IAAI,EAAE,oEAAoE;YAC1E,IAAI,EAAE,wDAAwD;SAC/D,CAAC;QACF,kCAAkC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACpF,IAAI,EAAE,oEAAoE;YAC1E,IAAI,EAAE,4DAA4D;YAClE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,iCAAiC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACnF,IAAI,EAAE,mEAAmE;YACzE,IAAI,EAAE,2DAA2D;YACjE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,oCAAoC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACtF,IAAI,EAAE,sEAAsE;YAC5E,IAAI,EAAE,8DAA8D;YACpE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,mCAAmC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACrF,IAAI,EAAE,qEAAqE;YAC3E,IAAI,EAAE,6DAA6D;YACnE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,mBAAmB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC7C,IAAI,EAAE,mDAAmD;YACzD,IAAI,EAAE,wEAAwE;YAC9E,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC;SAChC,CAAC;QACF,kCAAkC,EAAE,eAAe,CAAC,KAAK,CAAC;YACxD,IAAI,EAAE,8EAA8E;YACpF,IAAI,EAAE,4EAA4E;SACnF,CAAC;QACF,mCAAmC,EAAE,eAAe,CAAC,KAAK,CAAC;YACzD,IAAI,EAAE,+EAA+E;YACrF,IAAI,EAAE,6EAA6E;SACpF,CAAC;QACF,oCAAoC,EAAE,eAAe,CAAC,KAAK,CAAC;YAC1D,IAAI,EAAE,gFAAgF;YACtF,IAAI,EAAE,4EAA4E;SACnF,CAAC;QACF,qCAAqC,EAAE,eAAe,CAAC,KAAK,CAAC;YAC3D,IAAI,EAAE,iFAAiF;YACvF,IAAI,EAAE,6EAA6E;SACpF,CAAC;QACF,wBAAwB,EAAE,eAAe,CAAC,KAAK,CAAC;YAC9C,IAAI,EAAE,mEAAmE;YACzE,IAAI,EAAE,iDAAiD;SACxD,CAAC;QACF,wCAAwC,EAAE,eAAe,CAAC,KAAK,CAAC;YAC9D,IAAI,EAAE,qFAAqF;YAC3F,IAAI,EAAE,mEAAmE;SAC1E,CAAC;QACF,oBAAoB,EAAE,eAAe,CAAC,KAAK,CAAC;YAC1C,IAAI,EAAE,6DAA6D;YACnE,IAAI,EAAE,iCAAiC;SACxC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,eAAsC;IACtF,OAAO;QACL,GAAG,sCAAsC,CAAC,eAAe,CAAC;QAC1D,iCAAiC;QACjC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC;YACnC,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,kDAAkD;YACxD,0DAA0D;YAC1D,uBAAuB;YACvB,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,CAAC;SACvD,CAAC;QACF,aAAa,EAAE,eAAe,CAAC,SAAS,CAAC;YACvC,IAAI,EAAE,2DAA2D;YACjE,IAAI,EAAE,sEAAsE;YAC5E,kBAAkB;YAClB,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;SAC5B,CAAC;QACF,mBAAmB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC7C,IAAI,EAAE,yDAAyD;YAC/D,IAAI,EAAE,mEAAmE;YACzE,uBAAuB;YACvB,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,CAAC;SACvD,CAAC;QACF,qBAAqB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC/C,IAAI,EAAE,mDAAmD;YACzD,IAAI,EAAE,6EAA6E;YACnF,6EAA6E;YAC7E,OAAO,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;SACvC,CAAC;QACF,sBAAsB,EAAE,eAAe,CAAC,SAAS,CAAC;YAChD,IAAI,EAAE,sDAAsD;YAC5D,IAAI,EAAE,iDAAiD;YACvD,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;SAC3B,CAAC;QACF,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC;YACvC,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,0CAA0C;SACjD,CAAC;QACF,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC;YACvC,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,4CAA4C;SACnD,CAAC;QACF,eAAe,EAAE,eAAe,CAAC,KAAK,CAA2B;YAC/D,IAAI,EAAE,uCAAuC;YAC7C,IAAI,EAAE,wCAAwC;YAC9C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,eAAsC;IACtF,OAAO;QACL,MAAM,EAAE,eAAe,CAAC,KAAK,CAAC;YAC5B,IAAI,EAAE,wCAAwC;YAC9C,IAAI,EAAE,mCAAmC;SAC1C,CAAC;QACF,WAAW,EAAE,eAAe,CAAC,KAAK,CAAC;YACjC,IAAI,EAAE,oDAAoD;YAC1D,IAAI,EAAE,yCAAyC;SAChD,CAAC;QACF,OAAO,EAAE,eAAe,CAAC,SAAS,CAAC;YACjC,IAAI,EAAE,kDAAkD;YACxD,IAAI,EAAE,4CAA4C;YAClD,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC;SACjC,CAAC;QACF,WAAW,EAAE,eAAe,CAAC,SAAS,CAAC;YACrC,IAAI,EAAE,uDAAuD;YAC7D,IAAI,EAAE,+DAA+D;YACrE,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC;SACjC,CAAC;QACF,WAAW,EAAE,eAAe,CAAC,KAAK,CAAC;YACjC,IAAI,EAAE,6CAA6C;YACnD,IAAI,EAAE,oCAAoC;SAC3C,CAAC;KACH,CAAC;AACJ,CAAC"}

package/lib/chain/archiveStore/historicalState/types.d.ts

@@ -1,8 +1,6 @@
 import { ModuleThread } from "@chainsafe/threads";
 import { BeaconConfig, SpecJson } from "@lodestar/config";
 import { LoggerNode, LoggerNodeOpts } from "@lodestar/logger/node";
-import { BeaconStateTransitionMetrics } from "@lodestar/state-transition";
-import { Gauge, Histogram } from "@lodestar/utils";
 import { Metrics } from "../../../metrics/index.js";
 export type HistoricalStateRegenInitModules = {
     opts: {
@@ -37,16 +35,4 @@ export declare enum RegenErrorType {
     invalidStateRoot = "invalid_state_root",
     blockProcessing = "block_processing"
 }
-export type HistoricalStateRegenMetrics = BeaconStateTransitionMetrics & {
-    regenTime: Histogram;
-    loadStateTime: Histogram;
-    stateTransitionTime: Histogram;
-    stateTransitionBlocks: Histogram;
-    stateSerializationTime: Histogram;
-    regenRequestCount: Gauge;
-    regenSuccessCount: Gauge;
-    regenErrorCount: Gauge<{
-        reason: RegenErrorType;
-    }>;
-};
 //# sourceMappingURL=types.d.ts.map

package/lib/chain/archiveStore/historicalState/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/chain/archiveStore/historicalState/types.ts"],"names":[],"mappings":"AAsCA,MAAM,CAAN,IAAY,cAIX;AAJD,WAAY,cAAc;IACxB,0CAAwB,CAAA;IACxB,yDAAuC,CAAA;IACvC,sDAAoC,CAAA;AACtC,CAAC,EAJW,cAAc,KAAd,cAAc,QAIzB"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/chain/archiveStore/historicalState/types.ts"],"names":[],"mappings":"AAoCA,MAAM,CAAN,IAAY,cAIX;AAJD,WAAY,cAAc;IACxB,0CAAwB,CAAA;IACxB,yDAAuC,CAAA;IACvC,sDAAoC,CAAA;AACtC,CAAC,EAJW,cAAc,KAAd,cAAc,QAIzB"}

package/lib/chain/archiveStore/historicalState/worker.js

@@ -8,6 +8,7 @@ import { BeaconDb } from "../../../db/index.js";
 import { RegistryMetricCreator, collectNodeJSMetrics } from "../../../metrics/index.js";
 import { JobFnQueue } from "../../../util/queue/fnQueue.js";
 import { getHistoricalState } from "./getHistoricalState.js";
+import { createHistoricalStateQueueMetrics, createHistoricalStateRegenMetrics, } from "./metrics.js";
 // most of this setup copied from networkCoreWorker.ts
 const workerData = worker.workerData;
 const logger = getNodeLogger(workerData.loggerOpts);
@@ -22,169 +23,8 @@ let queueMetrics;
 if (metricsRegister) {
     const closeMetrics = collectNodeJSMetrics(metricsRegister, "lodestar_historical_state_worker_");
     abortController.signal.addEventListener("abort", closeMetrics, { once: true });
-    historicalStateRegenMetrics = {
-        // state transition metrics
-        epochTransitionTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_stfn_epoch_transition_seconds",
-            help: "Time to process a single epoch transition in seconds",
-            // Epoch transitions are 100ms on very fast clients, and average 800ms on heavy networks
-            buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1, 1.25, 1.5, 3, 10],
-        }),
-        epochTransitionCommitTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_stfn_epoch_transition_commit_seconds",
-            help: "Time to call commit after process a single epoch transition in seconds",
-            buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
-        }),
-        epochTransitionStepTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_stfn_epoch_transition_step_seconds",
-            help: "Time to call each step of epoch transition in seconds",
-            labelNames: ["step"],
-            buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
-        }),
-        processBlockTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_stfn_process_block_seconds",
-            help: "Time to process a single block in seconds",
-            // TODO: Add metrics for each step
-            // Block processing can take 5-40ms, 100ms max
-            buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1],
-        }),
-        processBlockCommitTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_stfn_process_block_commit_seconds",
-            help: "Time to call commit after process a single block in seconds",
-            buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1],
-        }),
-        stateHashTreeRootTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_stfn_hash_tree_root_seconds",
-            help: "Time to compute the hash tree root of a post state in seconds",
-            buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5],
-            labelNames: ["source"],
-        }),
-        numEffectiveBalanceUpdates: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_num_effective_balance_updates_count",
-            help: "Count of effective balance updates in epoch transition",
-        }),
-        preStateBalancesNodesPopulatedMiss: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total",
-            help: "Total count state.balances nodesPopulated is false on stfn",
-            labelNames: ["source"],
-        }),
-        preStateBalancesNodesPopulatedHit: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_balances_nodes_populated_hit_total",
-            help: "Total count state.balances nodesPopulated is true on stfn",
-            labelNames: ["source"],
-        }),
-        preStateValidatorsNodesPopulatedMiss: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_validators_nodes_populated_miss_total",
-            help: "Total count state.validators nodesPopulated is false on stfn",
-            labelNames: ["source"],
-        }),
-        preStateValidatorsNodesPopulatedHit: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_validators_nodes_populated_hit_total",
-            help: "Total count state.validators nodesPopulated is true on stfn",
-            labelNames: ["source"],
-        }),
-        preStateClonedCount: metricsRegister.histogram({
-            name: "lodestar_historical_state_stfn_state_cloned_count",
-            help: "Histogram of cloned count per state every time state.clone() is called",
-            buckets: [1, 2, 5, 10, 50, 250],
-        }),
-        postStateBalancesNodesPopulatedHit: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_hit_total",
-            help: "Total count state.validators nodesPopulated is true on stfn for post state",
-        }),
-        postStateBalancesNodesPopulatedMiss: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_post_state_balances_nodes_populated_miss_total",
-            help: "Total count state.validators nodesPopulated is false on stfn for post state",
-        }),
-        postStateValidatorsNodesPopulatedHit: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_hit_total",
-            help: "Total count state.validators nodesPopulated is true on stfn for post state",
-        }),
-        postStateValidatorsNodesPopulatedMiss: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_post_state_validators_nodes_populated_miss_total",
-            help: "Total count state.validators nodesPopulated is false on stfn for post state",
-        }),
-        newSeenAttestersPerBlock: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_new_seen_attesters_per_block_total",
-            help: "Count of new seen attesters in epoch transition",
-        }),
-        newSeenAttestersEffectiveBalancePerBlock: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_new_seen_attesters_effective_balance_per_block_total",
-            help: "Total effective balance increment of new seen attesters per block",
-        }),
-        attestationsPerBlock: metricsRegister.gauge({
-            name: "lodestar_historical_state_stfn_attestations_per_block_total",
-            help: "Count of attestations per block",
-        }),
-        // historical state regen metrics
-        regenTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_regen_time_seconds",
-            help: "Time to regenerate a historical state in seconds",
-            // Historical state regen can take up to 3h as of Aug 2024
-            // 5m, 10m, 30m, 1h, 3h
-            buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60],
-        }),
-        loadStateTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_load_nearest_state_time_seconds",
-            help: "Time to load a nearest historical state from the database in seconds",
-            // 30s, 1m, 2m, 4m
-            buckets: [30, 60, 120, 240],
-        }),
-        stateTransitionTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_state_transition_time_seconds",
-            help: "Time to run state transition to regen historical state in seconds",
-            // 5m, 10m, 30m, 1h, 3h
-            buckets: [5 * 60, 10 * 60, 30 * 60, 60 * 60, 180 * 60],
-        }),
-        stateTransitionBlocks: metricsRegister.histogram({
-            name: "lodestar_historical_state_state_transition_blocks",
-            help: "Count of blocks processed during state transition to regen historical state",
-            // given archiveStateEpochFrequency=1024, it could process up to 32768 blocks
-            buckets: [10, 100, 1000, 10000, 30000],
-        }),
-        stateSerializationTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_serialization_time_seconds",
-            help: "Time to serialize a historical state in seconds",
-            buckets: [0.25, 0.5, 1, 2],
-        }),
-        regenRequestCount: metricsRegister.gauge({
-            name: "lodestar_historical_state_request_count",
-            help: "Count of total historical state requests",
-        }),
-        regenSuccessCount: metricsRegister.gauge({
-            name: "lodestar_historical_state_success_count",
-            help: "Count of successful historical state regen",
-        }),
-        regenErrorCount: metricsRegister.gauge({
-            name: "lodestar_historical_state_error_count",
-            help: "Count of failed historical state regen",
-            labelNames: ["reason"],
-        }),
-    };
-    queueMetrics = {
-        length: metricsRegister.gauge({
-            name: "lodestar_historical_state_queue_length",
-            help: "Count of total regen queue length",
-        }),
-        droppedJobs: metricsRegister.gauge({
-            name: "lodestar_historical_state_queue_dropped_jobs_total",
-            help: "Count of total regen queue dropped jobs",
-        }),
-        jobTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_queue_job_time_seconds",
-            help: "Time to process regen queue job in seconds",
-            buckets: [0.01, 0.1, 1, 10, 100],
-        }),
-        jobWaitTime: metricsRegister.histogram({
-            name: "lodestar_historical_state_queue_job_wait_time_seconds",
-            help: "Time from job added to the regen queue to starting in seconds",
-            buckets: [0.01, 0.1, 1, 10, 100],
-        }),
-        concurrency: metricsRegister.gauge({
-            name: "lodestar_historical_state_queue_concurrency",
-            help: "Current concurrency of regen queue",
-        }),
-    };
+    historicalStateRegenMetrics = createHistoricalStateRegenMetrics(metricsRegister);
+    queueMetrics = createHistoricalStateQueueMetrics(metricsRegister);
 }
 const queue = new JobFnQueue({
     maxConcurrency: workerData.maxConcurrency,

package/lib/chain/archiveStore/historicalState/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../../src/chain/archiveStore/historicalState/worker.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,qBAAqB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAC,QAAQ,EAAE,MAAM,EAAC,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAC,mBAAmB,EAAE,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAC,iBAAiB,EAAC,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AAEpD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAC,qBAAqB,EAAE,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACtF,OAAO,EAAC,UAAU,EAAC,MAAM,gCAAgC,CAAC;AAE1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,yBAAyB,CAAC;AAQ3D,sDAAsD;AAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAuC,CAAC;AAElE,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AAEpD,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;AAE/C,MAAM,MAAM,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,UAAU,CAAC,qBAAqB,CAAC,CAAC;AAErH,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,iBAAiB,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,UAAU,CAAC,UAAU,EAAC,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC,CAAC;AAEzG,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;AAE9C,kDAAkD;AAClD,MAAM,eAAe,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACvF,IAAI,2BAAoE,CAAC;AACzE,IAAI,YAAsC,CAAC;AAC3C,IAAI,eAAe,EAAE,CAAC;IACpB,MAAM,YAAY,GAAG,oBAAoB,CAAC,eAAe,EAAE,mCAAmC,CAAC,CAAC;IAChG,eAAe,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;IAE7E,2BAA2B,GAAG;QAC5B,2BAA2B;QAC3B,mBAAmB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC7C,IAAI,EAAE,yDAAyD;YAC/D,IAAI,EAAE,sDAAsD;YAC5D,wFAAwF;YACxF,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;SAChE,CAAC;QACF,yBAAyB,EAAE,eAAe,CAAC,SAAS,CAAC;YACnD,IAAI,EAAE,gEAAgE;YACtE,IAAI,EAAE,wEAAwE;YAC9E,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC9C,CAAC;QACF,uBAAuB,EAAE,eAAe,CAAC,SAAS,CAA8B;YAC9E,IAAI,EAAE,8DAA8D;YACpE,IAAI,EAAE,uDAAuD;YAC7D,UAAU,EAAE,CAAC,MAAM,CAAC;YACpB,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC9C,CAAC;QACF,gBAAgB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC1C,IAAI,EAAE,sDAAsD;YAC5D,IAAI,EAAE,2CAA2C;YACjD,kCAAkC;YAClC,8CAA8C;YAC9C,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;SAC3C,CAAC;QACF,sBAAsB,EAAE,eAAe,CAAC,SAAS,CAAC;YAChD,IAAI,EAAE,6DAA6D;YACnE,IAAI,EAAE,6DAA6D;YACnE,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;SAC3C,CAAC;QACF,qBAAqB,EAAE,eAAe,CAAC,SAAS,CAAoC;YAClF,IAAI,EAAE,uDAAuD;YAC7D,IAAI,EAAE,+DAA+D;YACrE,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;YACtC,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,0BAA0B,EAAE,eAAe,CAAC,KAAK,CAAC;YAChD,IAAI,EAAE,oEAAoE;YAC1E,IAAI,EAAE,wDAAwD;SAC/D,CAAC;QACF,kCAAkC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACpF,IAAI,EAAE,oEAAoE;YAC1E,IAAI,EAAE,4DAA4D;YAClE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,iCAAiC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACnF,IAAI,EAAE,mEAAmE;YACzE,IAAI,EAAE,2DAA2D;YACjE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,oCAAoC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACtF,IAAI,EAAE,sEAAsE;YAC5E,IAAI,EAAE,8DAA8D;YACpE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,mCAAmC,EAAE,eAAe,CAAC,KAAK,CAA6B;YACrF,IAAI,EAAE,qEAAqE;YAC3E,IAAI,EAAE,6DAA6D;YACnE,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;QACF,mBAAmB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC7C,IAAI,EAAE,mDAAmD;YACzD,IAAI,EAAE,wEAAwE;YAC9E,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC;SAChC,CAAC;QACF,kCAAkC,EAAE,eAAe,CAAC,KAAK,CAAC;YACxD,IAAI,EAAE,8EAA8E;YACpF,IAAI,EAAE,4EAA4E;SACnF,CAAC;QACF,mCAAmC,EAAE,eAAe,CAAC,KAAK,CAAC;YACzD,IAAI,EAAE,+EAA+E;YACrF,IAAI,EAAE,6EAA6E;SACpF,CAAC;QACF,oCAAoC,EAAE,eAAe,CAAC,KAAK,CAAC;YAC1D,IAAI,EAAE,gFAAgF;YACtF,IAAI,EAAE,4EAA4E;SACnF,CAAC;QACF,qCAAqC,EAAE,eAAe,CAAC,KAAK,CAAC;YAC3D,IAAI,EAAE,iFAAiF;YACvF,IAAI,EAAE,6EAA6E;SACpF,CAAC;QACF,wBAAwB,EAAE,eAAe,CAAC,KAAK,CAAC;YAC9C,IAAI,EAAE,mEAAmE;YACzE,IAAI,EAAE,iDAAiD;SACxD,CAAC;QACF,wCAAwC,EAAE,eAAe,CAAC,KAAK,CAAC;YAC9D,IAAI,EAAE,qFAAqF;YAC3F,IAAI,EAAE,mEAAmE;SAC1E,CAAC;QACF,oBAAoB,EAAE,eAAe,CAAC,KAAK,CAAC;YAC1C,IAAI,EAAE,6DAA6D;YACnE,IAAI,EAAE,iCAAiC;SACxC,CAAC;QAEF,iCAAiC;QACjC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC;YACnC,IAAI,EAAE,8CAA8C;YACpD,IAAI,EAAE,kDAAkD;YACxD,0DAA0D;YAC1D,uBAAuB;YACvB,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,CAAC;SACvD,CAAC;QACF,aAAa,EAAE,eAAe,CAAC,SAAS,CAAC;YACvC,IAAI,EAAE,2DAA2D;YACjE,IAAI,EAAE,sEAAsE;YAC5E,kBAAkB;YAClB,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;SAC5B,CAAC;QACF,mBAAmB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC7C,IAAI,EAAE,yDAAyD;YAC/D,IAAI,EAAE,mEAAmE;YACzE,uBAAuB;YACvB,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,CAAC;SACvD,CAAC;QACF,qBAAqB,EAAE,eAAe,CAAC,SAAS,CAAC;YAC/C,IAAI,EAAE,mDAAmD;YACzD,IAAI,EAAE,6EAA6E;YACnF,6EAA6E;YAC7E,OAAO,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;SACvC,CAAC;QACF,sBAAsB,EAAE,eAAe,CAAC,SAAS,CAAC;YAChD,IAAI,EAAE,sDAAsD;YAC5D,IAAI,EAAE,iDAAiD;YACvD,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;SAC3B,CAAC;QACF,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC;YACvC,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,0CAA0C;SACjD,CAAC;QACF,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC;YACvC,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,4CAA4C;SACnD,CAAC;QACF,eAAe,EAAE,eAAe,CAAC,KAAK,CAA2B;YAC/D,IAAI,EAAE,uCAAuC;YAC7C,IAAI,EAAE,wCAAwC;YAC9C,UAAU,EAAE,CAAC,QAAQ,CAAC;SACvB,CAAC;KACH,CAAC;IAEF,YAAY,GAAG;QACb,MAAM,EAAE,eAAe,CAAC,KAAK,CAAC;YAC5B,IAAI,EAAE,wCAAwC;YAC9C,IAAI,EAAE,mCAAmC;SAC1C,CAAC;QACF,WAAW,EAAE,eAAe,CAAC,KAAK,CAAC;YACjC,IAAI,EAAE,oDAAoD;YAC1D,IAAI,EAAE,yCAAyC;SAChD,CAAC;QACF,OAAO,EAAE,eAAe,CAAC,SAAS,CAAC;YACjC,IAAI,EAAE,kDAAkD;YACxD,IAAI,EAAE,4CAA4C;YAClD,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC;SACjC,CAAC;QACF,WAAW,EAAE,eAAe,CAAC,SAAS,CAAC;YACrC,IAAI,EAAE,uDAAuD;YAC7D,IAAI,EAAE,+DAA+D;YACrE,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC;SACjC,CAAC;QACF,WAAW,EAAE,eAAe,CAAC,KAAK,CAAC;YACjC,IAAI,EAAE,6CAA6C;YACnD,IAAI,EAAE,oCAAoC;SAC3C,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,KAAK,GAAG,IAAI,UAAU,CAC1B;IACE,cAAc,EAAE,UAAU,CAAC,cAAc;IACzC,SAAS,EAAE,UAAU,CAAC,SAAS;IAC/B,MAAM,EAAE,eAAe,CAAC,MAAM;CAC/B,EACD,YAAY,CACb,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,cAAc,EAAE,CAAC;AAE1C,MAAM,GAAG,GAA6B;IACpC,KAAK,CAAC,KAAK;QACT,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IACD,KAAK,CAAC,aAAa;QACjB,OAAO,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;IACD,KAAK,CAAC,kBAAkB,CAAC,IAAI;QAC3B,2BAA2B,EAAE,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAErD,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,IAAI,CAAa,GAAG,EAAE,CACnD,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,EAAE,2BAA2B,CAAC,CAChF,CAAC;QACF,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAA0B,CAAC;QAElF,2BAA2B,EAAE,iBAAiB,CAAC,GAAG,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,GAAG,CAAC,CAAC"}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../../src/chain/archiveStore/historicalState/worker.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,qBAAqB,CAAC;AACzC,OAAO,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAC,QAAQ,EAAE,MAAM,EAAC,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAC,mBAAmB,EAAE,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAC,iBAAiB,EAAC,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAC,qBAAqB,EAAE,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACtF,OAAO,EAAC,UAAU,EAAC,MAAM,gCAAgC,CAAC;AAE1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAEL,iCAAiC,EACjC,iCAAiC,GAClC,MAAM,cAAc,CAAC;AAGtB,sDAAsD;AAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAuC,CAAC;AAElE,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AAEpD,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;AAE/C,MAAM,MAAM,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,UAAU,CAAC,qBAAqB,CAAC,CAAC;AAErH,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,iBAAiB,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,UAAU,CAAC,UAAU,EAAC,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC,CAAC;AAEzG,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;AAE9C,kDAAkD;AAClD,MAAM,eAAe,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACvF,IAAI,2BAAoE,CAAC;AACzE,IAAI,YAAsC,CAAC;AAE3C,IAAI,eAAe,EAAE,CAAC;IACpB,MAAM,YAAY,GAAG,oBAAoB,CAAC,eAAe,EAAE,mCAAmC,CAAC,CAAC;IAChG,eAAe,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;IAE7E,2BAA2B,GAAG,iCAAiC,CAAC,eAAe,CAAC,CAAC;IACjF,YAAY,GAAG,iCAAiC,CAAC,eAAe,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,KAAK,GAAG,IAAI,UAAU,CAC1B;IACE,cAAc,EAAE,UAAU,CAAC,cAAc;IACzC,SAAS,EAAE,UAAU,CAAC,SAAS;IAC/B,MAAM,EAAE,eAAe,CAAC,MAAM;CAC/B,EACD,YAAY,CACb,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,cAAc,EAAE,CAAC;AAE1C,MAAM,GAAG,GAA6B;IACpC,KAAK,CAAC,KAAK;QACT,eAAe,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IACD,KAAK,CAAC,aAAa;QACjB,OAAO,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;IACD,KAAK,CAAC,kBAAkB,CAAC,IAAI;QAC3B,2BAA2B,EAAE,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAErD,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,IAAI,CAAa,GAAG,EAAE,CACnD,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,EAAE,2BAA2B,CAAC,CAChF,CAAC;QACF,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAA0B,CAAC;QAElF,2BAA2B,EAAE,iBAAiB,CAAC,GAAG,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,GAAG,CAAC,CAAC"}

package/lib/chain/archiveStore/interface.d.ts

@@ -16,7 +16,7 @@ export interface StatesArchiveOpts {
 }
 export type ArchiveStoreOpts = StatesArchiveOpts & {
     disableArchiveOnCheckpoint?: boolean;
-    archiveBlobEpochs?: number;
+    archiveDataEpochs?: number;
     pruneHistory?: boolean;
     serveHistoricalState?: boolean;
 };

package/lib/chain/archiveStore/utils/archiveBlocks.d.ts

@@ -17,7 +17,7 @@ type CheckpointHex = {
  * Note that the finalized block still stay in forkchoice to check finalize checkpoint of next onBlock calls,
  * the next run should not reprocess finalzied block of this run.
  */
-export declare function archiveBlocks(config: ChainForkConfig, db: IBeaconDb, forkChoice: IForkChoice, lightclientServer: LightClientServer | undefined, logger: Logger, finalizedCheckpoint: CheckpointHex, currentEpoch: Epoch, archiveBlobEpochs?: number): Promise<void>;
+export declare function archiveBlocks(config: ChainForkConfig, db: IBeaconDb, forkChoice: IForkChoice, lightclientServer: LightClientServer | undefined, logger: Logger, finalizedCheckpoint: CheckpointHex, currentEpoch: Epoch, archiveDataEpochs?: number): Promise<void>;
 /**
  * ```
  * class SignedBeaconBlock(Container):

package/lib/chain/archiveStore/utils/archiveBlocks.js

@@ -1,6 +1,6 @@
 import { ForkSeq, SLOTS_PER_EPOCH } from "@lodestar/params";
 import { computeEpochAtSlot, computeStartSlotAtEpoch } from "@lodestar/state-transition";
-import { fromHex, toRootHex } from "@lodestar/utils";
+import { fromHex, toHex, toRootHex } from "@lodestar/utils";
 // Process in chunks to avoid OOM
 // this number of blocks per chunk is tested in e2e test blockArchive.test.ts
 // TODO: Review after merge since the size of blocks will increase significantly
@@ -15,12 +15,13 @@ const BLOB_SIDECAR_BATCH_SIZE = 32;
  * Note that the finalized block still stay in forkchoice to check finalize checkpoint of next onBlock calls,
  * the next run should not reprocess finalzied block of this run.
  */
-export async function archiveBlocks(config, db, forkChoice, lightclientServer, logger, finalizedCheckpoint, currentEpoch, archiveBlobEpochs) {
+export async function archiveBlocks(config, db, forkChoice, lightclientServer, logger, finalizedCheckpoint, currentEpoch, archiveDataEpochs) {
     // Use fork choice to determine the blocks to archive and delete
     // getAllAncestorBlocks response includes the finalized block, so it's also moved to the cold db
     const { ancestors: finalizedCanonicalBlocks, nonAncestors: finalizedNonCanonicalBlocks } = forkChoice.getAllAncestorAndNonAncestorBlocks(finalizedCheckpoint.rootHex);
     // NOTE: The finalized block will be exactly the first block of `epoch` or previous
     const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH;
+    const finalizedPostFulu = finalizedCheckpoint.epoch >= config.FULU_FORK_EPOCH;
     const finalizedCanonicalBlockRoots = finalizedCanonicalBlocks.map((block) => ({
         slot: block.slot,
         root: fromHex(block.blockRoot),
@@ -33,8 +34,12 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
             size: finalizedCanonicalBlockRoots.length,
         });
         if (finalizedPostDeneb) {
-            const migrate = await migrateBlobSidecarsFromHotToColdDb(config, db, finalizedCanonicalBlockRoots, currentEpoch);
-            logger.verbose(migrate ? "Migrated blobSidecars from hot DB to cold DB" : "Skip blobSidecars migration");
+            const migratedEntries = await migrateBlobSidecarsFromHotToColdDb(config, db, finalizedCanonicalBlockRoots, currentEpoch);
+            logger.verbose("Migrated blobSidecars from hot DB to cold DB", { migratedEntries });
+        }
+        if (finalizedPostFulu) {
+            const migratedEntries = await migrateDataColumnSidecarsFromHotToColdDb(config, db, finalizedCanonicalBlockRoots, currentEpoch);
+            logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", { migratedEntries });
         }
     }
     // deleteNonCanonicalBlocks
@@ -47,15 +52,19 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
         });
         if (finalizedPostDeneb) {
             await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
-            logger.verbose("Deleted non canonical blobsSider from hot DB");
+            logger.verbose("Deleted non canonical blobSidecars from hot DB");
+        }
+        if (finalizedPostFulu) {
+            await db.dataColumnSidecars.batchDelete(nonCanonicalBlockRoots);
+            logger.verbose("Deleted non canonical dataColumnSidecars from hot DB");
         }
     }
     // Delete expired blobs
-    // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveBlobEpochs)]
-    // if archiveBlobEpochs set to Infinity do not prune`
+    // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveDataEpochs)]`
+    // if archiveDataEpochs set to Infinity do not prune`
     if (finalizedPostDeneb) {
-        if (archiveBlobEpochs !== Infinity) {
-            const blobsArchiveWindow = Math.max(config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveBlobEpochs ?? 0);
+        if (archiveDataEpochs !== Infinity) {
+            const blobsArchiveWindow = Math.max(config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveDataEpochs ?? 0);
             const blobSidecarsMinEpoch = currentEpoch - blobsArchiveWindow;
             if (blobSidecarsMinEpoch >= config.DENEB_FORK_EPOCH) {
                 const slotsToDelete = await db.blobSidecarsArchive.keys({ lt: computeStartSlotAtEpoch(blobSidecarsMinEpoch) });
@@ -69,7 +78,30 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
             }
         }
         else {
-            logger.verbose("blobSidecars pruning skipped: archiveBlobEpochs set to Infinity");
+            logger.verbose("blobSidecars pruning skipped: archiveDataEpochs set to Infinity");
+        }
+    }
+    // Delete expired data column sidecars
+    // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, archiveDataEpochs)]`
+    if (finalizedPostFulu) {
+        if (archiveDataEpochs !== Infinity) {
+            const dataColumnSidecarsArchiveWindow = Math.max(config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, archiveDataEpochs ?? 0);
+            const dataColumnSidecarsMinEpoch = currentEpoch - dataColumnSidecarsArchiveWindow;
+            if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
+                const slotsToDelete = await db.dataColumnSidecarsArchive.keys({
+                    lt: computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch),
+                });
+                if (slotsToDelete.length > 0) {
+                    await db.dataColumnSidecarsArchive.batchDelete(slotsToDelete);
+                    logger.verbose(`dataColumnSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete.at(-1)}`);
+                }
+                else {
+                    logger.verbose(`dataColumnSidecars prune: no entries before epoch ${dataColumnSidecarsMinEpoch}`);
+                }
+            }
+        }
+        else {
+            logger.verbose("dataColumnSidecars pruning skipped: archiveDataEpochs set to Infinity");
         }
     }
     // Prunning potential checkpoint data
@@ -123,19 +155,22 @@ async function migrateBlocksFromHotToColdDb(db, blocks) {
  * @returns true if we do that, false if block is out of range data.
  */
 async function migrateBlobSidecarsFromHotToColdDb(config, db, blocks, currentEpoch) {
-    let result = false;
+    let migratedWrappedBlobSidecars = 0;
     for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
         const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
         const canonicalBlocks = blocks.slice(i, toIdx);
         // processCanonicalBlocks
         if (canonicalBlocks.length === 0)
-            return false;
+            break;
         // load Buffer instead of ssz deserialized to improve performance
         const canonicalBlobSidecarsEntries = await Promise.all(canonicalBlocks
             .filter((block) => {
             const blockSlot = block.slot;
             const blockEpoch = computeEpochAtSlot(blockSlot);
-            return (config.getForkSeq(blockSlot) >= ForkSeq.deneb &&
+            const forkSeq = config.getForkSeq(blockSlot);
+            return (forkSeq >= ForkSeq.deneb &&
+                forkSeq < ForkSeq.fulu &&
+                // if block is out of ${config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS}, skip this step
                 blockEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS);
         })
             .map(async (block) => {
@@ -145,17 +180,47 @@ async function migrateBlobSidecarsFromHotToColdDb(config, db, blocks, currentEpo
             }
             return { key: block.slot, value: bytes };
         }));
-        const migrate = canonicalBlobSidecarsEntries.length > 0;
-        if (migrate) {
-            // put to blockArchive db and delete block db
-            await Promise.all([
-                db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries),
-                db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)),
-            ]);
-        }
-        result = result || migrate;
+        // put to blockArchive db and delete block db
+        await Promise.all([
+            db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries),
+            db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)),
+        ]);
+        migratedWrappedBlobSidecars += canonicalBlobSidecarsEntries.length;
+    }
+    return migratedWrappedBlobSidecars;
+}
+async function migrateDataColumnSidecarsFromHotToColdDb(config, db, blocks, currentEpoch) {
+    let migratedWrappedDataColumns = 0;
+    for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
+        const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
+        const canonicalBlocks = blocks.slice(i, toIdx);
+        // processCanonicalBlocks
+        if (canonicalBlocks.length === 0)
+            break;
+        // load Buffer instead of ssz deserialized to improve performance
+        const canonicalDataColumnSidecarsEntries = await Promise.all(canonicalBlocks
+            .filter((block) => {
+            const blockSlot = block.slot;
+            const blockEpoch = computeEpochAtSlot(blockSlot);
+            return (config.getForkSeq(blockSlot) >= ForkSeq.fulu &&
+                // if block is out of ${config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS}, skip this step
+                blockEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS);
+        })
+            .map(async (block) => {
+            const bytes = await db.dataColumnSidecars.getBinary(block.root);
+            if (!bytes) {
+                throw Error(`No dataColumnSidecars found for slot ${block.slot} root ${toHex(block.root)}`);
+            }
+            return { key: block.slot, value: bytes };
+        }));
+        // put to blockArchive db and delete block db
+        await Promise.all([
+            db.dataColumnSidecarsArchive.batchPutBinary(canonicalDataColumnSidecarsEntries),
+            db.dataColumnSidecars.batchDelete(canonicalBlocks.map((block) => block.root)),
+        ]);
+        migratedWrappedDataColumns += canonicalDataColumnSidecarsEntries.length;
     }
-    return result;
+    return migratedWrappedDataColumns;
 }
 /**
  * ```