@lodestar/beacon-node 1.35.0-dev.f80d2d52da → 1.35.0-dev.fd1dac853d

package/src/api/impl/validator/utils.ts

@@ -0,0 +1,85 @@
+import {routes} from "@lodestar/api";
+import {ATTESTATION_SUBNET_COUNT} from "@lodestar/params";
+import {BeaconStateAllForks, computeSlotsSinceEpochStart} from "@lodestar/state-transition";
+import {BLSPubkey, CommitteeIndex, ProducedBlockSource, Slot, SubnetID, ValidatorIndex} from "@lodestar/types";
+import {MAX_BUILDER_BOOST_FACTOR} from "@lodestar/validator";
+import {BlockSelectionResult, BuilderBlockSelectionReason, EngineBlockSelectionReason} from "./index.js";
+
+export function computeSubnetForCommitteesAtSlot(
+  slot: Slot,
+  committeesAtSlot: number,
+  committeeIndex: CommitteeIndex
+): SubnetID {
+  const slotsSinceEpochStart = computeSlotsSinceEpochStart(slot);
+  const committeesSinceEpochStart = committeesAtSlot * slotsSinceEpochStart;
+  return (committeesSinceEpochStart + committeeIndex) % ATTESTATION_SUBNET_COUNT;
+}
+
+/**
+ * Precompute all pubkeys for given `validatorIndices`. Ensures that all `validatorIndices` are known
+ * before doing other expensive logic.
+ *
+ * Uses special BranchNodeStruct state.validators data structure to optimize getting pubkeys.
+ * Type-unsafe: assumes state.validators[i] is of BranchNodeStruct type.
+ * Note: This is the fastest way of getting compressed pubkeys.
+ *       See benchmark -> packages/beacon-node/test/perf/api/impl/validator/attester.test.ts
+ */
+export function getPubkeysForIndices(
+  validators: BeaconStateAllForks["validators"],
+  indexes: ValidatorIndex[]
+): BLSPubkey[] {
+  const validatorsLen = validators.length; // Get once, it's expensive
+
+  const pubkeys: BLSPubkey[] = [];
+  for (let i = 0, len = indexes.length; i < len; i++) {
+    const index = indexes[i];
+    if (index >= validatorsLen) {
+      throw Error(`validatorIndex ${index} too high. Current validator count ${validatorsLen}`);
+    }
+
+    // NOTE: This could be optimized further by traversing the tree optimally with .getNodes()
+    const validator = validators.getReadonly(index);
+    pubkeys.push(validator.pubkey);
+  }
+
+  return pubkeys;
+}
+
+export function selectBlockProductionSource({
+  builderSelection,
+  engineExecutionPayloadValue,
+  builderExecutionPayloadValue,
+  builderBoostFactor,
+}: {
+  builderSelection: routes.validator.BuilderSelection;
+  engineExecutionPayloadValue: bigint;
+  builderExecutionPayloadValue: bigint;
+  builderBoostFactor: bigint;
+}): BlockSelectionResult {
+  switch (builderSelection) {
+    case routes.validator.BuilderSelection.ExecutionAlways:
+    case routes.validator.BuilderSelection.ExecutionOnly:
+      return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
+
+    case routes.validator.BuilderSelection.Default:
+    case routes.validator.BuilderSelection.MaxProfit: {
+      if (builderBoostFactor === BigInt(0)) {
+        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
+      }
+
+      if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
+        return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
+      }
+
+      if (engineExecutionPayloadValue >= (builderExecutionPayloadValue * builderBoostFactor) / BigInt(100)) {
+        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
+      }
+
+      return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
+    }
+
+    case routes.validator.BuilderSelection.BuilderAlways:
+    case routes.validator.BuilderSelection.BuilderOnly:
+      return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
+  }
+}

package/src/api/index.ts

@@ -0,0 +1,2 @@
+export * from "./impl/index.js";
+export * from "./rest/index.js";

package/src/api/options.ts

@@ -0,0 +1,16 @@
+import {BeaconRestApiServerOpts, beaconRestApiServerOpts} from "./rest/index.js";
+
+export type ApiOptions = {
+  maxGindicesInProof?: number;
+  rest: BeaconRestApiServerOpts;
+  commit?: string;
+  version?: string;
+  private?: boolean;
+};
+
+export const defaultApiOptions: ApiOptions = {
+  maxGindicesInProof: 512,
+  rest: beaconRestApiServerOpts,
+  version: "dev",
+  private: false,
+};

package/src/api/rest/activeSockets.ts

@@ -0,0 +1,109 @@
+import http, {Server} from "node:http";
+import {Socket} from "node:net";
+import {Gauge, GaugeExtra, waitFor} from "@lodestar/utils";
+
+export type SocketMetrics = {
+  activeSockets: GaugeExtra;
+  socketsBytesRead: Gauge;
+  socketsBytesWritten: Gauge;
+};
+
+// Use relatively short timeout to speed up shutdown
+const GRACEFUL_TERMINATION_TIMEOUT = 1_000;
+
+/**
+ * From https://github.com/gajus/http-terminator/blob/aabca4751552e983f8a59ba896b7fb58ce3b4087/src/factories/createInternalHttpTerminator.ts#L24-L61
+ * But only handles HTTP sockets, exposes the count of sockets as metrics
+ */
+export class HttpActiveSocketsTracker {
+  private sockets = new Set<Socket>();
+  private terminating = false;
+
+  constructor(
+    private readonly server: Server,
+    metrics: SocketMetrics | null
+  ) {
+    server.on("connection", (socket) => {
+      if (this.terminating) {
+        socket.destroy(Error("Closing"));
+      } else {
+        this.sockets.add(socket);
+
+        socket.once("close", () => {
+          this.sockets.delete(socket);
+          if (metrics) {
+            metrics.socketsBytesRead.inc(socket.bytesRead);
+            metrics.socketsBytesWritten.inc(socket.bytesWritten);
+          }
+        });
+      }
+    });
+
+    if (metrics) {
+      metrics.activeSockets.addCollect(() => metrics.activeSockets.set(this.sockets.size));
+      // Note: After some testing seems that `socket.writableLength` does not provide useful data, it's always 0
+    }
+  }
+
+  /**
+   * Wait for all connections to drain, forcefully terminate any open connections after timeout
+   *
+   * From https://github.com/gajus/http-terminator/blob/aabca4751552e983f8a59ba896b7fb58ce3b4087/src/factories/createInternalHttpTerminator.ts#L78-L165
+   * But only handles HTTP sockets and does not close server, immediately closes eventstream API connections
+   */
+  async terminate(): Promise<void> {
+    if (this.terminating) return;
+    this.terminating = true;
+
+    // Can speed up shutdown by a few milliseconds
+    this.server.closeIdleConnections();
+
+    // Inform new incoming requests on keep-alive connections that
+    // the connection will be closed after the current response
+    this.server.on("request", (_req, res) => {
+      if (!res.headersSent) {
+        res.setHeader("Connection", "close");
+      }
+    });
+
+    for (const socket of this.sockets) {
+      // This is the HTTP CONNECT request socket.
+      // @ts-expect-error HTTP sockets have reference to server
+      if (!(socket.server instanceof http.Server)) {
+        continue;
+      }
+
+      // @ts-expect-error Internal property but only way to access response of socket
+      const res = socket._httpMessage as http.ServerResponse | undefined;
+
+      if (res == null) {
+        // Immediately destroy sockets without an attached HTTP request
+        this.destroySocket(socket);
+      } else if (res.getHeader("Content-Type") === "text/event-stream") {
+        // eventstream API will never stop and must be forcefully closed
+        socket.end();
+      } else if (!res.headersSent) {
+        // Inform existing keep-alive connections that they will be closed after the current response
+        res.setHeader("Connection", "close");
+      }
+    }
+
+    // Wait for all connections to drain, forcefully terminate after timeout
+    try {
+      await waitFor(() => this.sockets.size === 0, {
+        timeout: GRACEFUL_TERMINATION_TIMEOUT,
+      });
+    } catch (_e) {
+      // Ignore timeout error
+    } finally {
+      for (const socket of this.sockets) {
+        this.destroySocket(socket);
+      }
+    }
+  }
+
+  private destroySocket(socket: Socket): void {
+    socket.destroy(Error("Closing"));
+    this.sockets.delete(socket);
+  }
+}

package/src/api/rest/base.ts

@@ -0,0 +1,216 @@
+import bearerAuthPlugin from "@fastify/bearer-auth";
+import {fastifyCors} from "@fastify/cors";
+import {FastifyInstance, FastifyRequest, errorCodes, fastify} from "fastify";
+import {parse as parseQueryString} from "qs";
+import {addSszContentTypeParser} from "@lodestar/api/server";
+import {ErrorAborted, Gauge, Histogram, Logger} from "@lodestar/utils";
+import {isLocalhostIP} from "../../util/ip.js";
+import {ApiError, FailureList, IndexedError, NodeIsSyncing} from "../impl/errors.js";
+import {HttpActiveSocketsTracker, SocketMetrics} from "./activeSockets.js";
+
+export type RestApiServerOpts = {
+  port: number;
+  cors?: string;
+  address?: string;
+  bearerToken?: string;
+  headerLimit?: number;
+  bodyLimit?: number;
+  stacktraces?: boolean;
+  swaggerUI?: boolean;
+};
+
+export type RestApiServerModules = {
+  logger: Logger;
+  metrics: RestApiServerMetrics | null;
+};
+
+export type RestApiServerMetrics = SocketMetrics & {
+  requests: Gauge<{operationId: string}>;
+  responseTime: Histogram<{operationId: string}>;
+  errors: Gauge<{operationId: string}>;
+};
+
+/**
+ * Error response body format as defined in beacon-api spec
+ *
+ * See https://github.com/ethereum/beacon-APIs/blob/v3.1.0/types/http.yaml
+ */
+type ErrorResponse = {
+  code: number;
+  message: string;
+  stacktraces?: string[];
+};
+
+type IndexedErrorResponse = ErrorResponse & {
+  failures?: FailureList;
+};
+
+/**
+ * Error code used by Fastify if media type is not supported (415)
+ */
+const INVALID_MEDIA_TYPE_CODE = errorCodes.FST_ERR_CTP_INVALID_MEDIA_TYPE().code;
+
+/**
+ * Error code used by Fastify if JSON schema validation failed
+ */
+const SCHEMA_VALIDATION_ERROR_CODE = errorCodes.FST_ERR_VALIDATION().code;
+
+/**
+ * REST API powered by `fastify` server.
+ */
+export class RestApiServer {
+  protected readonly server: FastifyInstance;
+  protected readonly logger: Logger;
+  private readonly activeSockets: HttpActiveSocketsTracker;
+
+  constructor(
+    protected readonly opts: RestApiServerOpts,
+    modules: RestApiServerModules
+  ) {
+    // Apply opts defaults
+    const {logger, metrics} = modules;
+
+    const server = fastify({
+      logger: false,
+      ajv: {customOptions: {coerceTypes: "array"}},
+      querystringParser: (str) =>
+        parseQueryString(str, {
+          // Array as comma-separated values must be supported to be OpenAPI spec compliant
+          comma: true,
+          // Drop support for array query strings like `id[0]=1&id[1]=2&id[2]=3` as those are not required to
+          // be OpenAPI spec compliant and results are inconsistent, see https://github.com/ljharb/qs/issues/331.
+          // The schema validation will catch this and throw an error as parsed query string results in an object.
+          parseArrays: false,
+        }),
+      bodyLimit: opts.bodyLimit,
+      http: {maxHeaderSize: opts.headerLimit},
+    });
+
+    addSszContentTypeParser(server);
+
+    this.activeSockets = new HttpActiveSocketsTracker(server.server, metrics);
+
+    // To parse our ApiError -> statusCode
+    server.setErrorHandler((err, _req, res) => {
+      const stacktraces = opts.stacktraces ? err.stack?.split("\n") : undefined;
+      if (err.validation) {
+        const {instancePath, message} = err.validation[0];
+        const payload: ErrorResponse = {
+          code: 400,
+          message: `${instancePath.substring(instancePath.lastIndexOf("/") + 1)} ${message}`,
+          stacktraces,
+        };
+        void res.status(400).send(payload);
+      } else if (err instanceof IndexedError) {
+        const payload: IndexedErrorResponse = {
+          code: err.statusCode,
+          message: err.message,
+          failures: err.failures,
+          stacktraces,
+        };
+        void res.status(err.statusCode).send(payload);
+      } else {
+        // Convert our custom ApiError into status code
+        const statusCode = err instanceof ApiError ? err.statusCode : 500;
+        const payload: ErrorResponse = {code: statusCode, message: err.message, stacktraces};
+        void res.status(statusCode).send(payload);
+      }
+    });
+
+    server.setNotFoundHandler((req, res) => {
+      const message = `Route ${req.raw.method}:${req.raw.url} not found`;
+      this.logger.warn(message);
+      const payload: ErrorResponse = {code: 404, message};
+      void res.code(404).send(payload);
+    });
+
+    if (opts.cors) {
+      void server.register(fastifyCors, {origin: opts.cors});
+    }
+
+    if (opts.bearerToken) {
+      void server.register(bearerAuthPlugin, {keys: new Set([opts.bearerToken])});
+    }
+
+    // Log all incoming request to debug (before parsing). TODO: Should we hook latter in the lifecycle? https://www.fastify.io/docs/latest/Lifecycle/
+    // Note: Must be an async method so fastify can continue the release lifecycle. Otherwise we must call done() or the request stalls
+    server.addHook("onRequest", async (req, _res) => {
+      const operationId = getOperationId(req);
+      this.logger.debug(`Req ${req.id} ${req.ip} ${operationId}`);
+      metrics?.requests.inc({operationId});
+    });
+
+    server.addHook("preHandler", async (req, _res) => {
+      const operationId = getOperationId(req);
+      this.logger.debug(`Exec ${req.id} ${req.ip} ${operationId}`);
+    });
+
+    // Log after response
+    server.addHook("onResponse", async (req, res) => {
+      const operationId = getOperationId(req);
+      this.logger.debug(`Res ${req.id} ${operationId} - ${res.raw.statusCode}`);
+      metrics?.responseTime.observe({operationId}, res.elapsedTime / 1000);
+    });
+
+    server.addHook("onError", async (req, _res, err) => {
+      // Don't log ErrorAborted errors, they happen on node shutdown and are not useful
+      // Don't log NodeISSyncing errors, they happen very frequently while syncing and the validator polls duties
+      if (err instanceof ErrorAborted || err instanceof NodeIsSyncing) return;
+
+      const operationId = getOperationId(req);
+
+      if (err instanceof ApiError || [INVALID_MEDIA_TYPE_CODE, SCHEMA_VALIDATION_ERROR_CODE].includes(err.code)) {
+        this.logger.warn(`Req ${req.id} ${operationId} failed`, {reason: err.message});
+      } else {
+        this.logger.error(`Req ${req.id} ${operationId} error`, {}, err);
+      }
+      metrics?.errors.inc({operationId});
+    });
+
+    this.server = server;
+    this.logger = logger;
+  }
+
+  /**
+   * Start the REST API server.
+   */
+  async listen(): Promise<void> {
+    try {
+      const host = this.opts.address;
+      await this.server.listen({port: this.opts.port, host});
+      const {address, port} = this.server.addresses()[0];
+      this.logger.info("Started REST API server", {address: `http://${address}:${port}`});
+      if (!host || !isLocalhostIP(host)) {
+        this.logger.warn("REST API server is exposed, ensure untrusted traffic cannot reach this API");
+      }
+    } catch (e) {
+      this.logger.error("Error starting REST api server", this.opts, e as Error);
+      throw e;
+    }
+  }
+
+  /**
+   * Close the server instance and terminate all existing connections.
+   */
+  async close(): Promise<void> {
+    // In NodeJS land calling close() only causes new connections to be rejected.
+    // Existing connections can prevent .close() from resolving for potentially forever.
+    // In Lodestar case when the BeaconNode wants to close we will attempt to gracefully
+    // close all existing connections but forcefully terminate after timeout for a fast shutdown.
+    // Inspired by https://github.com/gajus/http-terminator/
+    await this.activeSockets.terminate();
+
+    await this.server.close();
+
+    this.logger.debug("REST API server closed");
+  }
+
+  /** For child classes to override */
+  protected shouldIgnoreError(_err: Error): boolean {
+    return false;
+  }
+}
+
+function getOperationId(req: FastifyRequest): string {
+  return req.routeOptions.schema?.operationId ?? "unknown";
+}

package/src/api/rest/index.ts

@@ -0,0 +1,63 @@
+import {Endpoints} from "@lodestar/api";
+import {BeaconApiMethods, registerRoutes} from "@lodestar/api/beacon/server";
+import {ChainForkConfig} from "@lodestar/config";
+import {ErrorAborted, Logger} from "@lodestar/utils";
+import {NodeIsSyncing} from "../impl/errors.js";
+import {RestApiServer, RestApiServerMetrics, RestApiServerModules, RestApiServerOpts} from "./base.js";
+import {registerSwaggerUIRoutes} from "./swaggerUI.js";
+
+export {allNamespaces} from "@lodestar/api";
+
+export type BeaconRestApiServerOpts = Omit<RestApiServerOpts, "bearerToken"> & {
+  enabled: boolean;
+  api: (keyof Endpoints)[];
+};
+
+export const beaconRestApiServerOpts: BeaconRestApiServerOpts = {
+  enabled: true,
+  api: ["beacon", "config", "debug", "events", "node", "validator", "lightclient"],
+  address: "127.0.0.1",
+  port: 9596,
+  cors: "*",
+  // beacon -> validator API is trusted, and for large amounts of keys the payload is multi-MB
+  bodyLimit: 20 * 1024 * 1024, // 20MB for big block + blobs
+  stacktraces: false,
+};
+
+export type BeaconRestApiServerModules = RestApiServerModules & {
+  config: ChainForkConfig;
+  logger: Logger;
+  api: BeaconApiMethods;
+  metrics: RestApiServerMetrics | null;
+};
+
+/**
+ * REST API powered by `fastify` server.
+ */
+export class BeaconRestApiServer extends RestApiServer {
+  readonly opts: BeaconRestApiServerOpts;
+  readonly modules: BeaconRestApiServerModules;
+
+  constructor(optsArg: Partial<BeaconRestApiServerOpts>, modules: BeaconRestApiServerModules) {
+    const opts = {...beaconRestApiServerOpts, ...optsArg};
+
+    super(opts, modules);
+
+    this.opts = opts;
+    this.modules = modules;
+  }
+
+  async registerRoutes(version?: string): Promise<void> {
+    if (this.opts.swaggerUI) {
+      await registerSwaggerUIRoutes(this.server, this.opts, version);
+    }
+    // Instantiate and register the routes with matching namespace in `opts.api`
+    registerRoutes(this.server, this.modules.config, this.modules.api, this.opts.api);
+  }
+
+  protected shouldIgnoreError(err: Error): boolean {
+    // Don't log ErrorAborted errors, they happen on node shutdown and are not useful
+    // Don't log NodeISSyncing errors, they happen very frequently while syncing and the validator polls duties
+    return err instanceof ErrorAborted || err instanceof NodeIsSyncing;
+  }
+}

package/src/api/rest/swaggerUI.ts

@@ -0,0 +1,80 @@
+import {FastifyInstance} from "fastify";
+import {BeaconRestApiServerOpts} from "./index.js";
+
+export async function registerSwaggerUIRoutes(
+  server: FastifyInstance,
+  opts: BeaconRestApiServerOpts,
+  version = ""
+): Promise<void> {
+  const {fastifySwagger} = await import("@fastify/swagger");
+  await server.register(fastifySwagger, {
+    openapi: {
+      info: {
+        title: "Lodestar API",
+        description: "API specification for the Lodestar beacon node",
+        version,
+        contact: {
+          name: "Lodestar Github",
+          url: "https://github.com/chainsafe/lodestar",
+        },
+      },
+      externalDocs: {
+        url: "https://chainsafe.github.io/lodestar",
+        description: "Lodestar documentation",
+      },
+      tags: opts.api.map((namespace) => ({name: namespace})),
+    },
+  });
+  const {fastifySwaggerUi} = await import("@fastify/swagger-ui");
+  await server.register(fastifySwaggerUi, {
+    theme: {
+      title: "Lodestar API",
+      favicon: await getFavicon(),
+    },
+    logo: await getLogo(),
+  });
+}
+
+/**
+ * Fallback-friendly function to get an asset
+ */
+async function getAsset(name: string): Promise<Buffer | undefined> {
+  try {
+    const path = await import("node:path");
+    const fs = await import("node:fs/promises");
+    const url = await import("node:url");
+    const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
+    return await fs.readFile(path.join(__dirname, "../../../../../assets/", name));
+  } catch (_e) {
+    return undefined;
+  }
+}
+
+export async function getFavicon() {
+  const content = await getAsset("round-icon.ico");
+  if (!content) {
+    return undefined;
+  }
+
+  return [
+    {
+      filename: "round-icon.ico",
+      rel: "icon",
+      sizes: "16x16",
+      type: "image/x-icon",
+      content,
+    },
+  ];
+}
+
+export async function getLogo() {
+  const content = await getAsset("lodestar_icon_text_white.png");
+  if (!content) {
+    return undefined;
+  }
+
+  return {
+    type: "image/png",
+    content,
+  };
+}

package/src/bun-wrappers/prometheus-gc-stats.ts

@@ -0,0 +1,7 @@
+// Bun does not support `GCProfiler` from `node:v8`a
+// Which is exposed from `@chainsafe/prometheus-gc-stats` and used in `packages/beacon-node/src/metrics/nodeJsMetrics.ts`
+// So we expose a dummy function for Bun runtime which do nothing for now
+// TODO: May look get some useful data from `bun:jsc`
+export function gcStats() {
+  return () => {};
+}

package/src/chain/ColumnReconstructionTracker.ts

@@ -0,0 +1,90 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {Logger, sleep} from "@lodestar/utils";
+import {Metrics} from "../metrics/metrics.js";
+import {DataColumnReconstructionCode, recoverDataColumnSidecars} from "../util/dataColumns.js";
+import {BlockInputColumns} from "./blocks/blockInput/index.js";
+import {ChainEventEmitter} from "./emitter.js";
+
+/**
+ * Minimum time to wait before attempting reconstruction
+ */
+const RECONSTRUCTION_DELAY_MIN_MS = 800;
+
+/**
+ * Maximum time to wait before attempting reconstruction
+ */
+const RECONSTRUCTION_DELAY_MAX_MS = 1200;
+
+export type ColumnReconstructionTrackerInit = {
+  logger: Logger;
+  emitter: ChainEventEmitter;
+  metrics: Metrics | null;
+  config: ChainForkConfig;
+};
+
+/**
+ * Tracks column reconstruction attempts to avoid duplicate and multiple in-flight calls
+ */
+export class ColumnReconstructionTracker {
+  logger: Logger;
+  emitter: ChainEventEmitter;
+  metrics: Metrics | null;
+  config: ChainForkConfig;
+
+  /**
+   * Track last attempted block root
+   *
+   * This is sufficient to avoid duplicate calls since we only call this
+   * function when we see a new data column sidecar from gossip.
+   */
+  lastBlockRootHex: string | null = null;
+  /** Track if a reconstruction attempt is in-flight */
+  running = false;
+
+  constructor(init: ColumnReconstructionTrackerInit) {
+    this.logger = init.logger;
+    this.emitter = init.emitter;
+    this.metrics = init.metrics;
+    this.config = init.config;
+  }
+
+  triggerColumnReconstruction(blockInput: BlockInputColumns): void {
+    if (this.running) {
+      return;
+    }
+
+    if (this.lastBlockRootHex === blockInput.blockRootHex) {
+      return;
+    }
+
+    // We don't care about the outcome of this call,
+    // just that it has been triggered for this block root.
+    this.running = true;
+    this.lastBlockRootHex = blockInput.blockRootHex;
+    const delay =
+      RECONSTRUCTION_DELAY_MIN_MS + Math.random() * (RECONSTRUCTION_DELAY_MAX_MS - RECONSTRUCTION_DELAY_MIN_MS);
+    sleep(delay)
+      .then(() => {
+        const logCtx = {slot: blockInput.slot, root: blockInput.blockRootHex};
+        this.logger.debug("Attempting data column sidecar reconstruction", logCtx);
+        recoverDataColumnSidecars(blockInput, this.emitter, this.metrics)
+          .then((result) => {
+            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({result});
+            this.logger.debug("Data column sidecar reconstruction complete", {...logCtx, result});
+          })
+          .catch((e) => {
+            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({
+              result: DataColumnReconstructionCode.Failed,
+            });
+            this.logger.debug("Error during data column sidecar reconstruction", logCtx, e as Error);
+          })
+          .finally(() => {
+            this.logger.debug("Data column sidecar reconstruction attempt finished", logCtx);
+            this.running = false;
+          });
+      })
+      .catch((err) => {
+        this.logger.debug("ColumnReconstructionTracker unreachable error", {}, err);
+      });
+  }
+}