@lossless.org/objectstorage 10.7.1 → 11.1.0

package/ts/index.ts

@@ -1,5 +1,12 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
+import type { ObjectStorageDashboard } from './classes.dashboard.js';
+import type {
+  IObjectStorageClusterSnapshot,
+  IObjectStorageDashboardConfig,
+} from './dashboard.interfaces.js';
+
+export * from './dashboard.interfaces.js';
 
 /**
  * Authentication configuration
@@ -62,6 +69,11 @@ export interface ILimitsConfig {
 export interface IMultipartConfig {
   expirationDays?: number;
   cleanupIntervalMinutes?: number;
+  /**
+   * Open multipart uploads that the prefix credentials of one scope (bucket
+   * and prefix) may hold together (default 100). Counted in memory.
+   */
+  maxUploadsPerPrefixScope?: number;
 }
 
 /**
@@ -118,6 +130,11 @@ export interface IStorageConfig {
 export interface IErasureConfig {
   dataShards?: number;
   parityShards?: number;
+  /**
+   * Bytes per erasure-coded chunk, between 1 and 64 MiB (67,108,864 bytes);
+   * startup refuses a value outside that range, because every manifest
+   * refuses a larger chunk. Default 4,194,304 (4 MiB).
+   */
   chunkSizeBytes?: number;
 }
 
@@ -126,6 +143,31 @@ export interface IErasureConfig {
  */
 export interface IDriveConfig {
   paths: string[];
+  /**
+   * Accept drive roots that share a device (`st_dev`). Each drive root counts
+   * as one failure domain, so by default startup refuses two drive roots on
+   * one device. Set this only for dev and test clusters whose drive roots are
+   * directories on one filesystem. Default false.
+   */
+  allowSharedDevice?: boolean;
+}
+
+/**
+ * Mutual TLS for the cluster transport. Every file is read once at startup and
+ * must lie outside the control and drive roots, at an absolute path without
+ * symbolic links; a restart reloads them.
+ */
+export interface IClusterTlsConfig {
+  /** PEM file with one or two CA certificates (two only during a CA rotation). */
+  caFile: string;
+  /**
+   * PEM file with this node's certificate, optionally followed by intermediates. It has one DNS SAN,
+   * equal to nodeId, and is valid for server and client authentication: an extended key usage
+   * extension listing serverAuth and clientAuth, or no extended key usage extension.
+   */
+  certFile: string;
+  /** PEM private key of certFile; owned by the engine's user, mode 0600 or 0400. */
+  keyFile: string;
 }
 
 /**
@@ -133,6 +175,11 @@ export interface IDriveConfig {
  */
 export interface IClusterConfig {
   enabled: boolean;
+  /**
+   * This node's id. Generated and persisted when omitted, except with `tls`:
+   * then it is required and must be a lowercase DNS name equal to the node
+   * certificate's DNS subject alternative name.
+   */
   nodeId?: string;
   quicPort?: number;
   seedNodes?: string[];
@@ -140,12 +187,30 @@ export interface IClusterConfig {
   drives?: IDriveConfig;
   heartbeatIntervalMs?: number;
   heartbeatTimeoutMs?: number;
+  /**
+   * Longest a single GET may run, in milliseconds, from its start to its last
+   * byte, including the client's transfer time. A longer GET is aborted.
+   * Superseded and deleted object generations keep their shards for this
+   * deadline plus the smaller of the deadline and one hour, then cleanup frees
+   * them. Must be the same on every node: a mismatch can abort streams but
+   * never serves wrong bytes. Default 21,600,000 (6 hours); allowed from
+   * 60,000 (1 minute) to 604,800,000 (7 days).
+   */
+  getDeadlineMs?: number;
+  /**
+   * Mutual TLS for the cluster transport. Without it the transport is
+   * encrypted but authenticates no peer. Every node of a cluster sets it, or
+   * none does: enabling it is a full stop of every node, not a rolling change.
+   */
+  tls?: IClusterTlsConfig;
 }
 
 /**
  * Configuration as the engine receives it: every section resolved.
  */
-export type TObjectStorageResolvedConfig = Required<Omit<IObjectStorageConfig, 'binaryPath'>>;
+export type TObjectStorageResolvedConfig = Required<
+  Omit<IObjectStorageConfig, 'binaryPath' | 'dashboard'>
+>;
 
 /**
  * Complete objectstorage configuration
@@ -166,6 +231,11 @@ export interface IObjectStorageConfig {
   limits?: ILimitsConfig;
   multipart?: IMultipartConfig;
   cluster?: IClusterConfig;
+  /**
+   * The read-only visibility dashboard, served by this process on its own
+   * port. Off unless `enabled` is true; never sent to the engine.
+   */
+  dashboard?: IObjectStorageDashboardConfig;
 }
 
 /**
@@ -341,6 +411,35 @@ export interface IBucketTenantMetadata {
   retention?: IBucketRetentionReceipt;
 }
 
+export interface IPrefixCredentialInput {
+  bucketName: string;
+  /** Unreserved characters (RFC 3986) and `/`; ends with `/`, no leading `/`. */
+  prefix: string;
+  /** First node: lifetime in seconds, 1..43200. */
+  ttlSeconds?: number;
+  /** Fan-out to further nodes: the first node's expiresAt (epoch ms). */
+  expiresAt?: number;
+  /** Supply both together with expiresAt to install an existing credential on another node. */
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  region?: string;
+}
+
+export interface IPrefixCredentialMetadata {
+  accessKeyId: string;
+  bucketName: string;
+  prefix: string;
+  region: string;
+  /** Epoch ms. */
+  createdAt: number;
+  /** Epoch ms; requests are refused from this instant and admitted ones end at it. */
+  expiresAt: number;
+}
+
+export interface IPrefixCredential extends IPrefixCredentialMetadata {
+  secretAccessKey: string;
+}
+
 export interface IBucketRetentionPolicyInput {
   mode: 'compliance';
   policyId: string;
@@ -998,6 +1097,10 @@ export interface IObjectStorageHealth {
     enabled: boolean;
     credentialCount: number;
     tenantCredentialCount: number;
+    prefixCredentialCount: number;
+    prefixCredentialTombstoneCount: number;
+    /** False when the prefix credential file is unreadable: every prefix credential is refused. */
+    prefixCredentialStoreAvailable: boolean;
   };
   bucketCount: number;
   objectCount: number;
@@ -1062,6 +1165,55 @@ const normalizeResourceFenceError = (errorArg: unknown): Error => {
   );
 };
 
+export const objectStorageCredentialErrorCodes = [
+  'ECREDENTIAL_AUTH_DISABLED',
+  'ECREDENTIAL_INVALID_PREFIX',
+  'ECREDENTIAL_INVALID_TTL',
+  'ECREDENTIAL_INVALID_EXPIRY',
+  'ECREDENTIAL_INVALID_KEY',
+  'ECREDENTIAL_NO_SUCH_BUCKET',
+  'ECREDENTIAL_BUCKET_UNAVAILABLE',
+  'ECREDENTIAL_CONFLICT',
+  'ECREDENTIAL_REVOKED',
+  'ECREDENTIAL_NOT_PREFIX',
+  'ECREDENTIAL_LIMIT',
+  'ECREDENTIAL_PERSISTENCE',
+  'ECREDENTIAL_STORE_UNAVAILABLE',
+] as const;
+
+export type TObjectStorageCredentialErrorCode =
+  (typeof objectStorageCredentialErrorCodes)[number];
+
+export class ObjectStorageCredentialError extends Error {
+  public readonly code: TObjectStorageCredentialErrorCode;
+
+  public constructor(
+    codeArg: TObjectStorageCredentialErrorCode,
+    messageArg: string,
+    causeArg?: unknown,
+  ) {
+    super(`${codeArg}: ${messageArg}`, { cause: causeArg });
+    this.name = 'ObjectStorageCredentialError';
+    this.code = codeArg;
+  }
+}
+
+const normalizeCredentialError = (errorArg: unknown): Error => {
+  const error = errorArg instanceof Error ? errorArg : new Error(String(errorArg));
+  const match = /(ECREDENTIAL_[A-Z_]+):\s*(.*)$/s.exec(error.message);
+  if (
+    !match ||
+    !objectStorageCredentialErrorCodes.includes(match[1] as TObjectStorageCredentialErrorCode)
+  ) {
+    return error;
+  }
+  return new ObjectStorageCredentialError(
+    match[1] as TObjectStorageCredentialErrorCode,
+    match[2],
+    error,
+  );
+};
+
 export interface IObjectStorageMetrics {
   bucketCount: number;
   objectCount: number;
@@ -1085,6 +1237,35 @@ export interface IClusterPeerHealth {
   missedHeartbeats?: number;
 }
 
+/**
+ * A drive's medium: `'rotational'` and `'flash'` (network volumes included),
+ * `'smr'` for drive-managed SMR, which reports itself as rotational and is
+ * never detected, and `'zoned'` for host-managed or host-aware SMR and ZNS.
+ */
+export type TDriveMedia = 'rotational' | 'smr' | 'flash' | 'zoned';
+
+/**
+ * A drive's media.
+ */
+export interface IClusterDriveMedia {
+  /**
+   * The media the configuration declares for the drive. Not reported yet:
+   * drives cannot declare their media.
+   */
+  declared?: TDriveMedia;
+  /**
+   * What the engine detected once at startup, from the whole disk behind the
+   * drive root's filesystem (Linux sysfs `queue/zoned` and `queue/rotational`;
+   * a partition counts as its disk). `'unknown'` for filesystems without a
+   * block device of their own (tmpfs, overlay, network filesystems),
+   * device-mapper, md, loop and other virtual block devices, and on every
+   * platform but Linux. Virtual and network block devices report the
+   * rotational flag their driver or hypervisor sets (QEMU disks show
+   * `'rotational'` unless configured with `rotation_rate=1`).
+   */
+  detected: TDriveMedia | 'unknown';
+}
+
 /**
  * Local drive health as measured by objectstorage's runtime probes.
  */
@@ -1099,6 +1280,7 @@ export interface IClusterDriveHealth {
   lastError?: string;
   lastCheck?: number;
   erasureSetId?: number;
+  media: IClusterDriveMedia;
 }
 
 export interface IClusterErasureHealth {
@@ -1119,10 +1301,31 @@ export interface IClusterRepairHealth {
   lastDurationMs?: number;
   shardsChecked?: number;
   shardsHealed?: number;
+  /** Pending shards (writes that failed at publication) the last scan left pending. */
+  shardsPending?: number;
   failed?: number;
   lastError?: string;
 }
 
+export interface IClusterTransportCertificate {
+  /** SHA-256 of the DER certificate, lowercase hex. */
+  sha256: string;
+  /** Milliseconds since the Unix epoch. */
+  notAfter: number;
+}
+
+/**
+ * How the cluster transport authenticates peers, and the certificates it
+ * loaded at startup.
+ */
+export interface IClusterTransportHealth {
+  mode: 'mutualTls' | 'unauthenticated';
+  /** This node's certificate; only in mode 'mutualTls'. */
+  certificate?: IClusterTransportCertificate;
+  /** The loaded CA bundle in file order; only in mode 'mutualTls'. */
+  caCertificates?: IClusterTransportCertificate[];
+}
+
 /**
  * Cluster runtime health from the Rust core.
  * When clustering is disabled, the response is `{ enabled: false }`.
@@ -1138,6 +1341,36 @@ export interface IClusterHealth {
   drives?: IClusterDriveHealth[];
   erasure?: IClusterErasureHealth;
   repairs?: IClusterRepairHealth;
+  transport?: IClusterTransportHealth;
+}
+
+/**
+ * The result of one cluster healing scan, as `ObjectStorage.runHealingScan()`
+ * returns it. Times are milliseconds since the Unix epoch.
+ */
+export interface IClusterHealingScanResult {
+  /** When the scan started; always after the request arrived. */
+  startedAt: number;
+  completedAt: number;
+  durationMs: number;
+  /** Shard placements checked at their drives (placements on offline nodes are not). */
+  shardsChecked: number;
+  /** Shards written: missing placements rebuilt and pending shards completed. */
+  shardsHealed: number;
+  /** Pending shards (writes that failed at publication) this scan left pending. */
+  shardsPending: number;
+  /** Failed checks, rebuilds, writes and manifest stores. */
+  failed: number;
+}
+
+export interface IRunHealingScanOptions {
+  /**
+   * How long to wait for the scan, in milliseconds. Defaults to the bridge's
+   * request timeout (300,000). The engine cannot cancel a scan: one that
+   * outlasts this keeps running, its result is discarded, and a later call
+   * joins the next scan that starts after it.
+   */
+  timeoutMs?: number;
 }
 
 /**
@@ -1185,6 +1418,7 @@ const DEFAULT_CONFIG: IObjectStorageConfig = {
   multipart: {
     expirationDays: 7,
     cleanupIntervalMinutes: 60,
+    maxUploadsPerPrefixScope: 100,
   },
 };
 
@@ -1309,10 +1543,41 @@ function mergeConfig(userConfig: IObjectStorageConfig): TObjectStorageResolvedCo
   } as TObjectStorageResolvedConfig;
 }
 
+const DEFAULT_DASHBOARD_ADDRESS = '127.0.0.1';
+const DEFAULT_DASHBOARD_PORT = 3001;
+
+function resolveDashboardConfig(
+  dashboardArg: IObjectStorageDashboardConfig | undefined,
+  s3PortArg: number,
+): Required<IObjectStorageDashboardConfig> | undefined {
+  if (dashboardArg === undefined) {
+    return undefined;
+  }
+  if (typeof dashboardArg.enabled !== 'boolean') {
+    throw new Error('dashboard.enabled must be a boolean.');
+  }
+  const address = dashboardArg.address ?? DEFAULT_DASHBOARD_ADDRESS;
+  if (typeof address !== 'string' || address.length === 0) {
+    throw new Error('dashboard.address must be a non-empty string.');
+  }
+  const port = dashboardArg.port ?? DEFAULT_DASHBOARD_PORT;
+  if (!Number.isInteger(port) || port < 0 || port > 65535) {
+    throw new Error('dashboard.port must be an integer from 0 to 65535.');
+  }
+  if (dashboardArg.enabled && port !== 0 && port === s3PortArg) {
+    throw new Error(`dashboard.port ${port} is the S3 port; the dashboard needs its own port.`);
+  }
+  return { enabled: dashboardArg.enabled, address, port };
+}
+
 function createAccessKeyId(): string {
   return `SS${plugins.crypto.randomBytes(10).toString('hex').toUpperCase()}`;
 }
 
+function createPrefixAccessKeyId(): string {
+  return `SP${plugins.crypto.randomBytes(10).toString('hex').toUpperCase()}`;
+}
+
 function createSecretAccessKey(): string {
   return plugins.crypto.randomBytes(32).toString('hex');
 }
@@ -1441,7 +1706,30 @@ type TRustStorageCommands = {
   listBucketSummaries: { params: {}; result: IBucketSummary[] };
   listCredentials: { params: {}; result: IStorageCredentialMetadata[] };
   replaceCredentials: { params: { credentials: IStorageCredential[] }; result: {} };
+  createPrefixCredential: {
+    params: {
+      bucketName: string;
+      prefix: string;
+      accessKeyId: string;
+      secretAccessKey: string;
+      region?: string;
+      ttlSeconds?: number;
+      expiresAt?: number;
+    };
+    result: IPrefixCredentialMetadata;
+  };
+  revokePrefixCredential: { params: { accessKeyId: string }; result: { revoked: boolean } };
+  listPrefixCredentials: {
+    params: { bucketName?: string };
+    result: IPrefixCredentialMetadata[];
+  };
+  getPrefixCredentialStatus: {
+    params: {};
+    result: { available: boolean; credentialCount: number; tombstoneCount: number };
+  };
   getClusterHealth: { params: {}; result: IClusterHealth };
+  getClusterSnapshot: { params: {}; result: IObjectStorageClusterSnapshot };
+  runHealingScan: { params: {}; result: IClusterHealingScanResult };
 };
 
 /**
@@ -1525,6 +1813,8 @@ export class ObjectStorage {
   private bridgeOwned = false;
   private stopRequested = false;
   private stopCompleted = false;
+  private readonly dashboardConfig: Required<IObjectStorageDashboardConfig> | undefined;
+  private dashboard: ObjectStorageDashboard | undefined;
 
   constructor(configArg: IObjectStorageConfig = {}) {
     const binaryPath = configArg.binaryPath;
@@ -1535,6 +1825,7 @@ export class ObjectStorage {
       throw new Error('binaryPath must be a non-empty executable path.');
     }
     this.config = mergeConfig(configArg);
+    this.dashboardConfig = resolveDashboardConfig(configArg.dashboard, this.config.server.port!);
     this.bridge = new plugins.RustBridge<TRustStorageCommands>({
       binaryName: 'ruststorage',
       ...(binaryPath !== undefined
@@ -1600,6 +1891,23 @@ export class ObjectStorage {
         throw new Error('Storage engine did not report a valid bound TCP port.');
       }
       this.boundPort = result.port;
+      if (this.dashboardConfig?.enabled) {
+        // Enabled but unable to start (a port in use, say) fails startup:
+        // the catch below stops the engine.
+        const { ObjectStorageDashboard } = await import('./classes.dashboard.js');
+        const dashboard = new ObjectStorageDashboard({
+          address: this.dashboardConfig.address,
+          port: this.dashboardConfig.port,
+          readSnapshot: () => this.getClusterSnapshot(),
+        });
+        try {
+          await dashboard.start();
+        } catch (dashboardError) {
+          await dashboard.stop();
+          throw dashboardError;
+        }
+        this.dashboard = dashboard;
+      }
     } catch (error) {
       try {
         if (this.bridgeOwned) {
@@ -1627,6 +1935,11 @@ export class ObjectStorage {
     return this.running ? this.boundPort : this.config.server.port!;
   }
 
+  /** The dashboard's bound TCP port while it runs; `undefined` otherwise. */
+  public get dashboardPort(): number | undefined {
+    return this.dashboard?.port;
+  }
+
   public async getStorageDescriptor(
     optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>,
   ): Promise<plugins.tsclass.storage.IS3Descriptor> {
@@ -1730,7 +2043,7 @@ export class ObjectStorage {
       });
       return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1745,7 +2058,7 @@ export class ObjectStorage {
       });
       return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1754,7 +2067,7 @@ export class ObjectStorage {
     try {
       await this.bridge.sendCommand('deleteBucketTenant', tenantArg);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1771,7 +2084,7 @@ export class ObjectStorage {
       });
       return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1790,6 +2103,95 @@ export class ObjectStorage {
     return this.buildBucketTenantDescriptor(credential, optionsArg.bucketName);
   }
 
+  /**
+   * Creates a credential confined to one key prefix of one bucket until its
+   * expiry. The first node takes `ttlSeconds`; further nodes of a fan-out take
+   * the first result's `accessKeyId`, `secretAccessKey` and `expiresAt`. The
+   * secret is returned only here and never kept by this instance.
+   */
+  public async createPrefixCredential(
+    inputArg: IPrefixCredentialInput,
+  ): Promise<IPrefixCredential> {
+    const hasTtl = inputArg.ttlSeconds !== undefined;
+    const hasExpiresAt = inputArg.expiresAt !== undefined;
+    if (hasTtl === hasExpiresAt) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_TTL',
+        'Exactly one of ttlSeconds and expiresAt is required.',
+      );
+    }
+    if (hasTtl && (!Number.isSafeInteger(inputArg.ttlSeconds) || inputArg.ttlSeconds! < 1)) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_TTL',
+        'ttlSeconds must be an integer from 1 to 43200.',
+      );
+    }
+    if (hasExpiresAt && !Number.isSafeInteger(inputArg.expiresAt)) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_EXPIRY',
+        'expiresAt must be an integer epoch millisecond.',
+      );
+    }
+    const hasAccessKeyId = inputArg.accessKeyId !== undefined;
+    const hasSecretAccessKey = inputArg.secretAccessKey !== undefined;
+    if (hasAccessKeyId !== hasSecretAccessKey) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_KEY',
+        'accessKeyId and secretAccessKey must be supplied together.',
+      );
+    }
+    if (hasAccessKeyId && !hasExpiresAt) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_KEY',
+        'Key material is accepted only with expiresAt, to install an existing credential.',
+      );
+    }
+    const secretAccessKey = inputArg.secretAccessKey ?? createSecretAccessKey();
+    try {
+      const metadata = await this.bridge.sendCommand('createPrefixCredential', {
+        bucketName: inputArg.bucketName,
+        prefix: inputArg.prefix,
+        accessKeyId: inputArg.accessKeyId ?? createPrefixAccessKeyId(),
+        secretAccessKey,
+        region: inputArg.region || this.config.server.region,
+        ...(hasTtl ? { ttlSeconds: inputArg.ttlSeconds } : { expiresAt: inputArg.expiresAt }),
+      });
+      return { ...metadata, secretAccessKey };
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
+  }
+
+  /**
+   * Revokes a prefix credential on this node. An unknown ID is recorded as
+   * revoked too, so a later create for it is refused; `revoked` is true only
+   * when a live credential was removed.
+   */
+  public async revokePrefixCredential(optionsArg: {
+    accessKeyId: string;
+  }): Promise<{ revoked: boolean }> {
+    try {
+      return await this.bridge.sendCommand('revokePrefixCredential', {
+        accessKeyId: optionsArg.accessKeyId,
+      });
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
+  }
+
+  /** Lists this node's live prefix credentials, without secrets. */
+  public async listPrefixCredentials(
+    optionsArg: { bucketName?: string } = {},
+  ): Promise<IPrefixCredentialMetadata[]> {
+    try {
+      return await this.bridge.sendCommand('listPrefixCredentials', {
+        ...(optionsArg.bucketName !== undefined ? { bucketName: optionsArg.bucketName } : {}),
+      });
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
+  }
+
   public async getBucketRetentionCapability(): Promise<IBucketRetentionCapability> {
     return this.bridge.sendCommand('getBucketRetentionCapability', {});
   }
@@ -1864,7 +2266,7 @@ export class ObjectStorage {
     try {
       return await this.bridge.sendCommand('deleteBucket', optionsArg);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1914,7 +2316,11 @@ export class ObjectStorage {
    * if this override introduces ownership or credential-material drift.
    */
   public async replaceCredentials(credentials: IStorageCredential[]): Promise<void> {
-    await this.bridge.sendCommand('replaceCredentials', { credentials });
+    try {
+      await this.bridge.sendCommand('replaceCredentials', { credentials });
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
     this.config.auth.credentials = credentials.map((credential) => ({ ...credential }));
   }
 
@@ -1922,6 +2328,38 @@ export class ObjectStorage {
     return this.bridge.sendCommand('getClusterHealth', {});
   }
 
+  /**
+   * The dashboard's snapshot, built by the engine from memory only: it never
+   * probes a drive, syncs or walks storage. A cluster node also asks each
+   * online peer for its own snapshot, waiting up to a second per peer, in
+   * parallel; a peer that does not answer is reported unavailable. One
+   * snapshot runs at a time: a call while another runs rejects.
+   */
+  public async getClusterSnapshot(): Promise<IObjectStorageClusterSnapshot> {
+    return this.bridge.sendCommand('getClusterSnapshot', {});
+  }
+
+  /**
+   * Runs a healing scan on this cluster node and resolves with its result.
+   * The scan covers the manifests this node coordinated. The request joins
+   * the next scan that starts after it arrives, never one already running,
+   * and requests that arrive together share one scan. It rejects in
+   * standalone mode, when the scan fails (for example without a majority of
+   * the cluster, or when a bucket's manifests cannot be listed), when 16
+   * requests already wait for a scan to start, and when the engine shuts down
+   * before the scan completes. The engine answers other management calls
+   * while the scan runs.
+   */
+  public async runHealingScan(
+    optionsArg: IRunHealingScanOptions = {},
+  ): Promise<IClusterHealingScanResult> {
+    return this.bridge.sendCommand(
+      'runHealingScan',
+      {},
+      optionsArg.timeoutMs === undefined ? undefined : { timeoutMs: optionsArg.timeoutMs },
+    );
+  }
+
   public async getHealth(): Promise<IObjectStorageHealth> {
     if (!this.running) {
       return {
@@ -1933,6 +2371,9 @@ export class ObjectStorage {
           enabled: this.config.auth.enabled,
           credentialCount: this.config.auth.credentials.length,
           tenantCredentialCount: 0,
+          prefixCredentialCount: 0,
+          prefixCredentialTombstoneCount: 0,
+          prefixCredentialStoreAvailable: false,
         },
         bucketCount: 0,
         objectCount: 0,
@@ -1947,13 +2388,15 @@ export class ObjectStorage {
       };
     }
 
-    const [stats, credentials, tenants, cluster, resourceFencing] = await Promise.all([
-      this.getStorageStats(),
-      this.listCredentials(),
-      this.config.auth.enabled ? this.listBucketTenants() : Promise.resolve([]),
-      this.getClusterHealth(),
-      this.getResourceFencingCapability(),
-    ]);
+    const [stats, credentials, tenants, prefixCredentials, cluster, resourceFencing] =
+      await Promise.all([
+        this.getStorageStats(),
+        this.listCredentials(),
+        this.config.auth.enabled ? this.listBucketTenants() : Promise.resolve([]),
+        this.bridge.sendCommand('getPrefixCredentialStatus', {}),
+        this.getClusterHealth(),
+        this.getResourceFencingCapability(),
+      ]);
     return {
       ok: true,
       running: true,
@@ -1963,6 +2406,9 @@ export class ObjectStorage {
         enabled: this.config.auth.enabled,
         credentialCount: credentials.length,
         tenantCredentialCount: tenants.length,
+        prefixCredentialCount: prefixCredentials.credentialCount,
+        prefixCredentialTombstoneCount: prefixCredentials.tombstoneCount,
+        prefixCredentialStoreAvailable: prefixCredentials.available,
       },
       bucketCount: stats.bucketCount,
       objectCount: stats.totalObjectCount,
@@ -2002,6 +2448,15 @@ export class ObjectStorage {
         '# HELP objectstorage_tenant_credentials_total Scoped bucket tenant credential count.',
         '# TYPE objectstorage_tenant_credentials_total gauge',
         `objectstorage_tenant_credentials_total ${health.auth.tenantCredentialCount}`,
+        '# HELP objectstorage_prefix_credentials_total Live prefix credential count.',
+        '# TYPE objectstorage_prefix_credentials_total gauge',
+        `objectstorage_prefix_credentials_total ${health.auth.prefixCredentialCount}`,
+        '# HELP objectstorage_prefix_credential_tombstones_total Unexpired prefix credential revocations.',
+        '# TYPE objectstorage_prefix_credential_tombstones_total gauge',
+        `objectstorage_prefix_credential_tombstones_total ${health.auth.prefixCredentialTombstoneCount}`,
+        '# HELP objectstorage_prefix_credential_store_available Prefix credential store readable.',
+        '# TYPE objectstorage_prefix_credential_store_available gauge',
+        `objectstorage_prefix_credential_store_available ${health.auth.prefixCredentialStoreAvailable ? 1 : 0}`,
         '# HELP objectstorage_cluster_enabled Cluster mode enabled.',
         '# TYPE objectstorage_cluster_enabled gauge',
         `objectstorage_cluster_enabled ${clusterEnabled ? 1 : 0}`,
@@ -2034,6 +2489,10 @@ export class ObjectStorage {
       // Startup keeps its own rejection; stop joins only its resource ownership.
       await Promise.allSettled([this.startPromise]);
     }
+    if (this.dashboard) {
+      await this.dashboard.stop();
+      this.dashboard = undefined;
+    }
     if (this.bridgeOwned) {
       if (this.running) {
         this.completedStopResult ??= await this.bridge.sendCommand('stop', {});