@lossless.org/objectstorage 11.0.0 → 11.2.0

package/ts/dashboard.interfaces.ts

@@ -0,0 +1,148 @@
+import type { ITypedRequest, implementsTR } from '@api.global/typedrequest-interfaces';
+import type { ITypedSocketEvent } from '@api.global/typedsocket';
+
+/**
+ * Engine snapshot shapes, as `ObjectStorage.getClusterSnapshot()` returns
+ * them. The engine builds them from memory only: the drive states the last
+ * heartbeat probe left, runtime stats, healing state, S3 listener counters
+ * and a bounded ring of recent events. Times are milliseconds since the Unix
+ * epoch; absent values are `null`.
+ */
+export interface IObjectStorageTelemetryEvent {
+  /** Numbered per node from 1; a gap means older events left the ring. */
+  seq: number;
+  at: number;
+  kind: 'engine' | 'peer' | 'drive' | 'healing';
+  severity: 'info' | 'warning' | 'error';
+  message: string;
+}
+
+/** S3 listener counters since the engine started; bytes include HTTP headers. */
+export interface IObjectStorageTrafficCounters {
+  requestsTotal: number;
+  errorResponses: number;
+  bytesIn: number;
+  bytesOut: number;
+}
+
+export interface IObjectStorageStorageTotals {
+  bucketCount: number;
+  objectCount: number;
+  totalBytes: number;
+}
+
+export interface IObjectStorageDriveSnapshot {
+  index: number;
+  path: string;
+  status: 'online' | 'degraded' | 'offline' | 'healing';
+  totalBytes: number;
+  usedBytes: number;
+  availableBytes: number;
+  errorCount: number;
+  lastError: string | null;
+  lastCheck: number | null;
+  erasureSetId: number | null;
+}
+
+export interface IObjectStorageHealingSnapshot {
+  active: boolean;
+  scanIntervalMs: number;
+  lastRunStartedAt: number | null;
+  lastRunCompletedAt: number | null;
+  lastDurationMs: number | null;
+  shardsChecked: number | null;
+  shardsHealed: number | null;
+  shardsPending: number | null;
+  failed: number | null;
+  lastError: string | null;
+}
+
+/** What one node reports about itself; local and remote nodes share this shape. */
+export interface IObjectStorageNodeSnapshot {
+  /** Absent in standalone mode. */
+  nodeId: string | null;
+  mode: 'standalone' | 'cluster';
+  startedAt: number;
+  capturedAt: number;
+  traffic: IObjectStorageTrafficCounters;
+  storage: IObjectStorageStorageTotals;
+  /** Local drives (cluster mode); empty in standalone mode. */
+  drives: IObjectStorageDriveSnapshot[];
+  healing: IObjectStorageHealingSnapshot | null;
+  /** Recent events, oldest first. */
+  events: IObjectStorageTelemetryEvent[];
+}
+
+export interface IObjectStoragePeerSnapshot {
+  nodeId: string;
+  status: 'online' | 'suspect' | 'offline';
+  quicAddress: string;
+  s3Address: string;
+  driveCount: number;
+  lastHeartbeat: number;
+  missedHeartbeats: number;
+  /** The peer's own snapshot, when it answered within a second. */
+  snapshot: IObjectStorageNodeSnapshot | null;
+  unavailableReason: string | null;
+}
+
+export interface IObjectStorageErasureSnapshot {
+  dataShards: number;
+  parityShards: number;
+  chunkSizeBytes: number;
+  totalShards: number;
+  readQuorum: number;
+  writeQuorum: number;
+  erasureSetCount: number;
+}
+
+export interface IObjectStorageClusterViewSnapshot {
+  nodeId: string;
+  quorumHealthy: boolean;
+  majorityHealthy: boolean;
+  erasure: IObjectStorageErasureSnapshot;
+  peers: IObjectStoragePeerSnapshot[];
+}
+
+export interface IObjectStorageClusterSnapshot {
+  capturedAt: number;
+  clusterEnabled: boolean;
+  local: IObjectStorageNodeSnapshot;
+  /** Absent in standalone mode. */
+  cluster: IObjectStorageClusterViewSnapshot | null;
+}
+
+/**
+ * Dashboard configuration. The dashboard is off unless `enabled` is true.
+ */
+export interface IObjectStorageDashboardConfig {
+  enabled: boolean;
+  /** Interface the dashboard listens on. Default `127.0.0.1`. */
+  address?: string;
+  /** TCP port; `0` asks the OS for one. Default 3001; must differ from the S3 port. */
+  port?: number;
+}
+
+/** One dashboard sample, as pushed to every connected dashboard. */
+export interface IObjectStorageDashboardSample {
+  sampledAt: number;
+  /** The snapshot, or `null` when the engine could not answer. */
+  snapshot: IObjectStorageClusterSnapshot | null;
+  error: string | null;
+  /** Events were left out of peer snapshots to keep the sample within the push limit. */
+  eventsTruncated: boolean;
+}
+
+/** Answers with the latest sample, taking one when none exists yet. */
+export interface IReq_ObjectStorageDashboardGetSample
+  extends implementsTR<ITypedRequest, IReq_ObjectStorageDashboardGetSample> {
+  method: 'objectStorageDashboardGetSample';
+  request: Record<string, never>;
+  response: IObjectStorageDashboardSample;
+}
+
+/** Pushed every two seconds while at least one dashboard is connected. */
+export interface IEvent_ObjectStorageDashboardSample
+  extends ITypedSocketEvent<IObjectStorageDashboardSample> {
+  name: 'objectStorageDashboardSample';
+}

package/ts/dashboard.plugins.ts

@@ -0,0 +1,13 @@
+// Loaded only by the dashboard module, which `ObjectStorage` imports
+// dynamically when the dashboard is enabled.
+
+// @api.global scope
+import * as typedrequest from '@api.global/typedrequest';
+import * as typedsocket from '@api.global/typedsocket';
+
+export { typedrequest, typedsocket };
+
+// @push.rocks scope
+import * as smartserve from '@push.rocks/smartserve';
+
+export { smartserve };

package/ts/index.ts

@@ -1,5 +1,12 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
+import type { ObjectStorageDashboard } from './classes.dashboard.js';
+import type {
+  IObjectStorageClusterSnapshot,
+  IObjectStorageDashboardConfig,
+} from './dashboard.interfaces.js';
+
+export * from './dashboard.interfaces.js';
 
 /**
  * Authentication configuration
@@ -145,11 +152,34 @@ export interface IDriveConfig {
   allowSharedDevice?: boolean;
 }
 
+/**
+ * Mutual TLS for the cluster transport. Every file is read once at startup and
+ * must lie outside the control and drive roots, at an absolute path without
+ * symbolic links; a restart reloads them.
+ */
+export interface IClusterTlsConfig {
+  /** PEM file with one or two CA certificates (two only during a CA rotation). */
+  caFile: string;
+  /**
+   * PEM file with this node's certificate, optionally followed by intermediates. It has one DNS SAN,
+   * equal to nodeId, and is valid for server and client authentication: an extended key usage
+   * extension listing serverAuth and clientAuth, or no extended key usage extension.
+   */
+  certFile: string;
+  /** PEM private key of certFile; owned by the engine's user, mode 0600 or 0400. */
+  keyFile: string;
+}
+
 /**
  * Cluster configuration for distributed mode
  */
 export interface IClusterConfig {
   enabled: boolean;
+  /**
+   * This node's id. Generated and persisted when omitted, except with `tls`:
+   * then it is required and must be a lowercase DNS name equal to the node
+   * certificate's DNS subject alternative name.
+   */
   nodeId?: string;
   quicPort?: number;
   seedNodes?: string[];
@@ -167,12 +197,20 @@ export interface IClusterConfig {
    * 60,000 (1 minute) to 604,800,000 (7 days).
    */
   getDeadlineMs?: number;
+  /**
+   * Mutual TLS for the cluster transport. Without it the transport is
+   * encrypted but authenticates no peer. Every node of a cluster sets it, or
+   * none does: enabling it is a full stop of every node, not a rolling change.
+   */
+  tls?: IClusterTlsConfig;
 }
 
 /**
  * Configuration as the engine receives it: every section resolved.
  */
-export type TObjectStorageResolvedConfig = Required<Omit<IObjectStorageConfig, 'binaryPath'>>;
+export type TObjectStorageResolvedConfig = Required<
+  Omit<IObjectStorageConfig, 'binaryPath' | 'dashboard'>
+>;
 
 /**
  * Complete objectstorage configuration
@@ -193,17 +231,34 @@ export interface IObjectStorageConfig {
   limits?: ILimitsConfig;
   multipart?: IMultipartConfig;
   cluster?: IClusterConfig;
+  /**
+   * The read-only visibility dashboard, served by this process on its own
+   * port. Off unless `enabled` is true; never sent to the engine.
+   */
+  dashboard?: IObjectStorageDashboardConfig;
 }
 
 /**
  * Logical bucket stats maintained by the Rust runtime.
  * Values are initialized from native storage on startup and updated on objectstorage mutations.
  */
+/** A versioned bucket's S3 versioning status; a bucket that was never versioned has none. */
+export type TBucketVersioningStatus = 'Enabled' | 'Suspended';
+
 export interface IBucketSummary {
   name: string;
+  /** Current objects; noncurrent versions and delete markers are counted below. */
   objectCount: number;
   totalSizeBytes: number;
   creationDate?: number;
+  /** Present only for a versioned bucket (standalone), and so are the counters below. */
+  versioning?: TBucketVersioningStatus;
+  /** Noncurrent data versions. */
+  noncurrentVersionCount?: number;
+  /** Payload bytes of the noncurrent data versions. */
+  noncurrentBytes?: number;
+  /** Delete markers, current and noncurrent. */
+  deleteMarkerCount?: number;
 }
 
 /**
@@ -332,6 +387,15 @@ export interface IStorageStats {
   buckets: IBucketSummary[];
   storageDirectory: string;
   storageLocations?: IStorageLocationSummary[];
+  /** Totals over the versioned buckets; absent without one. */
+  totalNoncurrentVersionCount?: number;
+  totalNoncurrentBytes?: number;
+  totalDeleteMarkerCount?: number;
+  /**
+   * How far version stamps run ahead of the wall clock, after the wall clock
+   * went backwards (or a forward jump was corrected); absent when they do not.
+   */
+  stampClockAheadOfWallMs?: number;
 }
 
 export interface IStorageStopResult {
@@ -1070,6 +1134,8 @@ export interface IObjectStorageHealth {
   resourcePublicationHoldSupported: boolean;
   resourcePublicationHoldRequiresExternalDrain: false;
   resourceFencingReason?: string;
+  /** See `IStorageStats.stampClockAheadOfWallMs`. */
+  stampClockAheadOfWallMs?: number;
 }
 
 export const objectStorageResourceFenceErrorCodes = [
@@ -1194,6 +1260,35 @@ export interface IClusterPeerHealth {
   missedHeartbeats?: number;
 }
 
+/**
+ * A drive's medium: `'rotational'` and `'flash'` (network volumes included),
+ * `'smr'` for drive-managed SMR, which reports itself as rotational and is
+ * never detected, and `'zoned'` for host-managed or host-aware SMR and ZNS.
+ */
+export type TDriveMedia = 'rotational' | 'smr' | 'flash' | 'zoned';
+
+/**
+ * A drive's media.
+ */
+export interface IClusterDriveMedia {
+  /**
+   * The media the configuration declares for the drive. Not reported yet:
+   * drives cannot declare their media.
+   */
+  declared?: TDriveMedia;
+  /**
+   * What the engine detected once at startup, from the whole disk behind the
+   * drive root's filesystem (Linux sysfs `queue/zoned` and `queue/rotational`;
+   * a partition counts as its disk). `'unknown'` for filesystems without a
+   * block device of their own (tmpfs, overlay, network filesystems),
+   * device-mapper, md, loop and other virtual block devices, and on every
+   * platform but Linux. Virtual and network block devices report the
+   * rotational flag their driver or hypervisor sets (QEMU disks show
+   * `'rotational'` unless configured with `rotation_rate=1`).
+   */
+  detected: TDriveMedia | 'unknown';
+}
+
 /**
  * Local drive health as measured by objectstorage's runtime probes.
  */
@@ -1208,6 +1303,7 @@ export interface IClusterDriveHealth {
   lastError?: string;
   lastCheck?: number;
   erasureSetId?: number;
+  media: IClusterDriveMedia;
 }
 
 export interface IClusterErasureHealth {
@@ -1234,6 +1330,25 @@ export interface IClusterRepairHealth {
   lastError?: string;
 }
 
+export interface IClusterTransportCertificate {
+  /** SHA-256 of the DER certificate, lowercase hex. */
+  sha256: string;
+  /** Milliseconds since the Unix epoch. */
+  notAfter: number;
+}
+
+/**
+ * How the cluster transport authenticates peers, and the certificates it
+ * loaded at startup.
+ */
+export interface IClusterTransportHealth {
+  mode: 'mutualTls' | 'unauthenticated';
+  /** This node's certificate; only in mode 'mutualTls'. */
+  certificate?: IClusterTransportCertificate;
+  /** The loaded CA bundle in file order; only in mode 'mutualTls'. */
+  caCertificates?: IClusterTransportCertificate[];
+}
+
 /**
  * Cluster runtime health from the Rust core.
  * When clustering is disabled, the response is `{ enabled: false }`.
@@ -1249,6 +1364,7 @@ export interface IClusterHealth {
   drives?: IClusterDriveHealth[];
   erasure?: IClusterErasureHealth;
   repairs?: IClusterRepairHealth;
+  transport?: IClusterTransportHealth;
 }
 
 /**
@@ -1333,6 +1449,14 @@ const DEFAULT_CONFIG: IObjectStorageConfig = {
 // Keep this aligned with the bounded Rust management-line reader.
 const MAXIMUM_MANAGEMENT_PAYLOAD_BYTES = 144 * 1024 * 1024;
 
+// The `start` command's timeout: the largest delay Node's setTimeout honours
+// (2^31 - 1 ms, about 24.8 days; Node clamps any larger delay, so also
+// Number.MAX_SAFE_INTEGER, to 1 ms). Startup recovers every version chain and
+// scans every current object, so it grows with the stored data and has no
+// fixed bound; an engine that dies during startup still rejects `start` at
+// once, through the bridge's exit detection.
+const START_COMMAND_TIMEOUT_MS = 2_147_483_647;
+
 function assertExactMountedDirectory(directoryArg: string, fieldArg: string): void {
   if (
     directoryArg.length === 0 ||
@@ -1450,6 +1574,33 @@ function mergeConfig(userConfig: IObjectStorageConfig): TObjectStorageResolvedCo
   } as TObjectStorageResolvedConfig;
 }
 
+const DEFAULT_DASHBOARD_ADDRESS = '127.0.0.1';
+const DEFAULT_DASHBOARD_PORT = 3001;
+
+function resolveDashboardConfig(
+  dashboardArg: IObjectStorageDashboardConfig | undefined,
+  s3PortArg: number,
+): Required<IObjectStorageDashboardConfig> | undefined {
+  if (dashboardArg === undefined) {
+    return undefined;
+  }
+  if (typeof dashboardArg.enabled !== 'boolean') {
+    throw new Error('dashboard.enabled must be a boolean.');
+  }
+  const address = dashboardArg.address ?? DEFAULT_DASHBOARD_ADDRESS;
+  if (typeof address !== 'string' || address.length === 0) {
+    throw new Error('dashboard.address must be a non-empty string.');
+  }
+  const port = dashboardArg.port ?? DEFAULT_DASHBOARD_PORT;
+  if (!Number.isInteger(port) || port < 0 || port > 65535) {
+    throw new Error('dashboard.port must be an integer from 0 to 65535.');
+  }
+  if (dashboardArg.enabled && port !== 0 && port === s3PortArg) {
+    throw new Error(`dashboard.port ${port} is the S3 port; the dashboard needs its own port.`);
+  }
+  return { enabled: dashboardArg.enabled, address, port };
+}
+
 function createAccessKeyId(): string {
   return `SS${plugins.crypto.randomBytes(10).toString('hex').toUpperCase()}`;
 }
@@ -1608,6 +1759,7 @@ type TRustStorageCommands = {
     result: { available: boolean; credentialCount: number; tombstoneCount: number };
   };
   getClusterHealth: { params: {}; result: IClusterHealth };
+  getClusterSnapshot: { params: {}; result: IObjectStorageClusterSnapshot };
   runHealingScan: { params: {}; result: IClusterHealingScanResult };
 };
 
@@ -1692,6 +1844,8 @@ export class ObjectStorage {
   private bridgeOwned = false;
   private stopRequested = false;
   private stopCompleted = false;
+  private readonly dashboardConfig: Required<IObjectStorageDashboardConfig> | undefined;
+  private dashboard: ObjectStorageDashboard | undefined;
 
   constructor(configArg: IObjectStorageConfig = {}) {
     const binaryPath = configArg.binaryPath;
@@ -1702,6 +1856,7 @@ export class ObjectStorage {
       throw new Error('binaryPath must be a non-empty executable path.');
     }
     this.config = mergeConfig(configArg);
+    this.dashboardConfig = resolveDashboardConfig(configArg.dashboard, this.config.server.port!);
     this.bridge = new plugins.RustBridge<TRustStorageCommands>({
       binaryName: 'ruststorage',
       ...(binaryPath !== undefined
@@ -1724,6 +1879,12 @@ export class ObjectStorage {
         console.error(`ruststorage engine exited unexpectedly (code: ${code}, signal: ${signal}).`);
       }
     });
+    // The engine logs (warnings, errors, startup recovery progress) to its
+    // stderr; forward every line verbatim to the host's stderr, as an
+    // inherited stderr would.
+    this.bridge.on('stderr', (line: string) => {
+      console.error(line);
+    });
   }
 
   public start(): Promise<void> {
@@ -1762,11 +1923,32 @@ export class ObjectStorage {
       if (!spawned) {
         throw new Error('Failed to spawn ruststorage binary. Make sure it is compiled (pnpm build).');
       }
-      const result = await this.bridge.sendCommand('start', { config: this.config });
+      const result = await this.bridge.sendCommand(
+        'start',
+        { config: this.config },
+        { timeoutMs: START_COMMAND_TIMEOUT_MS },
+      );
       if (!Number.isInteger(result.port) || result.port < 1 || result.port > 65535) {
         throw new Error('Storage engine did not report a valid bound TCP port.');
       }
       this.boundPort = result.port;
+      if (this.dashboardConfig?.enabled) {
+        // Enabled but unable to start (a port in use, say) fails startup:
+        // the catch below stops the engine.
+        const { ObjectStorageDashboard } = await import('./classes.dashboard.js');
+        const dashboard = new ObjectStorageDashboard({
+          address: this.dashboardConfig.address,
+          port: this.dashboardConfig.port,
+          readSnapshot: () => this.getClusterSnapshot(),
+        });
+        try {
+          await dashboard.start();
+        } catch (dashboardError) {
+          await dashboard.stop();
+          throw dashboardError;
+        }
+        this.dashboard = dashboard;
+      }
     } catch (error) {
       try {
         if (this.bridgeOwned) {
@@ -1794,6 +1976,11 @@ export class ObjectStorage {
     return this.running ? this.boundPort : this.config.server.port!;
   }
 
+  /** The dashboard's bound TCP port while it runs; `undefined` otherwise. */
+  public get dashboardPort(): number | undefined {
+    return this.dashboard?.port;
+  }
+
   public async getStorageDescriptor(
     optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>,
   ): Promise<plugins.tsclass.storage.IS3Descriptor> {
@@ -2182,6 +2369,17 @@ export class ObjectStorage {
     return this.bridge.sendCommand('getClusterHealth', {});
   }
 
+  /**
+   * The dashboard's snapshot, built by the engine from memory only: it never
+   * probes a drive, syncs or walks storage. A cluster node also asks each
+   * online peer for its own snapshot, waiting up to a second per peer, in
+   * parallel; a peer that does not answer is reported unavailable. One
+   * snapshot runs at a time: a call while another runs rejects.
+   */
+  public async getClusterSnapshot(): Promise<IObjectStorageClusterSnapshot> {
+    return this.bridge.sendCommand('getClusterSnapshot', {});
+  }
+
   /**
    * Runs a healing scan on this cluster node and resolves with its result.
    * The scan covers the manifests this node coordinated. The request joins
@@ -2265,11 +2463,15 @@ export class ObjectStorage {
       resourcePublicationHoldRequiresExternalDrain:
         resourceFencing.publicationHoldRequiresExternalDrain,
       resourceFencingReason: resourceFencing.reason,
+      ...(stats.stampClockAheadOfWallMs === undefined
+        ? {}
+        : { stampClockAheadOfWallMs: stats.stampClockAheadOfWallMs }),
     };
   }
 
   public async getMetrics(): Promise<IObjectStorageMetrics> {
     const health = await this.getHealth();
+    const stats = health.running ? await this.getStorageStats() : undefined;
     const clusterEnabled = health.cluster.enabled;
     return {
       bucketCount: health.bucketCount,
@@ -2303,6 +2505,15 @@ export class ObjectStorage {
         '# HELP objectstorage_cluster_enabled Cluster mode enabled.',
         '# TYPE objectstorage_cluster_enabled gauge',
         `objectstorage_cluster_enabled ${clusterEnabled ? 1 : 0}`,
+        '# HELP objectstorage_noncurrent_versions_total Noncurrent object versions.',
+        '# TYPE objectstorage_noncurrent_versions_total gauge',
+        `objectstorage_noncurrent_versions_total ${stats?.totalNoncurrentVersionCount ?? 0}`,
+        '# HELP objectstorage_noncurrent_bytes_total Payload bytes of noncurrent object versions.',
+        '# TYPE objectstorage_noncurrent_bytes_total gauge',
+        `objectstorage_noncurrent_bytes_total ${stats?.totalNoncurrentBytes ?? 0}`,
+        '# HELP objectstorage_delete_markers_total Delete markers, current and noncurrent.',
+        '# TYPE objectstorage_delete_markers_total gauge',
+        `objectstorage_delete_markers_total ${stats?.totalDeleteMarkerCount ?? 0}`,
       ].join('\n'),
     };
   }
@@ -2332,6 +2543,10 @@ export class ObjectStorage {
       // Startup keeps its own rejection; stop joins only its resource ownership.
       await Promise.allSettled([this.startPromise]);
     }
+    if (this.dashboard) {
+      await this.dashboard.stop();
+      this.dashboard = undefined;
+    }
     if (this.bridgeOwned) {
       if (this.running) {
         this.completedStopResult ??= await this.bridge.sendCommand('stop', {});