@lossless.org/objectstorage 10.7.1 → 11.0.0

package/third-party-notices.md

@@ -109,7 +109,7 @@ root, run:
 
 ```sh
 node scripts/generate-native-notices.mjs
-pnpm exec tstest test/test.notices.node.ts --verbose --logfile
+pnpm exec tsrepo test test/test.notices.node.ts --verbose --logfile
 ```
 
 The generator uses Cargo.lock and checksum-pinned local or Git license sources

package/third_party_licenses/inventory.json

@@ -2116,7 +2116,7 @@
     "zerocopy-derive@0.8.47": "third_party_licenses/zerocopy-derive-0.8.47-license.txt"
   },
   "files": {
-    ".smartconfig.json": "f6e76a718926bfb661e004f49ef0606cdecb48f773f74e2c38e224e29116dcfb",
+    ".smartconfig.json": "36557e7545b45c38432fa291e11551c20cc0e54fdc6436a0797429e32d1e4d8d",
     "license.md": "bd1cbee3bf4a1ec7f4bb842e315cb0c296744e93050493607cb32f1570f1e878",
     "rust-toolchain.toml": "a721b38eb6b02c7b6cfba956549c18282a3cd28347954021fc1e508c7b27d15b",
     "rust/Cargo.lock": "4b1fb196bca6bec6c326dd822db4d5ca5eb0740fea0544100f9e9d8984187762",
@@ -2125,7 +2125,7 @@
     "rust/about.toml": "dc5dbae4e4b746e12c3a810cdc2473c380e7b9b3c0ecdd095d02693dbb8876b2",
     "rust/notices.sources.json": "7c98fe57f606700bb8372a890f8e49a1240385224680ec0877c3c623ab9cf598",
     "scripts/generate-native-notices.mjs": "e654c52e292778ca72f09e96420b18a49271a7c6612cdd2e2f4dbcd01dec1bcb",
-    "third-party-notices.md": "bf09c39634523d9794862464eaadcf911ad5373709db40a41c811fcff4f2cd59",
+    "third-party-notices.md": "7e733804670e866e563b999ea600ddff72b0f8dee63552f223516ed88dc9afdd",
     "third_party_licenses/aho-corasick-1.1.4-license.txt": "0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f",
     "third_party_licenses/base64-0.22.1/LICENSE-MIT": "0dd882e53de11566d50f8e8e2d5a651bcf3fabee4987d70f306233cf39094ba7",
     "third_party_licenses/cargo-dependencies.html": "8a007780ac193d1fc51ebe8ef7d302dd39e2426a887219049c4ac392786fbed0",

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@lossless.org/objectstorage',
-  version: '10.7.1',
+  version: '11.0.0',
   description: 'A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.'
 }

package/ts/index.ts

@@ -62,6 +62,11 @@ export interface ILimitsConfig {
 export interface IMultipartConfig {
   expirationDays?: number;
   cleanupIntervalMinutes?: number;
+  /**
+   * Open multipart uploads that the prefix credentials of one scope (bucket
+   * and prefix) may hold together (default 100). Counted in memory.
+   */
+  maxUploadsPerPrefixScope?: number;
 }
 
 /**
@@ -118,6 +123,11 @@ export interface IStorageConfig {
 export interface IErasureConfig {
   dataShards?: number;
   parityShards?: number;
+  /**
+   * Bytes per erasure-coded chunk, between 1 and 64 MiB (67,108,864 bytes);
+   * startup refuses a value outside that range, because every manifest
+   * refuses a larger chunk. Default 4,194,304 (4 MiB).
+   */
   chunkSizeBytes?: number;
 }
 
@@ -126,6 +136,13 @@ export interface IErasureConfig {
  */
 export interface IDriveConfig {
   paths: string[];
+  /**
+   * Accept drive roots that share a device (`st_dev`). Each drive root counts
+   * as one failure domain, so by default startup refuses two drive roots on
+   * one device. Set this only for dev and test clusters whose drive roots are
+   * directories on one filesystem. Default false.
+   */
+  allowSharedDevice?: boolean;
 }
 
 /**
@@ -140,6 +157,16 @@ export interface IClusterConfig {
   drives?: IDriveConfig;
   heartbeatIntervalMs?: number;
   heartbeatTimeoutMs?: number;
+  /**
+   * Longest a single GET may run, in milliseconds, from its start to its last
+   * byte, including the client's transfer time. A longer GET is aborted.
+   * Superseded and deleted object generations keep their shards for this
+   * deadline plus the smaller of the deadline and one hour, then cleanup frees
+   * them. Must be the same on every node: a mismatch can abort streams but
+   * never serves wrong bytes. Default 21,600,000 (6 hours); allowed from
+   * 60,000 (1 minute) to 604,800,000 (7 days).
+   */
+  getDeadlineMs?: number;
 }
 
 /**
@@ -341,6 +368,35 @@ export interface IBucketTenantMetadata {
   retention?: IBucketRetentionReceipt;
 }
 
+export interface IPrefixCredentialInput {
+  bucketName: string;
+  /** Unreserved characters (RFC 3986) and `/`; ends with `/`, no leading `/`. */
+  prefix: string;
+  /** First node: lifetime in seconds, 1..43200. */
+  ttlSeconds?: number;
+  /** Fan-out to further nodes: the first node's expiresAt (epoch ms). */
+  expiresAt?: number;
+  /** Supply both together with expiresAt to install an existing credential on another node. */
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  region?: string;
+}
+
+export interface IPrefixCredentialMetadata {
+  accessKeyId: string;
+  bucketName: string;
+  prefix: string;
+  region: string;
+  /** Epoch ms. */
+  createdAt: number;
+  /** Epoch ms; requests are refused from this instant and admitted ones end at it. */
+  expiresAt: number;
+}
+
+export interface IPrefixCredential extends IPrefixCredentialMetadata {
+  secretAccessKey: string;
+}
+
 export interface IBucketRetentionPolicyInput {
   mode: 'compliance';
   policyId: string;
@@ -998,6 +1054,10 @@ export interface IObjectStorageHealth {
     enabled: boolean;
     credentialCount: number;
     tenantCredentialCount: number;
+    prefixCredentialCount: number;
+    prefixCredentialTombstoneCount: number;
+    /** False when the prefix credential file is unreadable: every prefix credential is refused. */
+    prefixCredentialStoreAvailable: boolean;
   };
   bucketCount: number;
   objectCount: number;
@@ -1062,6 +1122,55 @@ const normalizeResourceFenceError = (errorArg: unknown): Error => {
   );
 };
 
+export const objectStorageCredentialErrorCodes = [
+  'ECREDENTIAL_AUTH_DISABLED',
+  'ECREDENTIAL_INVALID_PREFIX',
+  'ECREDENTIAL_INVALID_TTL',
+  'ECREDENTIAL_INVALID_EXPIRY',
+  'ECREDENTIAL_INVALID_KEY',
+  'ECREDENTIAL_NO_SUCH_BUCKET',
+  'ECREDENTIAL_BUCKET_UNAVAILABLE',
+  'ECREDENTIAL_CONFLICT',
+  'ECREDENTIAL_REVOKED',
+  'ECREDENTIAL_NOT_PREFIX',
+  'ECREDENTIAL_LIMIT',
+  'ECREDENTIAL_PERSISTENCE',
+  'ECREDENTIAL_STORE_UNAVAILABLE',
+] as const;
+
+export type TObjectStorageCredentialErrorCode =
+  (typeof objectStorageCredentialErrorCodes)[number];
+
+export class ObjectStorageCredentialError extends Error {
+  public readonly code: TObjectStorageCredentialErrorCode;
+
+  public constructor(
+    codeArg: TObjectStorageCredentialErrorCode,
+    messageArg: string,
+    causeArg?: unknown,
+  ) {
+    super(`${codeArg}: ${messageArg}`, { cause: causeArg });
+    this.name = 'ObjectStorageCredentialError';
+    this.code = codeArg;
+  }
+}
+
+const normalizeCredentialError = (errorArg: unknown): Error => {
+  const error = errorArg instanceof Error ? errorArg : new Error(String(errorArg));
+  const match = /(ECREDENTIAL_[A-Z_]+):\s*(.*)$/s.exec(error.message);
+  if (
+    !match ||
+    !objectStorageCredentialErrorCodes.includes(match[1] as TObjectStorageCredentialErrorCode)
+  ) {
+    return error;
+  }
+  return new ObjectStorageCredentialError(
+    match[1] as TObjectStorageCredentialErrorCode,
+    match[2],
+    error,
+  );
+};
+
 export interface IObjectStorageMetrics {
   bucketCount: number;
   objectCount: number;
@@ -1119,6 +1228,8 @@ export interface IClusterRepairHealth {
   lastDurationMs?: number;
   shardsChecked?: number;
   shardsHealed?: number;
+  /** Pending shards (writes that failed at publication) the last scan left pending. */
+  shardsPending?: number;
   failed?: number;
   lastError?: string;
 }
@@ -1140,6 +1251,35 @@ export interface IClusterHealth {
   repairs?: IClusterRepairHealth;
 }
 
+/**
+ * The result of one cluster healing scan, as `ObjectStorage.runHealingScan()`
+ * returns it. Times are milliseconds since the Unix epoch.
+ */
+export interface IClusterHealingScanResult {
+  /** When the scan started; always after the request arrived. */
+  startedAt: number;
+  completedAt: number;
+  durationMs: number;
+  /** Shard placements checked at their drives (placements on offline nodes are not). */
+  shardsChecked: number;
+  /** Shards written: missing placements rebuilt and pending shards completed. */
+  shardsHealed: number;
+  /** Pending shards (writes that failed at publication) this scan left pending. */
+  shardsPending: number;
+  /** Failed checks, rebuilds, writes and manifest stores. */
+  failed: number;
+}
+
+export interface IRunHealingScanOptions {
+  /**
+   * How long to wait for the scan, in milliseconds. Defaults to the bridge's
+   * request timeout (300,000). The engine cannot cancel a scan: one that
+   * outlasts this keeps running, its result is discarded, and a later call
+   * joins the next scan that starts after it.
+   */
+  timeoutMs?: number;
+}
+
 /**
  * Default configuration values
  */
@@ -1185,6 +1325,7 @@ const DEFAULT_CONFIG: IObjectStorageConfig = {
   multipart: {
     expirationDays: 7,
     cleanupIntervalMinutes: 60,
+    maxUploadsPerPrefixScope: 100,
   },
 };
 
@@ -1313,6 +1454,10 @@ function createAccessKeyId(): string {
   return `SS${plugins.crypto.randomBytes(10).toString('hex').toUpperCase()}`;
 }
 
+function createPrefixAccessKeyId(): string {
+  return `SP${plugins.crypto.randomBytes(10).toString('hex').toUpperCase()}`;
+}
+
 function createSecretAccessKey(): string {
   return plugins.crypto.randomBytes(32).toString('hex');
 }
@@ -1441,7 +1586,29 @@ type TRustStorageCommands = {
   listBucketSummaries: { params: {}; result: IBucketSummary[] };
   listCredentials: { params: {}; result: IStorageCredentialMetadata[] };
   replaceCredentials: { params: { credentials: IStorageCredential[] }; result: {} };
+  createPrefixCredential: {
+    params: {
+      bucketName: string;
+      prefix: string;
+      accessKeyId: string;
+      secretAccessKey: string;
+      region?: string;
+      ttlSeconds?: number;
+      expiresAt?: number;
+    };
+    result: IPrefixCredentialMetadata;
+  };
+  revokePrefixCredential: { params: { accessKeyId: string }; result: { revoked: boolean } };
+  listPrefixCredentials: {
+    params: { bucketName?: string };
+    result: IPrefixCredentialMetadata[];
+  };
+  getPrefixCredentialStatus: {
+    params: {};
+    result: { available: boolean; credentialCount: number; tombstoneCount: number };
+  };
   getClusterHealth: { params: {}; result: IClusterHealth };
+  runHealingScan: { params: {}; result: IClusterHealingScanResult };
 };
 
 /**
@@ -1730,7 +1897,7 @@ export class ObjectStorage {
       });
       return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1745,7 +1912,7 @@ export class ObjectStorage {
       });
       return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1754,7 +1921,7 @@ export class ObjectStorage {
     try {
       await this.bridge.sendCommand('deleteBucketTenant', tenantArg);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1771,7 +1938,7 @@ export class ObjectStorage {
       });
       return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1790,6 +1957,95 @@ export class ObjectStorage {
     return this.buildBucketTenantDescriptor(credential, optionsArg.bucketName);
   }
 
+  /**
+   * Creates a credential confined to one key prefix of one bucket until its
+   * expiry. The first node takes `ttlSeconds`; further nodes of a fan-out take
+   * the first result's `accessKeyId`, `secretAccessKey` and `expiresAt`. The
+   * secret is returned only here and never kept by this instance.
+   */
+  public async createPrefixCredential(
+    inputArg: IPrefixCredentialInput,
+  ): Promise<IPrefixCredential> {
+    const hasTtl = inputArg.ttlSeconds !== undefined;
+    const hasExpiresAt = inputArg.expiresAt !== undefined;
+    if (hasTtl === hasExpiresAt) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_TTL',
+        'Exactly one of ttlSeconds and expiresAt is required.',
+      );
+    }
+    if (hasTtl && (!Number.isSafeInteger(inputArg.ttlSeconds) || inputArg.ttlSeconds! < 1)) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_TTL',
+        'ttlSeconds must be an integer from 1 to 43200.',
+      );
+    }
+    if (hasExpiresAt && !Number.isSafeInteger(inputArg.expiresAt)) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_EXPIRY',
+        'expiresAt must be an integer epoch millisecond.',
+      );
+    }
+    const hasAccessKeyId = inputArg.accessKeyId !== undefined;
+    const hasSecretAccessKey = inputArg.secretAccessKey !== undefined;
+    if (hasAccessKeyId !== hasSecretAccessKey) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_KEY',
+        'accessKeyId and secretAccessKey must be supplied together.',
+      );
+    }
+    if (hasAccessKeyId && !hasExpiresAt) {
+      throw new ObjectStorageCredentialError(
+        'ECREDENTIAL_INVALID_KEY',
+        'Key material is accepted only with expiresAt, to install an existing credential.',
+      );
+    }
+    const secretAccessKey = inputArg.secretAccessKey ?? createSecretAccessKey();
+    try {
+      const metadata = await this.bridge.sendCommand('createPrefixCredential', {
+        bucketName: inputArg.bucketName,
+        prefix: inputArg.prefix,
+        accessKeyId: inputArg.accessKeyId ?? createPrefixAccessKeyId(),
+        secretAccessKey,
+        region: inputArg.region || this.config.server.region,
+        ...(hasTtl ? { ttlSeconds: inputArg.ttlSeconds } : { expiresAt: inputArg.expiresAt }),
+      });
+      return { ...metadata, secretAccessKey };
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
+  }
+
+  /**
+   * Revokes a prefix credential on this node. An unknown ID is recorded as
+   * revoked too, so a later create for it is refused; `revoked` is true only
+   * when a live credential was removed.
+   */
+  public async revokePrefixCredential(optionsArg: {
+    accessKeyId: string;
+  }): Promise<{ revoked: boolean }> {
+    try {
+      return await this.bridge.sendCommand('revokePrefixCredential', {
+        accessKeyId: optionsArg.accessKeyId,
+      });
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
+  }
+
+  /** Lists this node's live prefix credentials, without secrets. */
+  public async listPrefixCredentials(
+    optionsArg: { bucketName?: string } = {},
+  ): Promise<IPrefixCredentialMetadata[]> {
+    try {
+      return await this.bridge.sendCommand('listPrefixCredentials', {
+        ...(optionsArg.bucketName !== undefined ? { bucketName: optionsArg.bucketName } : {}),
+      });
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
+  }
+
   public async getBucketRetentionCapability(): Promise<IBucketRetentionCapability> {
     return this.bridge.sendCommand('getBucketRetentionCapability', {});
   }
@@ -1864,7 +2120,7 @@ export class ObjectStorage {
     try {
       return await this.bridge.sendCommand('deleteBucket', optionsArg);
     } catch (error) {
-      throw normalizeResourceFenceError(error);
+      throw normalizeCredentialError(normalizeResourceFenceError(error));
     }
   }
 
@@ -1914,7 +2170,11 @@ export class ObjectStorage {
    * if this override introduces ownership or credential-material drift.
    */
   public async replaceCredentials(credentials: IStorageCredential[]): Promise<void> {
-    await this.bridge.sendCommand('replaceCredentials', { credentials });
+    try {
+      await this.bridge.sendCommand('replaceCredentials', { credentials });
+    } catch (error) {
+      throw normalizeCredentialError(error);
+    }
     this.config.auth.credentials = credentials.map((credential) => ({ ...credential }));
   }
 
@@ -1922,6 +2182,27 @@ export class ObjectStorage {
     return this.bridge.sendCommand('getClusterHealth', {});
   }
 
+  /**
+   * Runs a healing scan on this cluster node and resolves with its result.
+   * The scan covers the manifests this node coordinated. The request joins
+   * the next scan that starts after it arrives, never one already running,
+   * and requests that arrive together share one scan. It rejects in
+   * standalone mode, when the scan fails (for example without a majority of
+   * the cluster, or when a bucket's manifests cannot be listed), when 16
+   * requests already wait for a scan to start, and when the engine shuts down
+   * before the scan completes. The engine answers other management calls
+   * while the scan runs.
+   */
+  public async runHealingScan(
+    optionsArg: IRunHealingScanOptions = {},
+  ): Promise<IClusterHealingScanResult> {
+    return this.bridge.sendCommand(
+      'runHealingScan',
+      {},
+      optionsArg.timeoutMs === undefined ? undefined : { timeoutMs: optionsArg.timeoutMs },
+    );
+  }
+
   public async getHealth(): Promise<IObjectStorageHealth> {
     if (!this.running) {
       return {
@@ -1933,6 +2214,9 @@ export class ObjectStorage {
           enabled: this.config.auth.enabled,
           credentialCount: this.config.auth.credentials.length,
           tenantCredentialCount: 0,
+          prefixCredentialCount: 0,
+          prefixCredentialTombstoneCount: 0,
+          prefixCredentialStoreAvailable: false,
         },
         bucketCount: 0,
         objectCount: 0,
@@ -1947,13 +2231,15 @@ export class ObjectStorage {
       };
     }
 
-    const [stats, credentials, tenants, cluster, resourceFencing] = await Promise.all([
-      this.getStorageStats(),
-      this.listCredentials(),
-      this.config.auth.enabled ? this.listBucketTenants() : Promise.resolve([]),
-      this.getClusterHealth(),
-      this.getResourceFencingCapability(),
-    ]);
+    const [stats, credentials, tenants, prefixCredentials, cluster, resourceFencing] =
+      await Promise.all([
+        this.getStorageStats(),
+        this.listCredentials(),
+        this.config.auth.enabled ? this.listBucketTenants() : Promise.resolve([]),
+        this.bridge.sendCommand('getPrefixCredentialStatus', {}),
+        this.getClusterHealth(),
+        this.getResourceFencingCapability(),
+      ]);
     return {
       ok: true,
       running: true,
@@ -1963,6 +2249,9 @@ export class ObjectStorage {
         enabled: this.config.auth.enabled,
         credentialCount: credentials.length,
         tenantCredentialCount: tenants.length,
+        prefixCredentialCount: prefixCredentials.credentialCount,
+        prefixCredentialTombstoneCount: prefixCredentials.tombstoneCount,
+        prefixCredentialStoreAvailable: prefixCredentials.available,
       },
       bucketCount: stats.bucketCount,
       objectCount: stats.totalObjectCount,
@@ -2002,6 +2291,15 @@ export class ObjectStorage {
         '# HELP objectstorage_tenant_credentials_total Scoped bucket tenant credential count.',
         '# TYPE objectstorage_tenant_credentials_total gauge',
         `objectstorage_tenant_credentials_total ${health.auth.tenantCredentialCount}`,
+        '# HELP objectstorage_prefix_credentials_total Live prefix credential count.',
+        '# TYPE objectstorage_prefix_credentials_total gauge',
+        `objectstorage_prefix_credentials_total ${health.auth.prefixCredentialCount}`,
+        '# HELP objectstorage_prefix_credential_tombstones_total Unexpired prefix credential revocations.',
+        '# TYPE objectstorage_prefix_credential_tombstones_total gauge',
+        `objectstorage_prefix_credential_tombstones_total ${health.auth.prefixCredentialTombstoneCount}`,
+        '# HELP objectstorage_prefix_credential_store_available Prefix credential store readable.',
+        '# TYPE objectstorage_prefix_credential_store_available gauge',
+        `objectstorage_prefix_credential_store_available ${health.auth.prefixCredentialStoreAvailable ? 1 : 0}`,
         '# HELP objectstorage_cluster_enabled Cluster mode enabled.',
         '# TYPE objectstorage_cluster_enabled gauge',
         `objectstorage_cluster_enabled ${clusterEnabled ? 1 : 0}`,