@lossless.org/client 2.2.0 → 2.4.0

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@lossless.org/client',
-  version: '2.2.0',
+  version: '2.4.0',
   description: 'One typed client for NoSQLDB, MongoDB, SQLDB, MariaDB, ClickHouse and S3 object storage.'
 }

package/ts/nosqldb/classes.doc.ts

@@ -4176,7 +4176,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   declare public manager: TManager;
 
   /**
-   * Helper to get collection with fallback to static for Deno compatibility
+   * Helper to get the collection from the instance, falling back to the model's static accessor
    */
   private getCollectionSafe(): SmartdataCollection<any> {
     return this.collection || (this.constructor as any).collection;
@@ -6643,14 +6643,14 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
 
   /**
    * an array of saveable properties of ALL doc
-   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
+   * Note: Set by decorators on prototype - NOT declared as instance property, so no class field shadows it
    * Declared with definite assignment assertion to satisfy TypeScript without creating instance property
    */
   declare globalSaveableProperties: string[];
 
   /**
    * unique indexes
-   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
+   * Note: Set by decorators on prototype - NOT declared as instance property, so no class field shadows it
    */
   declare uniqueIndexes: string[];
 
@@ -6659,7 +6659,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
 
   /**
    * regular indexes with their options
-   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
+   * Note: Set by decorators on prototype - NOT declared as instance property, so no class field shadows it
    */
   declare regularIndexes: Array<{field: string, options: IIndexOptions}>;
 
@@ -6674,7 +6674,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
 
   /**
    * an array of saveable properties of a specific doc
-   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
+   * Note: Set by decorators on prototype - NOT declared as instance property, so no class field shadows it
    */
   declare saveableProperties: string[];
 

package/ts/objectstorage/classes.bucket.ts

@@ -18,6 +18,10 @@ import {
   purgeExactPath as purgeExactPathOperation,
 } from './internal.exactpathpurge.operations.js';
 import { beginBasicBucketOperation } from './internal.exactupload.capability.js';
+import {
+  copyObjectMultipart,
+  multipartCopyThresholdBytes,
+} from './internal.multipartcopy.js';
 import {
   getProviderStatus,
   headBucketExists,
@@ -25,6 +29,7 @@ import {
   runAccountRequest,
 } from './internal.bucketrequests.js';
 import { ObjectRangeError } from './classes.objectrangeerror.js';
+import { ObjectStreamLengthError } from './classes.objectstreamlengtherror.js';
 
 export interface IListObjectKeysPageOptions {
   prefix?: string;
@@ -253,6 +258,21 @@ const readServedByteRange = (
   return { range: { start, end }, size };
 };
 
+/**
+ * Answers whether the S3 client can measure an upload source of undeclared
+ * length itself: it sizes a file stream from its path or descriptor
+ * (`calculateBodyLength` of `@smithy/core`), and no other stream. With
+ * request checksums on, an unmeasured stream body reaches the wire with an
+ * invalid `x-amz-decoded-content-length: undefined` header.
+ */
+const isMeasurableFileStream = (sourceArg: plugins.stream.Readable | ReadableStream): boolean => {
+  if (!(sourceArg instanceof plugins.fs.ReadStream)) {
+    return false;
+  }
+  // `fd` is set at runtime but not declared by the Node typings.
+  return sourceArg.path != null || typeof (sourceArg as { fd?: unknown }).fd === 'number';
+};
+
 const destroyReadableAndWaitForSettlement = async (
   readableArg: plugins.stream.Readable,
 ): Promise<void> => {
@@ -804,7 +824,11 @@ export class Bucket {
   }
 
   /**
-   * store file as stream
+   * Stores a stream with one PutObject, which must declare the body's
+   * length: pass `contentLength` unless the source is a file stream
+   * (`fs.ReadStream`), which the S3 client measures itself. Any other stream
+   * of unknown length is refused with `ObjectStreamLengthError` before any
+   * request; it uploads with `fastPutStreamExact()` and `maxContentLength`.
    */
   public async fastPutStream(
     optionsArg: interfaces.IFastPutStreamOptions,
@@ -818,6 +842,12 @@ export class Bucket {
     ) {
       throw new Error('contentLength must be a non-negative safe integer');
     }
+    if (optionsArg.contentLength === undefined && !isMeasurableFileStream(optionsArg.readableStream)) {
+      throw new ObjectStreamLengthError(
+        `fastPutStream for '${optionsArg.path}' needs contentLength: one PutObject cannot upload a ` +
+        `stream of unknown length. Pass contentLength, or upload with fastPutStreamExact and maxContentLength.`,
+      );
+    }
     const operation = beginBasicBucketOperation(this, optionsArg.signal);
     let uploadStream: plugins.stream.Readable | undefined;
     let sourceOwnershipStarted = false;
@@ -930,8 +960,18 @@ export class Bucket {
     return purgeExactPathOperation(this, optionsArg);
   }
 
+  /**
+   * Copies an object within the storage account. Sources up to 5 GiB are
+   * copied with one CopyObject; larger sources, which CopyObject refuses, are
+   * copied part by part with UploadPartCopy, each part conditional on the
+   * source ETag read first.
+   * A failed or aborted part copy aborts the multipart upload. Object tags,
+   * storage class, server-side encryption settings and a website redirect are
+   * not carried by a part-by-part copy.
+   */
   public async fastCopy(optionsArg: interfaces.IFastCopyOptions): Promise<void> {
     const operation = beginBasicBucketOperation(this, optionsArg.signal);
+    const deferredCleanups: Promise<void>[] = [];
     try {
       const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
 
@@ -951,10 +991,41 @@ export class Bucket {
 
       // Define the copy operation
       const copySource = `${operation.bucketName}/${optionsArg.sourcePath}`;
+      const targetKey = optionsArg.destinationPath || optionsArg.sourcePath;
+      const sourceSize = currentObjInfo.ContentLength;
+      if (typeof sourceSize === 'number' && sourceSize > multipartCopyThresholdBytes) {
+        if (typeof currentObjInfo.ETag !== 'string' || currentObjInfo.ETag.length === 0) {
+          throw new Error('HeadObject returned no ETag for the copy source.');
+        }
+        await copyObjectMultipart(operation, {
+          copySource,
+          sourceSize,
+          sourceEtag: currentObjInfo.ETag,
+          targetBucketName,
+          targetKey,
+          // Mirrors CopyObject: COPY keeps the source metadata and content
+          // headers, REPLACE sets only the given metadata.
+          objectProperties: optionsArg.deleteExistingNativeMetadata
+            ? { Metadata: newNativeMetadata }
+            : {
+                Metadata: currentObjInfo.Metadata,
+                ContentType: currentObjInfo.ContentType,
+                CacheControl: currentObjInfo.CacheControl,
+                ContentDisposition: currentObjInfo.ContentDisposition,
+                ContentEncoding: currentObjInfo.ContentEncoding,
+                ContentLanguage: currentObjInfo.ContentLanguage,
+                Expires: currentObjInfo.Expires,
+              },
+          onDeferredCleanup: (settlementArg) => {
+            deferredCleanups.push(settlementArg);
+          },
+        });
+        return;
+      }
       const command = new plugins.s3.CopyObjectCommand({
         Bucket: targetBucketName,
         CopySource: copySource,
-        Key: optionsArg.destinationPath || optionsArg.sourcePath,
+        Key: targetKey,
         Metadata: newNativeMetadata,
         MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY',
       });
@@ -962,7 +1033,11 @@ export class Bucket {
         () => operation.client.send(command, { abortSignal: operation.signal }),
       );
     } finally {
-      operation.finish();
+      if (deferredCleanups.length > 0) {
+        void Promise.allSettled(deferredCleanups).then(() => operation.finish());
+      } else {
+        operation.finish();
+      }
     }
   }
 

package/ts/objectstorage/classes.objectstreamlengtherror.ts

@@ -0,0 +1,15 @@
+/**
+ * The refusal of a single-request stream upload whose length is not known.
+ * One PutObject must declare the length of its body, and the S3 client
+ * measures only a file stream (`fs.ReadStream`) itself, so any other stream
+ * needs `contentLength`; a stream of unknown length uploads with
+ * `fastPutStreamExact()` and `maxContentLength`. It is thrown before any
+ * request is sent and before the source is read; the caller still owns the
+ * source.
+ */
+export class ObjectStreamLengthError extends Error {
+  constructor(messageArg: string) {
+    super(messageArg);
+    this.name = 'ObjectStreamLengthError';
+  }
+}

package/ts/objectstorage/classes.smartbucket.ts

@@ -3,7 +3,7 @@
 import * as plugins from './plugins.js';
 import { Bucket } from './classes.bucket.js';
 import { normalizeStorageDescriptor } from './helpers.js';
-import { headBucketExists } from './internal.bucketrequests.js';
+import { headBucketExists, isMissingBucketError } from './internal.bucketrequests.js';
 import {
   bindRequestDeadline,
   createBoundedClientConfig,
@@ -28,7 +28,7 @@ import type {
 
 type TSmartBucketLifecycleState = 'open' | 'closing' | 'closed';
 type TSmartBucketReadinessAbortSource = 'caller' | 'deadline';
-type TSmartBucketReadinessSendOutcome = 'ready' | 'aborted' | 'unavailable';
+type TSmartBucketReadinessSendOutcome = 'ready' | 'aborted' | 'bucket-missing' | 'unavailable';
 
 const readinessDefaultTimeoutMs = 2_000;
 const readinessMaximumTimeoutMs = 30_000;
@@ -314,8 +314,8 @@ export class SmartBucket {
       }
       return this.createReadinessFailure('unavailable', startedAt);
     }
-    if (sendOutcome === 'unavailable') {
-      return this.createReadinessFailure('unavailable', startedAt);
+    if (sendOutcome === 'bucket-missing' || sendOutcome === 'unavailable') {
+      return this.createReadinessFailure(sendOutcome, startedAt);
     }
 
     return {
@@ -432,7 +432,7 @@ export class SmartBucket {
         );
         void providerPromise.then(
           () => settle('ready'),
-          () => settle('unavailable'),
+          (errorArg: unknown) => settle(isMissingBucketError(errorArg) ? 'bucket-missing' : 'unavailable'),
         );
       } catch {
         settle('unavailable');

package/ts/objectstorage/index.ts

@@ -3,6 +3,7 @@ export * from './classes.bucket.js';
 export * from './classes.exactuploaderror.js';
 export * from './classes.exactpathpurgeerror.js';
 export * from './classes.objectrangeerror.js';
+export * from './classes.objectstreamlengtherror.js';
 export * from './classes.requestdeadlineerror.js';
 export * from './classes.directory.js';
 export * from './classes.file.js';

package/ts/objectstorage/interfaces.ts

@@ -177,12 +177,17 @@ export interface ISmartBucketReadinessOptions {
 
 /**
  * Sanitized reasons why a SmartBucket readiness probe is unavailable.
+ * `bucket-missing` means the provider answered the probe definitively that
+ * the bucket does not exist (`NoSuchBucket`, or HTTP 404). A denied probe
+ * (HTTP 403) says nothing about existence and stays `unavailable`, as do
+ * other provider and transport failures.
  */
 export type TSmartBucketReadinessFailureReason =
   | 'closing'
   | 'closed'
   | 'aborted'
   | 'timeout'
+  | 'bucket-missing'
   | 'unavailable';
 
 /**

package/ts/objectstorage/internal.bucketrequests.ts

@@ -49,6 +49,16 @@ export const isMissingObjectError = (errorArg: unknown): boolean => {
   return name === 'NoSuchKey' || name === 'NotFound' || httpStatusCode === 404;
 };
 
+/**
+ * Answers whether a provider error says the addressed bucket does not exist:
+ * `NoSuchBucket`, HeadBucket's body-less `NotFound`, or any other HTTP 404.
+ * A denied request (HTTP 403) says nothing about existence and is not one.
+ */
+export const isMissingBucketError = (errorArg: unknown): boolean => {
+  const { name, httpStatusCode } = getProviderStatus(errorArg);
+  return name === 'NoSuchBucket' || name === 'NotFound' || httpStatusCode === 404;
+};
+
 /**
  * Answers whether the bucket exists with one HeadBucket request, which a
  * bucket-scoped credential may send for its own bucket. A missing bucket
@@ -68,10 +78,10 @@ export const headBucketExists = async (
     );
     return true;
   } catch (error) {
-    const { name, httpStatusCode } = getProviderStatus(error);
-    if (httpStatusCode === 404 || name === 'NotFound' || name === 'NoSuchBucket') {
+    if (isMissingBucketError(error)) {
       return false;
     }
+    const { name, httpStatusCode } = getProviderStatus(error);
     if (httpStatusCode === 403 || name === 'Forbidden' || name === 'AccessDenied') {
       throw new Error(
         `Access to bucket '${bucketNameArg}' was denied (HTTP 403): the credentials may not ` +

package/ts/objectstorage/internal.exactupload.operations.ts

@@ -47,7 +47,7 @@ interface IIndependentCleanupResult {
   settlement: Promise<void>;
 }
 
-interface IMultipartAbortResult {
+export interface IMultipartAbortResult {
   cleanup: TExactUploadMultipartCleanup;
   timedOut: boolean;
   settlement: Promise<void>;
@@ -1266,10 +1266,11 @@ export const probeExactUploadCapability = async (
   }
 };
 
-const abortMultipartUpload = async (
+export const abortMultipartUpload = async (
   operationArg: IExactUploadOperation,
   keyArg: string,
   uploadIdArg: string,
+  bucketNameArg = operationArg.bucketName,
 ): Promise<IMultipartAbortResult> => {
   let reconciliationResult: 'aborted' | 'absent' | undefined;
   const cleanupResult = await runIndependentCleanup(
@@ -1277,7 +1278,7 @@ const abortMultipartUpload = async (
       async (signalArg) => {
         reconciliationResult = await reconcileMultipartUploadAbort(
           operationArg.client,
-          operationArg.bucketName,
+          bucketNameArg,
           keyArg,
           uploadIdArg,
           signalArg,

package/ts/objectstorage/internal.multipartcopy.ts

@@ -0,0 +1,169 @@
+import * as plugins from './plugins.js';
+import type { IBasicBucketOperation } from './internal.exactupload.capability.js';
+import { abortMultipartUpload } from './internal.exactupload.operations.js';
+
+const MiB = 1024 * 1024;
+const GiB = 1024 * MiB;
+
+/**
+ * Sources larger than this are copied with UploadPartCopy. It is the largest
+ * object one CopyObject may copy, so every copy that one CopyObject can do
+ * stays one. A capped MinIO (2 CPUs, 2 GiB) answered a single CopyObject after
+ * about 3 s per GiB, inside the default 60 s request timeout at this size.
+ */
+export const multipartCopyThresholdBytes = 5 * GiB;
+/** One 256 MiB UploadPartCopy answered within 0.9 s on the same MinIO. */
+export const multipartCopyPartSizeBytes = 256 * MiB;
+const maximumCopyPartSizeBytes = 5 * GiB;
+const maximumCopyPartCount = 10_000;
+const maximumCopyObjectSizeBytes = 5 * 1024 * GiB;
+
+export interface IMultipartCopyPlan {
+  partSize: number;
+  partCount: number;
+}
+
+/**
+ * Grows the part size so the object fits in 10,000 parts and refuses objects
+ * that S3 cannot hold.
+ */
+export const planMultipartCopy = (
+  objectSizeArg: number,
+  partSizeArg = multipartCopyPartSizeBytes,
+): IMultipartCopyPlan => {
+  if (
+    !Number.isSafeInteger(objectSizeArg) ||
+    objectSizeArg <= 0 ||
+    objectSizeArg > maximumCopyObjectSizeBytes
+  ) {
+    throw new RangeError(
+      `A multipart copy needs a source of 1 byte to ${maximumCopyObjectSizeBytes} bytes, not ${objectSizeArg}.`,
+    );
+  }
+  const partSize = Math.max(
+    partSizeArg,
+    Math.ceil(objectSizeArg / maximumCopyPartCount),
+  );
+  if (partSize > maximumCopyPartSizeBytes) {
+    throw new RangeError(
+      `A multipart copy part cannot exceed ${maximumCopyPartSizeBytes} bytes.`,
+    );
+  }
+  return {
+    partSize,
+    partCount: Math.ceil(objectSizeArg / partSize),
+  };
+};
+
+export interface IMultipartCopyOptions {
+  copySource: string;
+  sourceSize: number;
+  /** Every part copy requires the source to still carry this ETag. */
+  sourceEtag: string;
+  targetBucketName: string;
+  targetKey: string;
+  /** Object properties of the new object; a multipart upload copies none itself. */
+  objectProperties: Pick<
+    plugins.s3.CreateMultipartUploadCommandInput,
+    | 'Metadata'
+    | 'ContentType'
+    | 'CacheControl'
+    | 'ContentDisposition'
+    | 'ContentEncoding'
+    | 'ContentLanguage'
+    | 'Expires'
+  >;
+  partSize?: number;
+  /** Receives cleanup that outlived its deadline; the operation must not finish before it settles. */
+  onDeferredCleanup: (settlementArg: Promise<void>) => void;
+}
+
+/**
+ * Copies one object part by part. Any failure or abort aborts the multipart
+ * upload instead of completing it.
+ */
+export const copyObjectMultipart = async (
+  operationArg: IBasicBucketOperation,
+  optionsArg: IMultipartCopyOptions,
+): Promise<void> => {
+  const plan = planMultipartCopy(optionsArg.sourceSize, optionsArg.partSize);
+  const abortUpload = (uploadIdArg: string) =>
+    abortMultipartUpload(
+      operationArg,
+      optionsArg.targetKey,
+      uploadIdArg,
+      optionsArg.targetBucketName,
+    );
+  const createOutput = await operationArg.runProvider(
+    () => operationArg.client.send(
+      new plugins.s3.CreateMultipartUploadCommand({
+        Bucket: optionsArg.targetBucketName,
+        Key: optionsArg.targetKey,
+        ...optionsArg.objectProperties,
+      }),
+      { abortSignal: operationArg.signal },
+    ),
+    {
+      // A create that ignored cancellation still owns an upload.
+      disposeLateResult: async (lateOutputArg) => {
+        if (lateOutputArg.UploadId) {
+          const abortResult = await abortUpload(lateOutputArg.UploadId);
+          await abortResult.settlement;
+        }
+      },
+    },
+  );
+  const uploadId = createOutput.UploadId;
+  if (typeof uploadId !== 'string' || uploadId.length === 0) {
+    throw new Error('CreateMultipartUpload returned no UploadId.');
+  }
+  try {
+    const parts: plugins.s3.CompletedPart[] = [];
+    for (let partNumber = 1; partNumber <= plan.partCount; partNumber++) {
+      const start = (partNumber - 1) * plan.partSize;
+      const end = Math.min(optionsArg.sourceSize, start + plan.partSize) - 1;
+      const partOutput = await operationArg.runProvider(
+        () => operationArg.client.send(
+          new plugins.s3.UploadPartCopyCommand({
+            Bucket: optionsArg.targetBucketName,
+            Key: optionsArg.targetKey,
+            UploadId: uploadId,
+            PartNumber: partNumber,
+            CopySource: optionsArg.copySource,
+            CopySourceRange: `bytes=${start}-${end}`,
+            CopySourceIfMatch: optionsArg.sourceEtag,
+          }),
+          { abortSignal: operationArg.signal },
+        ),
+      );
+      const etag = partOutput.CopyPartResult?.ETag;
+      if (typeof etag !== 'string' || etag.length === 0) {
+        throw new Error('UploadPartCopy returned no ETag.');
+      }
+      parts.push({ PartNumber: partNumber, ETag: etag });
+    }
+    await operationArg.runProvider(
+      () => operationArg.client.send(
+        new plugins.s3.CompleteMultipartUploadCommand({
+          Bucket: optionsArg.targetBucketName,
+          Key: optionsArg.targetKey,
+          UploadId: uploadId,
+          MultipartUpload: { Parts: parts },
+        }),
+        { abortSignal: operationArg.signal },
+      ),
+    );
+  } catch (error) {
+    const abortResult = await abortUpload(uploadId);
+    if (abortResult.cleanup.status === 'abort-failed') {
+      throw new AggregateError(
+        [error, abortResult.cleanup.cause],
+        `The multipart copy failed and its upload '${uploadId}' could not be aborted.`,
+      );
+    }
+    if (abortResult.timedOut) {
+      optionsArg.onDeferredCleanup(abortResult.settlement);
+    }
+    throw error;
+  }
+};

package/ts/objectstorage/plugins.ts

@@ -2,11 +2,12 @@
 
 // node native
 import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
 import * as path from 'node:path';
 import * as stream from 'node:stream';
 import * as nodeUtil from 'node:util';
 
-export { crypto, nodeUtil, path, stream };
+export { crypto, fs, nodeUtil, path, stream };
 
 // @push.rocks scope
 import * as smartmime from '@push.rocks/smartmime';