@opengeni/storage 0.2.68 → 0.2.87

package/src/index.ts

@@ -1,5 +1,7 @@
 import type { Settings } from "@opengeni/config";
 import { RETAINED_OUTPUT_MAX_PAGE_BYTES, type FileAsset } from "@opengeni/contracts";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 import {
   BlobSASPermissions,
   BlobServiceClient,
@@ -23,14 +25,22 @@ import {
   type StorageOptions,
 } from "@google-cloud/storage";
 
+export * from "./bounded-object-read";
+export * from "./bounded-object-write";
+export * from "./object-storage-bounded";
+
 export const MAX_SINGLE_PUT_SIZE_BYTES = 5_000_000_000;
 export const UPLOAD_URL_TTL_SECONDS = 15 * 60;
 export const DOWNLOAD_URL_TTL_SECONDS = 5 * 60;
+const INTERNAL_STREAM_BUFFER_BYTES = 1024 * 1024;
+const INTERNAL_STREAM_CONCURRENCY = 2;
 
 export type ObjectHead = {
   ContentLength?: number;
   ContentType?: string;
   Metadata?: Record<string, string>;
+  /** Opaque provider generation/etag used only for conditional internal reads. */
+  VersionToken?: string;
 };
 
 export type ObjectStorage = {
@@ -61,6 +71,14 @@ export type ObjectStorage = {
   ) => Promise<Uint8Array | null>;
   /** Fetch an object by raw storage key (not a tracked FileAsset). Returns null on 404/missing. */
   getObjectBytes: (key: string) => Promise<{ bytes: Uint8Array; contentType?: string } | null>;
+  /** Provider-versioned raw-key primitives used by bounded immutable adapters. */
+  headObject?: (key: string) => Promise<ObjectHead | null>;
+  getObjectRange?: (args: {
+    key: string;
+    start: number;
+    endInclusive: number;
+    expectedVersionToken: string;
+  }) => Promise<{ bytes: Uint8Array; versionToken: string } | null>;
   /**
    * SERVER-SIDE authenticated direct PUT (no presign + browser fetch). For an
    * in-process upload from a trusted holder of the storage credentials (e.g. the
@@ -79,6 +97,25 @@ export type ObjectStorage = {
     body: Uint8Array;
     sha256?: string | null;
   }) => Promise<void>;
+  /** Atomic create-only raw PUT. Returns false when the key already exists. */
+  putObjectIfAbsent?: (args: {
+    key: string;
+    contentType: string;
+    body: Uint8Array;
+    sha256: string;
+  }) => Promise<boolean>;
+  /**
+   * Atomic create-only raw upload from a bounded asynchronous byte stream.
+   * Providers may buffer a small fixed number of chunks, never the whole body.
+   */
+  putObjectStreamIfAbsent?: (args: {
+    key: string;
+    contentType: string;
+    chunks: AsyncIterable<Uint8Array>;
+    byteSize: number;
+    sha256: string;
+    signal?: AbortSignal;
+  }) => Promise<boolean>;
   /**
    * SERVER-SIDE authenticated delete of a single object by raw storage key.
    * Idempotent: a missing key is a no-op (S3/GCS/Azure delete-by-key does not
@@ -182,6 +219,44 @@ function createS3CompatibleObjectStorage(settings: Settings): ObjectStorage | nu
         }),
       );
     },
+    async putObjectIfAbsent(args) {
+      try {
+        await requestClient.send(
+          new PutObjectCommand({
+            Bucket: settings.objectStorageBucket,
+            Key: args.key,
+            ContentType: args.contentType,
+            Body: args.body,
+            Metadata: { sha256: args.sha256 },
+            IfNoneMatch: "*",
+          }),
+        );
+        return true;
+      } catch (error) {
+        if (isS3VersionMismatch(error)) return false;
+        throw error;
+      }
+    },
+    async putObjectStreamIfAbsent(args) {
+      try {
+        await requestClient.send(
+          new PutObjectCommand({
+            Bucket: settings.objectStorageBucket,
+            Key: args.key,
+            ContentType: args.contentType,
+            ContentLength: args.byteSize,
+            Body: Readable.from(args.chunks),
+            Metadata: { sha256: args.sha256 },
+            IfNoneMatch: "*",
+          }),
+          args.signal ? { abortSignal: args.signal } : undefined,
+        );
+        return true;
+      } catch (error) {
+        if (isS3VersionMismatch(error)) return false;
+        throw error;
+      }
+    },
     async headFile(file) {
       const head = await requestClient.send(
         new HeadObjectCommand({
@@ -234,6 +309,43 @@ function createS3CompatibleObjectStorage(settings: Settings): ObjectStorage | nu
         throw error;
       }
     },
+    async headObject(key) {
+      try {
+        const head = await requestClient.send(
+          new HeadObjectCommand({ Bucket: settings.objectStorageBucket, Key: key }),
+        );
+        return objectHead({
+          contentLength: head.ContentLength,
+          contentType: head.ContentType,
+          metadata: head.Metadata,
+          versionToken: head.ETag,
+        });
+      } catch (error) {
+        if (isS3NotFound(error)) return null;
+        throw error;
+      }
+    },
+    async getObjectRange(args) {
+      const length = assertRawObjectByteRange(args.key, args.start, args.endInclusive);
+      try {
+        const result = await requestClient.send(
+          new GetObjectCommand({
+            Bucket: settings.objectStorageBucket,
+            Key: args.key,
+            Range: `bytes=${args.start}-${args.endInclusive}`,
+            IfMatch: args.expectedVersionToken,
+          }),
+        );
+        if (!result.ETag || result.ETag !== args.expectedVersionToken) return null;
+        return {
+          bytes: await s3BodyToBoundedBytes(result.Body, args.key, length),
+          versionToken: result.ETag,
+        };
+      } catch (error) {
+        if (isS3NotFound(error) || isS3VersionMismatch(error)) return null;
+        throw error;
+      }
+    },
     async getObjectBytes(key) {
       try {
         const result = await requestClient.send(
@@ -319,6 +431,15 @@ function isS3NotFound(error: unknown): boolean {
   return metadata?.httpStatusCode === 404;
 }
 
+function isS3VersionMismatch(error: unknown): boolean {
+  if (!error || typeof error !== "object") return false;
+  const metadata =
+    "$metadata" in error
+      ? (error as { $metadata?: { httpStatusCode?: number } }).$metadata
+      : undefined;
+  return metadata?.httpStatusCode === 412;
+}
+
 function createGcsObjectStorage(settings: Settings): ObjectStorage {
   const client = new GcsClient(gcsClientOptions(settings));
   const bucket = client.bucket(settings.objectStorageBucket);
@@ -365,6 +486,39 @@ function createGcsObjectStorage(settings: Settings): ObjectStorage {
         ...(args.sha256 ? { metadata: { metadata: { sha256: args.sha256 } } } : {}),
       });
     },
+    async putObjectIfAbsent(args) {
+      try {
+        await bucket.file(args.key).save(Buffer.from(args.body), {
+          contentType: args.contentType,
+          metadata: { metadata: { sha256: args.sha256 } },
+          preconditionOpts: { ifGenerationMatch: 0 },
+        });
+        return true;
+      } catch (error) {
+        if (isGcsVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
+    async putObjectStreamIfAbsent(args) {
+      const destination = bucket.file(args.key).createWriteStream({
+        resumable: false,
+        contentType: args.contentType,
+        highWaterMark: INTERNAL_STREAM_BUFFER_BYTES,
+        metadata: { metadata: { sha256: args.sha256 } },
+        preconditionOpts: { ifGenerationMatch: 0 },
+      });
+      try {
+        if (args.signal) {
+          await pipeline(Readable.from(args.chunks), destination, { signal: args.signal });
+        } else {
+          await pipeline(Readable.from(args.chunks), destination);
+        }
+        return true;
+      } catch (error) {
+        if (isGcsVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
     async headFile(file) {
       const [metadata] = await bucket.file(file.objectKey).getMetadata();
       return objectHead({
@@ -398,6 +552,35 @@ function createGcsObjectStorage(settings: Settings): ObjectStorage {
         throw error;
       }
     },
+    async headObject(key) {
+      try {
+        const [metadata] = await bucket.file(key).getMetadata();
+        return objectHead({
+          contentLength: parseContentLength(metadata.size),
+          contentType: metadata.contentType,
+          metadata: stringMetadata(metadata.metadata),
+          versionToken: metadata.generation === undefined ? undefined : String(metadata.generation),
+        });
+      } catch (error) {
+        if (isGcsNotFound(error)) return null;
+        throw error;
+      }
+    },
+    async getObjectRange(args) {
+      const length = assertRawObjectByteRange(args.key, args.start, args.endInclusive);
+      try {
+        const [bytes] = await bucket
+          .file(args.key, { generation: args.expectedVersionToken })
+          .download({ start: args.start, end: args.endInclusive });
+        return {
+          bytes: exactRangeBytes(bytes, args.key, length),
+          versionToken: args.expectedVersionToken,
+        };
+      } catch (error) {
+        if (isGcsNotFound(error) || isGcsVersionMismatch(error)) return null;
+        throw error;
+      }
+    },
     async getObjectBytes(key) {
       try {
         const [bytes] = await bucket.file(key).download();
@@ -420,12 +603,24 @@ function isGcsNotFound(error: unknown): boolean {
   return Boolean(error) && typeof error === "object" && (error as { code?: unknown }).code === 404;
 }
 
+function isGcsVersionMismatch(error: unknown): boolean {
+  return Boolean(error) && typeof error === "object" && (error as { code?: unknown }).code === 412;
+}
+
 function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null {
   const sharedKey = azureSharedKeyCredential(settings);
-  const serviceClient = settings.objectStorageAzureConnectionString
+  const requestServiceClient = settings.objectStorageAzureConnectionString
     ? BlobServiceClient.fromConnectionString(settings.objectStorageAzureConnectionString)
     : new BlobServiceClient(azureBlobServiceUrl(settings), sharedKey);
-  const containerClient = serviceClient.getContainerClient(settings.objectStorageBucket);
+  const presignServiceClient = settings.objectStorageAzureEndpoint
+    ? new BlobServiceClient(azureBlobServiceUrl(settings), sharedKey)
+    : requestServiceClient;
+  const requestContainerClient = requestServiceClient.getContainerClient(
+    settings.objectStorageBucket,
+  );
+  const presignContainerClient = presignServiceClient.getContainerClient(
+    settings.objectStorageBucket,
+  );
 
   return {
     bucket: settings.objectStorageBucket,
@@ -434,7 +629,7 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
     async createPutUrl(args) {
       const expiresIn = args.expiresInSeconds ?? UPLOAD_URL_TTL_SECONDS;
       const expiresAt = new Date(Date.now() + expiresIn * 1000);
-      const blobClient = containerClient.getBlockBlobClient(args.key);
+      const blobClient = presignContainerClient.getBlockBlobClient(args.key);
       const sas = generateBlobSASQueryParameters(
         {
           containerName: settings.objectStorageBucket,
@@ -458,7 +653,7 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
     async createGetUrl(args) {
       const expiresIn = args.expiresInSeconds ?? DOWNLOAD_URL_TTL_SECONDS;
       const expiresAt = new Date(Date.now() + expiresIn * 1000);
-      const blobClient = containerClient.getBlobClient(args.key);
+      const blobClient = presignContainerClient.getBlobClient(args.key);
       const sas = generateBlobSASQueryParameters(
         {
           containerName: settings.objectStorageBucket,
@@ -475,21 +670,56 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
     },
     async putObject(args) {
       // Authenticated in-process upload via the shared-key Azure client (no SAS).
-      const blobClient = containerClient.getBlockBlobClient(args.key);
+      const blobClient = requestContainerClient.getBlockBlobClient(args.key);
       const body = Buffer.from(args.body);
       await blobClient.upload(body, body.byteLength, {
         blobHTTPHeaders: { blobContentType: args.contentType },
         ...(args.sha256 ? { metadata: { sha256: args.sha256 } } : {}),
       });
     },
+    async putObjectIfAbsent(args) {
+      const blobClient = requestContainerClient.getBlockBlobClient(args.key);
+      const body = Buffer.from(args.body);
+      try {
+        await blobClient.upload(body, body.byteLength, {
+          blobHTTPHeaders: { blobContentType: args.contentType },
+          metadata: { sha256: args.sha256 },
+          conditions: { ifNoneMatch: "*" },
+        });
+        return true;
+      } catch (error) {
+        if (isAzureVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
+    async putObjectStreamIfAbsent(args) {
+      const blobClient = requestContainerClient.getBlockBlobClient(args.key);
+      try {
+        await blobClient.uploadStream(
+          Readable.from(args.chunks),
+          INTERNAL_STREAM_BUFFER_BYTES,
+          INTERNAL_STREAM_CONCURRENCY,
+          {
+            blobHTTPHeaders: { blobContentType: args.contentType },
+            metadata: { sha256: args.sha256 },
+            conditions: { ifNoneMatch: "*" },
+            ...(args.signal ? { abortSignal: args.signal } : {}),
+          },
+        );
+        return true;
+      } catch (error) {
+        if (isAzureVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
     async headFile(file) {
       return azureHeadToObjectHead(
-        await containerClient.getBlobClient(file.objectKey).getProperties(),
+        await requestContainerClient.getBlobClient(file.objectKey).getProperties(),
       );
     },
     async fileExists(file) {
       try {
-        await containerClient.getBlobClient(file.objectKey).getProperties();
+        await requestContainerClient.getBlobClient(file.objectKey).getProperties();
         return true;
       } catch (error) {
         if (isAzureNotFound(error)) return false;
@@ -498,14 +728,14 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
     },
     async getFileBytes(file) {
       return await azureDownloadToBytes(
-        await containerClient.getBlobClient(file.objectKey).download(),
+        await requestContainerClient.getBlobClient(file.objectKey).download(),
       );
     },
     async getFileRange(file, range) {
       const length = assertFileByteRange(file, range);
       try {
         return await azureDownloadToBoundedBytes(
-          await containerClient.getBlobClient(file.objectKey).download(range.start, length),
+          await requestContainerClient.getBlobClient(file.objectKey).download(range.start, length),
           file.objectKey,
           length,
         );
@@ -514,9 +744,39 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
         throw error;
       }
     },
+    async headObject(key) {
+      try {
+        const properties = await requestContainerClient.getBlobClient(key).getProperties();
+        return objectHead({
+          contentLength: properties.contentLength,
+          contentType: properties.contentType,
+          metadata: properties.metadata,
+          versionToken: properties.etag,
+        });
+      } catch (error) {
+        if (isAzureNotFound(error)) return null;
+        throw error;
+      }
+    },
+    async getObjectRange(args) {
+      const length = assertRawObjectByteRange(args.key, args.start, args.endInclusive);
+      try {
+        const download = await requestContainerClient
+          .getBlobClient(args.key)
+          .download(args.start, length, { conditions: { ifMatch: args.expectedVersionToken } });
+        if (!download.etag || download.etag !== args.expectedVersionToken) return null;
+        return {
+          bytes: await azureDownloadToBoundedBytes(download, args.key, length),
+          versionToken: download.etag,
+        };
+      } catch (error) {
+        if (isAzureNotFound(error) || isAzureVersionMismatch(error)) return null;
+        throw error;
+      }
+    },
     async getObjectBytes(key) {
       try {
-        const download = await containerClient.getBlobClient(key).download();
+        const download = await requestContainerClient.getBlobClient(key).download();
         const bytes = await azureDownloadToBytes(download);
         return { bytes, ...(download.contentType ? { contentType: download.contentType } : {}) };
       } catch (error) {
@@ -528,7 +788,7 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
     },
     async deleteObject(key) {
       // deleteIfExists keeps the delete idempotent (a missing blob is a no-op).
-      await containerClient.getBlockBlobClient(key).deleteIfExists();
+      await requestContainerClient.getBlockBlobClient(key).deleteIfExists();
     },
   };
 }
@@ -541,6 +801,14 @@ function isAzureNotFound(error: unknown): boolean {
   );
 }
 
+function isAzureVersionMismatch(error: unknown): boolean {
+  return (
+    Boolean(error) &&
+    typeof error === "object" &&
+    (error as { statusCode?: unknown }).statusCode === 412
+  );
+}
+
 function azureSharedKeyCredential(settings: Settings): StorageSharedKeyCredential {
   if (settings.objectStorageAzureConnectionString) {
     const parsed = parseConnectionString(settings.objectStorageAzureConnectionString);
@@ -704,6 +972,25 @@ function assertFileByteRange(
   return length;
 }
 
+function assertRawObjectByteRange(key: string, start: number, endInclusive: number): number {
+  if (
+    typeof key !== "string" ||
+    key.length < 1 ||
+    key.length > 2048 ||
+    !Number.isSafeInteger(start) ||
+    !Number.isSafeInteger(endInclusive) ||
+    start < 0 ||
+    endInclusive < start
+  ) {
+    throw new RangeError("Invalid raw object byte range");
+  }
+  const length = endInclusive - start + 1;
+  if (length > RETAINED_OUTPUT_MAX_PAGE_BYTES) {
+    throw new RangeError("Raw object byte range exceeds the bounded page limit");
+  }
+  return length;
+}
+
 function exactRangeBytes(bytes: Uint8Array, objectKey: string, expectedBytes: number): Uint8Array {
   if (bytes.byteLength !== expectedBytes) {
     throw new Error(
@@ -717,11 +1004,13 @@ function objectHead(input: {
   contentLength?: number | undefined;
   contentType?: string | undefined;
   metadata?: Record<string, string> | undefined;
+  versionToken?: string | undefined;
 }): ObjectHead {
   return {
     ...(input.contentLength !== undefined ? { ContentLength: input.contentLength } : {}),
     ...(input.contentType !== undefined ? { ContentType: input.contentType } : {}),
     ...(input.metadata !== undefined ? { Metadata: input.metadata } : {}),
+    ...(input.versionToken !== undefined ? { VersionToken: input.versionToken } : {}),
   };
 }