@opengeni/storage 0.2.75 → 0.2.93

package/src/index.ts

@@ -1,5 +1,7 @@
 import type { Settings } from "@opengeni/config";
 import { RETAINED_OUTPUT_MAX_PAGE_BYTES, type FileAsset } from "@opengeni/contracts";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 import {
   BlobSASPermissions,
   BlobServiceClient,
@@ -23,14 +25,22 @@ import {
   type StorageOptions,
 } from "@google-cloud/storage";
 
+export * from "./bounded-object-read";
+export * from "./bounded-object-write";
+export * from "./object-storage-bounded";
+
 export const MAX_SINGLE_PUT_SIZE_BYTES = 5_000_000_000;
 export const UPLOAD_URL_TTL_SECONDS = 15 * 60;
 export const DOWNLOAD_URL_TTL_SECONDS = 5 * 60;
+const INTERNAL_STREAM_BUFFER_BYTES = 1024 * 1024;
+const INTERNAL_STREAM_CONCURRENCY = 2;
 
 export type ObjectHead = {
   ContentLength?: number;
   ContentType?: string;
   Metadata?: Record<string, string>;
+  /** Opaque provider generation/etag used only for conditional internal reads. */
+  VersionToken?: string;
 };
 
 export type ObjectStorage = {
@@ -42,10 +52,14 @@ export type ObjectStorage = {
     contentType: string;
     sha256?: string | null;
     expiresInSeconds?: number;
+    /** Select the network endpoint embedded in the signed URL. */
+    audience?: "public" | "sandbox";
   }) => Promise<{ url: string; requiredHeaders: Record<string, string>; expiresAt: Date }>;
   createGetUrl: (args: {
     key: string;
     expiresInSeconds?: number;
+    /** Select the network endpoint embedded in the signed URL. */
+    audience?: "public" | "sandbox";
   }) => Promise<{ url: string; expiresAt: Date }>;
   headFile: (file: FileAsset) => Promise<ObjectHead>;
   /** Check provider existence without downloading object bytes. */
@@ -61,6 +75,14 @@ export type ObjectStorage = {
   ) => Promise<Uint8Array | null>;
   /** Fetch an object by raw storage key (not a tracked FileAsset). Returns null on 404/missing. */
   getObjectBytes: (key: string) => Promise<{ bytes: Uint8Array; contentType?: string } | null>;
+  /** Provider-versioned raw-key primitives used by bounded immutable adapters. */
+  headObject?: (key: string) => Promise<ObjectHead | null>;
+  getObjectRange?: (args: {
+    key: string;
+    start: number;
+    endInclusive: number;
+    expectedVersionToken: string;
+  }) => Promise<{ bytes: Uint8Array; versionToken: string } | null>;
   /**
    * SERVER-SIDE authenticated direct PUT (no presign + browser fetch). For an
    * in-process upload from a trusted holder of the storage credentials (e.g. the
@@ -79,6 +101,25 @@ export type ObjectStorage = {
     body: Uint8Array;
     sha256?: string | null;
   }) => Promise<void>;
+  /** Atomic create-only raw PUT. Returns false when the key already exists. */
+  putObjectIfAbsent?: (args: {
+    key: string;
+    contentType: string;
+    body: Uint8Array;
+    sha256: string;
+  }) => Promise<boolean>;
+  /**
+   * Atomic create-only raw upload from a bounded asynchronous byte stream.
+   * Providers may buffer a small fixed number of chunks, never the whole body.
+   */
+  putObjectStreamIfAbsent?: (args: {
+    key: string;
+    contentType: string;
+    chunks: AsyncIterable<Uint8Array>;
+    byteSize: number;
+    sha256: string;
+    signal?: AbortSignal;
+  }) => Promise<boolean>;
   /**
    * SERVER-SIDE authenticated delete of a single object by raw storage key.
    * Idempotent: a missing key is a no-op (S3/GCS/Azure delete-by-key does not
@@ -126,6 +167,12 @@ function createS3CompatibleObjectStorage(settings: Settings): ObjectStorage | nu
     ...sharedClientConfig,
     ...(settings.objectStorageEndpoint ? { endpoint: settings.objectStorageEndpoint } : {}),
   });
+  const sandboxPresignClient = settings.objectStorageSandboxEndpoint
+    ? new S3Client({
+        ...sharedClientConfig,
+        endpoint: settings.objectStorageSandboxEndpoint,
+      })
+    : presignClient;
   const requestClient = settings.objectStorageInternalEndpoint
     ? new S3Client({
         ...sharedClientConfig,
@@ -148,7 +195,13 @@ function createS3CompatibleObjectStorage(settings: Settings): ObjectStorage | nu
         Metadata: args.sha256 ? { sha256: args.sha256 } : undefined,
       });
       return {
-        url: await getSignedUrl(presignClient, command, { expiresIn }),
+        url: await getSignedUrl(
+          args.audience === "sandbox" ? sandboxPresignClient : presignClient,
+          command,
+          {
+            expiresIn,
+          },
+        ),
         requiredHeaders,
         expiresAt: new Date(Date.now() + expiresIn * 1000),
       };
@@ -157,7 +210,7 @@ function createS3CompatibleObjectStorage(settings: Settings): ObjectStorage | nu
       const expiresIn = args.expiresInSeconds ?? DOWNLOAD_URL_TTL_SECONDS;
       return {
         url: await getSignedUrl(
-          presignClient,
+          args.audience === "sandbox" ? sandboxPresignClient : presignClient,
           new GetObjectCommand({
             Bucket: settings.objectStorageBucket,
             Key: args.key,
@@ -182,6 +235,44 @@ function createS3CompatibleObjectStorage(settings: Settings): ObjectStorage | nu
         }),
       );
     },
+    async putObjectIfAbsent(args) {
+      try {
+        await requestClient.send(
+          new PutObjectCommand({
+            Bucket: settings.objectStorageBucket,
+            Key: args.key,
+            ContentType: args.contentType,
+            Body: args.body,
+            Metadata: { sha256: args.sha256 },
+            IfNoneMatch: "*",
+          }),
+        );
+        return true;
+      } catch (error) {
+        if (isS3VersionMismatch(error)) return false;
+        throw error;
+      }
+    },
+    async putObjectStreamIfAbsent(args) {
+      try {
+        await requestClient.send(
+          new PutObjectCommand({
+            Bucket: settings.objectStorageBucket,
+            Key: args.key,
+            ContentType: args.contentType,
+            ContentLength: args.byteSize,
+            Body: Readable.from(args.chunks),
+            Metadata: { sha256: args.sha256 },
+            IfNoneMatch: "*",
+          }),
+          args.signal ? { abortSignal: args.signal } : undefined,
+        );
+        return true;
+      } catch (error) {
+        if (isS3VersionMismatch(error)) return false;
+        throw error;
+      }
+    },
     async headFile(file) {
       const head = await requestClient.send(
         new HeadObjectCommand({
@@ -234,6 +325,43 @@ function createS3CompatibleObjectStorage(settings: Settings): ObjectStorage | nu
         throw error;
       }
     },
+    async headObject(key) {
+      try {
+        const head = await requestClient.send(
+          new HeadObjectCommand({ Bucket: settings.objectStorageBucket, Key: key }),
+        );
+        return objectHead({
+          contentLength: head.ContentLength,
+          contentType: head.ContentType,
+          metadata: head.Metadata,
+          versionToken: head.ETag,
+        });
+      } catch (error) {
+        if (isS3NotFound(error)) return null;
+        throw error;
+      }
+    },
+    async getObjectRange(args) {
+      const length = assertRawObjectByteRange(args.key, args.start, args.endInclusive);
+      try {
+        const result = await requestClient.send(
+          new GetObjectCommand({
+            Bucket: settings.objectStorageBucket,
+            Key: args.key,
+            Range: `bytes=${args.start}-${args.endInclusive}`,
+            IfMatch: args.expectedVersionToken,
+          }),
+        );
+        if (!result.ETag || result.ETag !== args.expectedVersionToken) return null;
+        return {
+          bytes: await s3BodyToBoundedBytes(result.Body, args.key, length),
+          versionToken: result.ETag,
+        };
+      } catch (error) {
+        if (isS3NotFound(error) || isS3VersionMismatch(error)) return null;
+        throw error;
+      }
+    },
     async getObjectBytes(key) {
       try {
         const result = await requestClient.send(
@@ -319,6 +447,15 @@ function isS3NotFound(error: unknown): boolean {
   return metadata?.httpStatusCode === 404;
 }
 
+function isS3VersionMismatch(error: unknown): boolean {
+  if (!error || typeof error !== "object") return false;
+  const metadata =
+    "$metadata" in error
+      ? (error as { $metadata?: { httpStatusCode?: number } }).$metadata
+      : undefined;
+  return metadata?.httpStatusCode === 412;
+}
+
 function createGcsObjectStorage(settings: Settings): ObjectStorage {
   const client = new GcsClient(gcsClientOptions(settings));
   const bucket = client.bucket(settings.objectStorageBucket);
@@ -365,6 +502,39 @@ function createGcsObjectStorage(settings: Settings): ObjectStorage {
         ...(args.sha256 ? { metadata: { metadata: { sha256: args.sha256 } } } : {}),
       });
     },
+    async putObjectIfAbsent(args) {
+      try {
+        await bucket.file(args.key).save(Buffer.from(args.body), {
+          contentType: args.contentType,
+          metadata: { metadata: { sha256: args.sha256 } },
+          preconditionOpts: { ifGenerationMatch: 0 },
+        });
+        return true;
+      } catch (error) {
+        if (isGcsVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
+    async putObjectStreamIfAbsent(args) {
+      const destination = bucket.file(args.key).createWriteStream({
+        resumable: false,
+        contentType: args.contentType,
+        highWaterMark: INTERNAL_STREAM_BUFFER_BYTES,
+        metadata: { metadata: { sha256: args.sha256 } },
+        preconditionOpts: { ifGenerationMatch: 0 },
+      });
+      try {
+        if (args.signal) {
+          await pipeline(Readable.from(args.chunks), destination, { signal: args.signal });
+        } else {
+          await pipeline(Readable.from(args.chunks), destination);
+        }
+        return true;
+      } catch (error) {
+        if (isGcsVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
     async headFile(file) {
       const [metadata] = await bucket.file(file.objectKey).getMetadata();
       return objectHead({
@@ -398,6 +568,35 @@ function createGcsObjectStorage(settings: Settings): ObjectStorage {
         throw error;
       }
     },
+    async headObject(key) {
+      try {
+        const [metadata] = await bucket.file(key).getMetadata();
+        return objectHead({
+          contentLength: parseContentLength(metadata.size),
+          contentType: metadata.contentType,
+          metadata: stringMetadata(metadata.metadata),
+          versionToken: metadata.generation === undefined ? undefined : String(metadata.generation),
+        });
+      } catch (error) {
+        if (isGcsNotFound(error)) return null;
+        throw error;
+      }
+    },
+    async getObjectRange(args) {
+      const length = assertRawObjectByteRange(args.key, args.start, args.endInclusive);
+      try {
+        const [bytes] = await bucket
+          .file(args.key, { generation: args.expectedVersionToken })
+          .download({ start: args.start, end: args.endInclusive });
+        return {
+          bytes: exactRangeBytes(bytes, args.key, length),
+          versionToken: args.expectedVersionToken,
+        };
+      } catch (error) {
+        if (isGcsNotFound(error) || isGcsVersionMismatch(error)) return null;
+        throw error;
+      }
+    },
     async getObjectBytes(key) {
       try {
         const [bytes] = await bucket.file(key).download();
@@ -420,6 +619,10 @@ function isGcsNotFound(error: unknown): boolean {
   return Boolean(error) && typeof error === "object" && (error as { code?: unknown }).code === 404;
 }
 
+function isGcsVersionMismatch(error: unknown): boolean {
+  return Boolean(error) && typeof error === "object" && (error as { code?: unknown }).code === 412;
+}
+
 function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null {
   const sharedKey = azureSharedKeyCredential(settings);
   const requestServiceClient = settings.objectStorageAzureConnectionString
@@ -490,6 +693,41 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
         ...(args.sha256 ? { metadata: { sha256: args.sha256 } } : {}),
       });
     },
+    async putObjectIfAbsent(args) {
+      const blobClient = requestContainerClient.getBlockBlobClient(args.key);
+      const body = Buffer.from(args.body);
+      try {
+        await blobClient.upload(body, body.byteLength, {
+          blobHTTPHeaders: { blobContentType: args.contentType },
+          metadata: { sha256: args.sha256 },
+          conditions: { ifNoneMatch: "*" },
+        });
+        return true;
+      } catch (error) {
+        if (isAzureVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
+    async putObjectStreamIfAbsent(args) {
+      const blobClient = requestContainerClient.getBlockBlobClient(args.key);
+      try {
+        await blobClient.uploadStream(
+          Readable.from(args.chunks),
+          INTERNAL_STREAM_BUFFER_BYTES,
+          INTERNAL_STREAM_CONCURRENCY,
+          {
+            blobHTTPHeaders: { blobContentType: args.contentType },
+            metadata: { sha256: args.sha256 },
+            conditions: { ifNoneMatch: "*" },
+            ...(args.signal ? { abortSignal: args.signal } : {}),
+          },
+        );
+        return true;
+      } catch (error) {
+        if (isAzureVersionMismatch(error)) return false;
+        throw error;
+      }
+    },
     async headFile(file) {
       return azureHeadToObjectHead(
         await requestContainerClient.getBlobClient(file.objectKey).getProperties(),
@@ -522,6 +760,36 @@ function createAzureBlobObjectStorage(settings: Settings): ObjectStorage | null
         throw error;
       }
     },
+    async headObject(key) {
+      try {
+        const properties = await requestContainerClient.getBlobClient(key).getProperties();
+        return objectHead({
+          contentLength: properties.contentLength,
+          contentType: properties.contentType,
+          metadata: properties.metadata,
+          versionToken: properties.etag,
+        });
+      } catch (error) {
+        if (isAzureNotFound(error)) return null;
+        throw error;
+      }
+    },
+    async getObjectRange(args) {
+      const length = assertRawObjectByteRange(args.key, args.start, args.endInclusive);
+      try {
+        const download = await requestContainerClient
+          .getBlobClient(args.key)
+          .download(args.start, length, { conditions: { ifMatch: args.expectedVersionToken } });
+        if (!download.etag || download.etag !== args.expectedVersionToken) return null;
+        return {
+          bytes: await azureDownloadToBoundedBytes(download, args.key, length),
+          versionToken: download.etag,
+        };
+      } catch (error) {
+        if (isAzureNotFound(error) || isAzureVersionMismatch(error)) return null;
+        throw error;
+      }
+    },
     async getObjectBytes(key) {
       try {
         const download = await requestContainerClient.getBlobClient(key).download();
@@ -549,6 +817,14 @@ function isAzureNotFound(error: unknown): boolean {
   );
 }
 
+function isAzureVersionMismatch(error: unknown): boolean {
+  return (
+    Boolean(error) &&
+    typeof error === "object" &&
+    (error as { statusCode?: unknown }).statusCode === 412
+  );
+}
+
 function azureSharedKeyCredential(settings: Settings): StorageSharedKeyCredential {
   if (settings.objectStorageAzureConnectionString) {
     const parsed = parseConnectionString(settings.objectStorageAzureConnectionString);
@@ -712,6 +988,25 @@ function assertFileByteRange(
   return length;
 }
 
+function assertRawObjectByteRange(key: string, start: number, endInclusive: number): number {
+  if (
+    typeof key !== "string" ||
+    key.length < 1 ||
+    key.length > 2048 ||
+    !Number.isSafeInteger(start) ||
+    !Number.isSafeInteger(endInclusive) ||
+    start < 0 ||
+    endInclusive < start
+  ) {
+    throw new RangeError("Invalid raw object byte range");
+  }
+  const length = endInclusive - start + 1;
+  if (length > RETAINED_OUTPUT_MAX_PAGE_BYTES) {
+    throw new RangeError("Raw object byte range exceeds the bounded page limit");
+  }
+  return length;
+}
+
 function exactRangeBytes(bytes: Uint8Array, objectKey: string, expectedBytes: number): Uint8Array {
   if (bytes.byteLength !== expectedBytes) {
     throw new Error(
@@ -725,11 +1020,13 @@ function objectHead(input: {
   contentLength?: number | undefined;
   contentType?: string | undefined;
   metadata?: Record<string, string> | undefined;
+  versionToken?: string | undefined;
 }): ObjectHead {
   return {
     ...(input.contentLength !== undefined ? { ContentLength: input.contentLength } : {}),
     ...(input.contentType !== undefined ? { ContentType: input.contentType } : {}),
     ...(input.metadata !== undefined ? { Metadata: input.metadata } : {}),
+    ...(input.versionToken !== undefined ? { VersionToken: input.versionToken } : {}),
   };
 }