@opengeni/storage 0.2.126 → 0.2.127-canary.0

package/dist/index.d.ts

@@ -14,6 +14,7 @@ export type ObjectHead = {
     /** Opaque provider generation/etag used only for conditional internal reads. */
     VersionToken?: string;
 };
+export { uploadWorkspaceArchiveSpool, downloadWorkspaceArchiveSpool, WorkspaceArchiveStorageError, } from "./workspace-archive-spool.js";
 export type ObjectStorage = {
     bucket: string;
     backend: "s3-compatible" | "aws-s3" | "azure-blob" | "gcs";
@@ -85,6 +86,19 @@ export type ObjectStorage = {
         body: Uint8Array;
         sha256?: string | null;
     }) => Promise<void>;
+    /**
+     * Unconditional authenticated upload from a bounded byte stream. May overwrite
+     * an existing key; this is NOT an atomic create-only operation. Callers needing
+     * write isolation must supply a fresh unique key and verify stored content.
+     */
+    putObjectStream?: (args: {
+        key: string;
+        contentType: string;
+        chunks: AsyncIterable<Uint8Array>;
+        byteSize: number;
+        sha256?: string;
+        signal?: AbortSignal;
+    }) => Promise<void>;
     /** Atomic create-only raw PUT. Returns false when the key already exists. */
     putObjectIfAbsent?: (args: {
         key: string;

package/dist/index.js

@@ -629,6 +629,263 @@ function throwIfAborted4(signal) {
   if (signal?.aborted) throw new Error("Object storage operation was cancelled");
 }
 
+// src/workspace-archive-spool.ts
+import { createHash as createHash3 } from "crypto";
+import { createReadStream } from "fs";
+import { chmod as chmod2, mkdtemp as mkdtemp2, open as open2, rm as rm2 } from "fs/promises";
+import { tmpdir as tmpdir2 } from "os";
+import { join as join2 } from "path";
+var CHUNK_BYTES = DEFAULT_BOUNDED_OBJECT_CHUNK_BYTES;
+var WorkspaceArchiveStorageError = class extends Error {
+  constructor(code, message, retryable, options) {
+    super(message, options);
+    this.code = code;
+    this.retryable = retryable;
+    this.name = "WorkspaceArchiveStorageError";
+  }
+};
+async function uploadWorkspaceArchiveSpool(storage, key, spool) {
+  requireBoundedReads(storage);
+  if (!storage.putObjectStream) {
+    throw new WorkspaceArchiveStorageError(
+      "archive_hydration_failed",
+      "Workspace archive storage unsupported: unconditional streaming upload required",
+      false
+    );
+  }
+  const expected = { bytes: spool.byteSize, sha256: spool.sha256 };
+  validateExpected(expected);
+  let validated = false;
+  let streamFailed = false;
+  let streamFailure;
+  const chunks = async function* () {
+    const digest = createHash3("sha256");
+    let bytes = 0;
+    try {
+      for await (const chunk of spool.open()) {
+        if (!(chunk instanceof Uint8Array) || chunk.byteLength > expected.bytes - bytes) {
+          throw new WorkspaceArchiveStorageError(
+            "archive_hash_mismatch",
+            "Workspace archive upload stream has invalid size",
+            false
+          );
+        }
+        bytes += chunk.byteLength;
+        digest.update(chunk);
+        yield chunk;
+      }
+      if (bytes !== expected.bytes || digest.digest("hex") !== expected.sha256) {
+        throw new WorkspaceArchiveStorageError(
+          "archive_hash_mismatch",
+          "Workspace archive upload stream digest or size mismatch",
+          false
+        );
+      }
+      validated = true;
+    } catch (error) {
+      streamFailed = true;
+      streamFailure = error;
+      throw error;
+    }
+  }();
+  try {
+    await storage.putObjectStream({
+      key,
+      contentType: "application/x-tar",
+      chunks,
+      byteSize: expected.bytes,
+      sha256: expected.sha256
+    });
+    if (!validated) {
+      if (streamFailed) throw streamFailure;
+      throw new WorkspaceArchiveStorageError(
+        "archive_hydration_failed",
+        "Workspace archive upload provider did not completely consume and validate the stream",
+        false
+      );
+    }
+    await verifyRanges(storage, key, expected);
+  } catch (error) {
+    const failure = streamFailed ? streamFailure : error;
+    if (failure instanceof WorkspaceArchiveStorageError) throw failure;
+    throw new WorkspaceArchiveStorageError(
+      "archive_hydration_failed",
+      "Workspace archive upload or readback failed",
+      true,
+      { cause: failure }
+    );
+  } finally {
+    await chunks.return(void 0);
+  }
+}
+async function downloadWorkspaceArchiveSpool(storage, key, inputExpected) {
+  const expected = { bytes: inputExpected.bytes, sha256: inputExpected.sha256 };
+  requireBoundedReads(storage);
+  validateExpected(expected);
+  const directory = await mkdtemp2(join2(tmpdir2(), "opengeni-workspace-archive-"));
+  const path = join2(directory, "archive.tar");
+  let handle;
+  try {
+    await chmod2(directory, 448);
+    handle = await open2(path, "wx", 384);
+    await verifyRanges(storage, key, expected, async (chunk) => {
+      let offset = 0;
+      while (offset < chunk.byteLength) {
+        const { bytesWritten } = await handle.write(chunk, offset, chunk.byteLength - offset);
+        if (bytesWritten <= 0)
+          throw new WorkspaceArchiveStorageError(
+            "archive_hydration_failed",
+            "Workspace archive spool write made no progress",
+            true
+          );
+        offset += bytesWritten;
+      }
+    });
+    await handle.close();
+    handle = void 0;
+    let disposed = false;
+    return {
+      path,
+      byteSize: expected.bytes,
+      sha256: expected.sha256,
+      async *open() {
+        if (disposed) throw new Error("Workspace archive spool is disposed");
+        const stream = createReadStream(path, { highWaterMark: CHUNK_BYTES });
+        try {
+          for await (const chunk of stream) yield chunk;
+        } finally {
+          stream.destroy();
+        }
+      },
+      async dispose() {
+        disposed = true;
+        await rm2(directory, { recursive: true, force: true });
+      }
+    };
+  } catch (error) {
+    await handle?.close().catch(() => void 0);
+    await rm2(directory, { recursive: true, force: true });
+    if (error instanceof WorkspaceArchiveStorageError) throw error;
+    throw new WorkspaceArchiveStorageError(
+      "archive_hydration_failed",
+      "Workspace archive download failed",
+      true,
+      { cause: error }
+    );
+  }
+}
+function requireBoundedReads(storage) {
+  if (!storage.headObject || !storage.getObjectRange) {
+    throw new WorkspaceArchiveStorageError(
+      "archive_hydration_failed",
+      "Workspace archive storage unsupported: versioned head/range reads required",
+      false
+    );
+  }
+}
+function validateExpected(expected) {
+  if (!Number.isSafeInteger(expected.bytes) || expected.bytes < 0 || !/^[0-9a-f]{64}$/u.test(expected.sha256)) {
+    throw new WorkspaceArchiveStorageError(
+      "archive_hydration_failed",
+      "Workspace archive expected size or SHA-256 is invalid",
+      false
+    );
+  }
+}
+async function verifyRanges(storage, key, expected, consume) {
+  const head = await storage.headObject(key);
+  if (!head)
+    throw new WorkspaceArchiveStorageError(
+      "archive_base64_invalid",
+      "Workspace archive object is missing",
+      false
+    );
+  if (head.ContentLength !== expected.bytes)
+    throw new WorkspaceArchiveStorageError(
+      "archive_hash_mismatch",
+      "Workspace archive object size mismatch",
+      false
+    );
+  const version = head.VersionToken;
+  if (typeof version !== "string" || version.length === 0 || version.length > 2048) {
+    throw new WorkspaceArchiveStorageError(
+      "archive_hydration_failed",
+      "Workspace archive storage unsupported: valid object version token required",
+      false
+    );
+  }
+  const digest = createHash3("sha256");
+  let bytes = 0;
+  while (bytes < expected.bytes) {
+    const length = Math.min(CHUNK_BYTES, expected.bytes - bytes);
+    const result = await storage.getObjectRange({
+      key,
+      start: bytes,
+      endInclusive: bytes + length - 1,
+      expectedVersionToken: version
+    });
+    if (!result) {
+      const current = await storage.headObject(key);
+      if (!current) {
+        throw new WorkspaceArchiveStorageError(
+          "archive_base64_invalid",
+          "Workspace archive object is missing during range read",
+          false
+        );
+      }
+      throw new WorkspaceArchiveStorageError(
+        "archive_hydration_failed",
+        current.VersionToken !== version ? "Workspace archive object version changed" : "Workspace archive pinned range is temporarily unavailable",
+        true
+      );
+    }
+    if (result.versionToken !== version)
+      throw new WorkspaceArchiveStorageError(
+        "archive_hydration_failed",
+        "Workspace archive object version changed",
+        true
+      );
+    if (!(result.bytes instanceof Uint8Array) || result.bytes.byteLength !== length) {
+      throw new WorkspaceArchiveStorageError(
+        "archive_hash_mismatch",
+        "Workspace archive object range is truncated or has invalid size",
+        false
+      );
+    }
+    digest.update(result.bytes);
+    await consume?.(result.bytes);
+    bytes += result.bytes.byteLength;
+  }
+  const finalHead = await storage.headObject(key);
+  if (!finalHead)
+    throw new WorkspaceArchiveStorageError(
+      "archive_base64_invalid",
+      "Workspace archive object is missing after range read",
+      false
+    );
+  if (finalHead.VersionToken !== version) {
+    throw new WorkspaceArchiveStorageError(
+      "archive_hydration_failed",
+      "Workspace archive object version changed",
+      true
+    );
+  }
+  if (finalHead.ContentLength !== expected.bytes) {
+    throw new WorkspaceArchiveStorageError(
+      "archive_hash_mismatch",
+      "Workspace archive object size changed",
+      false
+    );
+  }
+  if (bytes !== expected.bytes || digest.digest("hex") !== expected.sha256) {
+    throw new WorkspaceArchiveStorageError(
+      "archive_hash_mismatch",
+      "Workspace archive object digest or size mismatch",
+      false
+    );
+  }
+}
+
 // src/index.ts
 var MAX_SINGLE_PUT_SIZE_BYTES = 5e9;
 var UPLOAD_URL_TTL_SECONDS = 15 * 60;
@@ -741,6 +998,46 @@ function createS3CompatibleObjectStorage(settings) {
         throw error;
       }
     },
+    async putObjectStream(args) {
+      const body = Readable.from(args.chunks, {
+        objectMode: false,
+        highWaterMark: INTERNAL_STREAM_BUFFER_BYTES
+      });
+      const request = new AbortController();
+      let producerFailed = false;
+      let producerError;
+      const onBodyError = (error) => {
+        producerFailed = true;
+        producerError = error;
+        request.abort(error);
+      };
+      const onCallerAbort = () => request.abort(args.signal?.reason);
+      body.once("error", onBodyError);
+      const detachBodyError = () => body.off("error", onBodyError);
+      body.once("close", detachBodyError);
+      args.signal?.addEventListener("abort", onCallerAbort, { once: true });
+      if (args.signal?.aborted) onCallerAbort();
+      try {
+        await requestClient.send(
+          new PutObjectCommand({
+            Bucket: settings.objectStorageBucket,
+            Key: args.key,
+            ContentType: args.contentType,
+            ContentLength: args.byteSize,
+            Body: body,
+            Metadata: args.sha256 ? { sha256: args.sha256 } : void 0
+          }),
+          { abortSignal: request.signal }
+        );
+        if (producerFailed) throw producerError;
+      } catch (error) {
+        if (producerFailed) throw producerError;
+        throw error;
+      } finally {
+        args.signal?.removeEventListener("abort", onCallerAbort);
+        body.destroy();
+      }
+    },
     async putObjectStreamIfAbsent(args) {
       try {
         await requestClient.send(
@@ -986,6 +1283,23 @@ function createGcsObjectStorage(settings) {
         throw error;
       }
     },
+    async putObjectStream(args) {
+      const destination = bucket.file(args.key).createWriteStream({
+        resumable: false,
+        contentType: args.contentType,
+        highWaterMark: INTERNAL_STREAM_BUFFER_BYTES,
+        ...args.sha256 ? { metadata: { metadata: { sha256: args.sha256 } } } : {}
+      });
+      const source = Readable.from(args.chunks, {
+        objectMode: false,
+        highWaterMark: INTERNAL_STREAM_BUFFER_BYTES
+      });
+      if (args.signal) {
+        await pipeline(source, destination, { signal: args.signal });
+      } else {
+        await pipeline(source, destination);
+      }
+    },
     async putObjectStreamIfAbsent(args) {
       const destination = bucket.file(args.key).createWriteStream({
         resumable: false,
@@ -1165,6 +1479,27 @@ function createAzureBlobObjectStorage(settings) {
         throw error;
       }
     },
+    async putObjectStream(args) {
+      const blobClient = requestContainerClient.getBlockBlobClient(args.key);
+      const source = Readable.from(args.chunks, {
+        objectMode: false,
+        highWaterMark: INTERNAL_STREAM_BUFFER_BYTES
+      });
+      try {
+        await blobClient.uploadStream(
+          source,
+          INTERNAL_STREAM_BUFFER_BYTES,
+          INTERNAL_STREAM_CONCURRENCY,
+          {
+            blobHTTPHeaders: { blobContentType: args.contentType },
+            ...args.sha256 ? { metadata: { sha256: args.sha256 } } : {},
+            ...args.signal ? { abortSignal: args.signal } : {}
+          }
+        );
+      } finally {
+        source.destroy();
+      }
+    },
     async putObjectStreamIfAbsent(args) {
       const blobClient = requestContainerClient.getBlockBlobClient(args.key);
       try {
@@ -1429,11 +1764,14 @@ export {
   MAX_SINGLE_PUT_SIZE_BYTES,
   OBJECT_VISIBILITY_RETRY_DELAYS_MS,
   UPLOAD_URL_TTL_SECONDS,
+  WorkspaceArchiveStorageError,
   bytesToDataUrl,
   createBoundedImmutableObjectWritePort,
   createBoundedObjectReadPort,
   createObjectStorage,
   createObjectStorageBoundedPorts,
-  retryWhileMissing
+  downloadWorkspaceArchiveSpool,
+  retryWhileMissing,
+  uploadWorkspaceArchiveSpool
 };
 //# sourceMappingURL=index.js.map