@amerged/ohmyhost-runtime 0.1.23 → 0.1.25

package/src/packages/customer-runtime/src/storage-batch-stream.ts

@@ -0,0 +1,318 @@
+import { CustomerRuntimeGatewayError } from "./errors.js";
+import { exactRecord, invalidResponse } from "./http.js";
+
+export type StorageReadResult =
+  | Readonly<{ state: "not_found"; objectKey: string }>
+  | Readonly<{
+      state: "found";
+      objectKey: string;
+      contentType: string;
+      contentLength: number;
+      etag: string;
+      body: ReadableStream<Uint8Array>;
+    }>;
+export type StorageDeleteResult =
+  | Readonly<{ state: "completed"; objectKey: string; deletionId: string }>
+  | Readonly<{ state: "pending"; objectKey: string; retryAt: string }>
+  | Readonly<{
+      state: "failed";
+      objectKey: string;
+      code: "object_not_found" | "storage_gateway_unavailable";
+      retryable: boolean;
+    }>;
+export const STORAGE_BATCH_CONTENT_TYPE = "application/vnd.ohmyhost.storage-batch.v1";
+const MAX_CONTROL_BYTES = 4 * 1024 * 1024;
+const MAX_HEADER_BYTES = 2048;
+
+/** Pending reads are bounded; errors never expose provider text. */
+async function timedRead(
+  reader: ReadableStreamDefaultReader<Uint8Array>,
+  timeoutMilliseconds: number,
+  signal?: AbortSignal,
+): Promise<ReadableStreamReadResult<Uint8Array>> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let abortListener: (() => void) | undefined;
+  try {
+    if (signal?.aborted) {
+      void reader.cancel().catch(() => undefined);
+      throw new CustomerRuntimeGatewayError("gateway_timeout", null, true);
+    }
+    return await Promise.race([
+      reader.read(),
+      new Promise<never>((_resolve, reject) => {
+        const abort = () => {
+          reject(new CustomerRuntimeGatewayError("gateway_timeout", null, true));
+          void reader.cancel().catch(() => undefined);
+        };
+        timer = setTimeout(abort, timeoutMilliseconds);
+        if (signal !== undefined) {
+          abortListener = abort;
+          signal.addEventListener("abort", abortListener, { once: true });
+        }
+      }),
+    ]);
+  } catch (error) {
+    if (error instanceof CustomerRuntimeGatewayError) throw error;
+    if (signal?.aborted) throw new CustomerRuntimeGatewayError("gateway_timeout", null, true);
+    throw new CustomerRuntimeGatewayError("gateway_unavailable", null, true);
+  } finally {
+    clearTimeout(timer);
+    if (abortListener !== undefined) signal?.removeEventListener("abort", abortListener);
+  }
+}
+
+/** Yields one bounded body at a time; completion requires the final frame and exact EOF. */
+export function decodeStorageBatch(
+  response: Response,
+  keys: readonly string[],
+  timeoutMilliseconds: number,
+): AsyncIterable<StorageReadResult> {
+  if (
+    response.headers.get("content-type") !== STORAGE_BATCH_CONTENT_TYPE ||
+    response.body === null
+  ) {
+    void response.body?.cancel().catch(() => undefined);
+    invalidResponse();
+  }
+  const source = response.body.getReader();
+  let buffer: Uint8Array = new Uint8Array();
+  let offset = 0;
+  let controlBytes = 0;
+  let started = false;
+  let finished = false;
+  let cancelled = false;
+  let released = false;
+  let activeBody: ReadableStreamDefaultController<Uint8Array> | undefined;
+  function cancellationError() {
+    return new CustomerRuntimeGatewayError(
+      "gateway_request_invalid",
+      null,
+      false,
+      "this storage read was cancelled; start another readMany() to read more objects",
+    );
+  }
+  function closeSource() {
+    if (released) return;
+    released = true;
+    if (!finished) void source.cancel().catch(() => undefined);
+    source.releaseLock();
+  }
+  async function chunk(maximum: number): Promise<Uint8Array> {
+    if (cancelled) throw cancellationError();
+    while (offset === buffer.byteLength) {
+      const next = await timedRead(source, timeoutMilliseconds);
+      if (cancelled) throw cancellationError();
+      if (next.done || !(next.value instanceof Uint8Array)) invalidResponse();
+      buffer = next.value;
+      offset = 0;
+    }
+    const length = Math.min(maximum, buffer.byteLength - offset);
+    const bytes = buffer.subarray(offset, offset + length);
+    offset += length;
+    return bytes;
+  }
+  async function exactBytes(length: number): Promise<Uint8Array> {
+    const bytes = new Uint8Array(length);
+    let written = 0;
+    while (written < length) {
+      const next = await chunk(length - written);
+      bytes.set(next, written);
+      written += next.byteLength;
+    }
+    return bytes;
+  }
+  async function header(): Promise<unknown> {
+    const length = new DataView((await exactBytes(4)).buffer).getUint32(0);
+    controlBytes += 4 + length;
+    if (length < 1 || length > MAX_HEADER_BYTES || controlBytes > MAX_CONTROL_BYTES)
+      invalidResponse();
+    const bytes = await exactBytes(length);
+    try {
+      return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) as unknown;
+    } catch {
+      return invalidResponse();
+    }
+  }
+  const iterator = iterate();
+  // Returning an unstarted async generator skips its finally block.
+  return {
+    [Symbol.asyncIterator]() {
+      return {
+        next: () => (cancelled ? Promise.reject(cancellationError()) : iterator.next()),
+        async return() {
+          cancelled = true;
+          activeBody?.error(cancellationError());
+          activeBody = undefined;
+          closeSource();
+          if (!started) return { done: true as const, value: undefined };
+          return iterator.return(undefined);
+        },
+      };
+    },
+  };
+  async function* iterate(): AsyncGenerator<StorageReadResult, void, unknown> {
+    started = true;
+    let currentConsumed = true;
+    let bodyFailure: unknown;
+    try {
+      for (let index = 0; index < keys.length; index++) {
+        const objectKey = keys[index];
+        if (objectKey === undefined) invalidResponse();
+        if (!currentConsumed)
+          throw new CustomerRuntimeGatewayError(
+            "gateway_request_invalid",
+            null,
+            false,
+            "consume or cancel each object's body before reading the next object",
+          );
+        if (bodyFailure !== undefined) throw bodyFailure;
+        const value = await header();
+        const candidate = value as Record<string, unknown>;
+        const record = exactRecord(
+          value,
+          candidate?.["state"] === "found"
+            ? ["index", "objectKey", "state", "contentType", "contentLength", "etag"]
+            : ["index", "objectKey", "state"],
+        );
+        if (record["index"] !== index || record["objectKey"] !== keys[index]) invalidResponse();
+        if (record["state"] === "not_found") {
+          yield Object.freeze({ state: "not_found", objectKey });
+          continue;
+        }
+        if (
+          record["state"] !== "found" ||
+          typeof record["contentType"] !== "string" ||
+          !/^[a-z0-9][a-z0-9!#$&^_.+-]{0,63}\/[a-z0-9][a-z0-9!#$&^_.+-]{0,63}$/u.test(
+            record["contentType"],
+          ) ||
+          !Number.isSafeInteger(record["contentLength"]) ||
+          Number(record["contentLength"]) < 0 ||
+          Number(record["contentLength"]) > 1_073_741_824 ||
+          typeof record["etag"] !== "string" ||
+          !/^[0-9a-f]{32}(?:-[1-9][0-9]{0,4})?$/u.test(record["etag"])
+        )
+          invalidResponse();
+        let remaining = Number(record["contentLength"]);
+        currentConsumed = remaining === 0;
+        const body = new ReadableStream<Uint8Array>(
+          {
+            start(controller) {
+              if (remaining === 0) controller.close();
+              else activeBody = controller;
+            },
+            async pull(controller) {
+              try {
+                const bytes = await chunk(remaining);
+                remaining -= bytes.byteLength;
+                controller.enqueue(bytes);
+                if (remaining === 0) {
+                  currentConsumed = true;
+                  activeBody = undefined;
+                  controller.close();
+                }
+              } catch (error) {
+                bodyFailure = error;
+                currentConsumed = true;
+                activeBody = undefined;
+                controller.error(error);
+                closeSource();
+              }
+            },
+            async cancel() {
+              activeBody = undefined;
+              try {
+                while (remaining > 0) remaining -= (await chunk(remaining)).byteLength;
+                currentConsumed = true;
+              } catch (error) {
+                bodyFailure = error;
+                currentConsumed = true;
+                closeSource();
+                throw error;
+              }
+            },
+          },
+          { highWaterMark: 0 },
+        );
+        yield Object.freeze({
+          state: "found",
+          objectKey,
+          contentType: record["contentType"],
+          contentLength: Number(record["contentLength"]),
+          etag: record["etag"],
+          body,
+        });
+      }
+      if (!currentConsumed)
+        throw new CustomerRuntimeGatewayError(
+          "gateway_request_invalid",
+          null,
+          false,
+          "consume or cancel the last object's body before completing the read",
+        );
+      if (bodyFailure !== undefined) throw bodyFailure;
+      const terminal = exactRecord(await header(), ["state", "count"]);
+      if (
+        terminal["state"] !== "completed" ||
+        terminal["count"] !== keys.length ||
+        offset !== buffer.byteLength
+      )
+        invalidResponse();
+      const end = await timedRead(source, timeoutMilliseconds);
+      if (!end.done) invalidResponse();
+      if (cancelled) throw cancellationError();
+      finished = true;
+    } finally {
+      activeBody?.error(bodyFailure ?? cancellationError());
+      activeBody = undefined;
+      closeSource();
+    }
+  }
+}
+
+/** Control JSON has its own four-MiB budget and never buffers object payloads. */
+export async function readStorageBatchJson(
+  response: Response,
+  timeoutMilliseconds: number,
+  signal?: AbortSignal,
+): Promise<unknown> {
+  const declared = response.headers.get("content-length");
+  if (
+    response.headers.get("content-type")?.split(";", 1)[0]?.trim() !== "application/json" ||
+    (declared !== null && (!/^\d+$/u.test(declared) || Number(declared) > MAX_CONTROL_BYTES)) ||
+    response.body === null
+  ) {
+    void response.body?.cancel().catch(() => undefined);
+    invalidResponse();
+  }
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let length = 0;
+  let finished = false;
+  try {
+    for (;;) {
+      const next = await timedRead(reader, timeoutMilliseconds, signal);
+      if (next.done) {
+        finished = true;
+        break;
+      }
+      if (!(next.value instanceof Uint8Array)) invalidResponse();
+      length += next.value.byteLength;
+      if (length > MAX_CONTROL_BYTES) invalidResponse();
+      chunks.push(next.value);
+    }
+  } finally {
+    if (!finished) void reader.cancel().catch(() => undefined);
+    reader.releaseLock();
+  }
+  const bytes = new Uint8Array(length);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  try {
+    return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) as unknown;
+  } catch {
+    return invalidResponse();
+  }
+}

package/src/packages/customer-runtime/{storage.ts → src/storage.ts}

@@ -1,12 +1,21 @@
 import { CustomerRuntimeGatewayError } from "./errors.js";
 import {
+  configurationInvalid,
   exactRecord,
   fetchWithTimeout,
   invalidResponse,
   postGatewayJson,
+  requireConfiguredUlid,
   validateGatewayConnection,
   type GatewayConnection,
 } from "./http.js";
+import {
+  decodeStorageBatch,
+  readStorageBatchJson,
+  type StorageReadResult,
+  type StorageDeleteResult,
+} from "./storage-batch-stream.js";
+export type { StorageReadResult, StorageDeleteResult } from "./storage-batch-stream.js";
 
 export interface StorageCapability {
   readonly operation: "GET" | "PUT";
@@ -33,11 +42,19 @@ export function createPrivateStorageClient(
 ) {
   const connection = validateGatewayConnection(configuration);
   if (typeof configuration.capabilityFetch !== "function") {
-    throw new CustomerRuntimeGatewayError("gateway_configuration_invalid", null, false);
+    configurationInvalid("capabilityFetch is not a function; pass (request) => fetch(request)");
   }
   const capabilityFetch = configuration.capabilityFetch;
-  const projectId = requireUlid(configuration.projectId);
-  const environmentId = requireUlid(configuration.environmentId);
+  const projectId = requireConfiguredUlid(
+    configuration.projectId,
+    "projectId",
+    "OHMYHOST_PROJECT_ID",
+  );
+  const environmentId = requireConfiguredUlid(
+    configuration.environmentId,
+    "environmentId",
+    "OHMYHOST_ENVIRONMENT_ID",
+  );
   const authenticatedBody = Object.freeze({ projectId, environmentId });
 
   async function reserveUpload(
@@ -59,7 +76,7 @@ export function createPrivateStorageClient(
       8 * 1024,
       input.idempotencyKey,
     );
-    if (response.status !== 200) throwStorageError(response.status, response.body);
+    if (response.status !== 200) throwStorageError(response.status, response.body, environmentId);
     const record = response.body as Record<string, unknown>;
     if (record?.["state"] === "completed") {
       const completed = exactRecord(response.body, ["state", "transfer_id"]);
@@ -88,7 +105,7 @@ export function createPrivateStorageClient(
       8 * 1024,
     );
     if (response.status !== 200 && response.status !== 202) {
-      throwStorageError(response.status, response.body);
+      throwStorageError(response.status, response.body, environmentId);
     }
     const record = exactRecord(response.body, ["state", "transfer_id"]);
     if (record["state"] !== "completed" && record["state"] !== "pending") invalidResponse();
@@ -109,7 +126,7 @@ export function createPrivateStorageClient(
       },
       8 * 1024,
     );
-    if (response.status !== 200) throwStorageError(response.status, response.body);
+    if (response.status !== 200) throwStorageError(response.status, response.body, environmentId);
     const record = exactRecord(response.body, ["state", "capability"]);
     if (record["state"] !== "ready") invalidResponse();
     return parseCapability(record["capability"], "GET", objectKey);
@@ -124,7 +141,7 @@ export function createPrivateStorageClient(
       8 * 1024,
       input.idempotencyKey,
     );
-    if (response.status !== 200) throwStorageError(response.status, response.body);
+    if (response.status !== 200) throwStorageError(response.status, response.body, environmentId);
     const record = exactRecord(response.body, ["state", "deletion_id"]);
     if (record["state"] !== "completed") invalidResponse();
     return Object.freeze({
@@ -151,13 +168,177 @@ export function createPrivateStorageClient(
       }),
       connection.timeoutMilliseconds,
     );
+    // The create-only PUT answers 412 when an earlier attempt already stored these bytes under
+    // this reservation; recording them is the retry's whole remaining work.
+    if (uploaded.status === 412) return completeUpload(reservation.transferId);
     if (uploaded.status < 200 || uploaded.status >= 300) {
-      throw new CustomerRuntimeGatewayError("storage_upload_rejected", uploaded.status, false);
+      throw new CustomerRuntimeGatewayError(
+        "storage_upload_rejected",
+        uploaded.status,
+        uploaded.status >= 500 || uploaded.status === 429,
+        `the signed upload answered ${String(uploaded.status)}. For 5xx or 429, repeat upload() with the same idempotencyKey and objectKey.`,
+      );
     }
     return completeUpload(reservation.transferId);
   }
 
-  return Object.freeze({ reserveUpload, completeUpload, createSignedRead, deleteObject, upload });
+  async function batchRequest(
+    path: string,
+    body: Readonly<Record<string, unknown>>,
+    signal?: AbortSignal,
+  ) {
+    const serialized = JSON.stringify({ ...authenticatedBody, ...body });
+    if (new TextEncoder().encode(serialized).byteLength > 4 * 1024 * 1024) invalidRequest();
+    return fetchWithTimeout(
+      connection.fetch,
+      new Request(new URL(path, connection.endpoint), {
+        method: "POST",
+        headers: { authorization: `Bearer ${connection.key}`, "content-type": "application/json" },
+        body: serialized,
+        ...(signal === undefined ? {} : { signal }),
+      }),
+      connection.timeoutMilliseconds,
+    );
+  }
+  async function readMany(input: {
+    readonly objectKeys: readonly string[];
+  }): Promise<AsyncIterable<StorageReadResult>> {
+    if (
+      typeof input !== "object" ||
+      input === null ||
+      Object.keys(input).join(",") !== "objectKeys"
+    )
+      invalidRequest();
+    const objectKeys = validateBatchKeys(input.objectKeys);
+    const response = await batchRequest("/v1/downloads/batch", { objectKeys });
+    if (response.status !== 200)
+      throwStorageError(
+        response.status,
+        await readStorageBatchJson(response, connection.timeoutMilliseconds),
+        environmentId,
+      );
+    return decodeStorageBatch(response, objectKeys, connection.timeoutMilliseconds);
+  }
+  async function deleteMany(input: {
+    readonly objects: readonly { readonly objectKey: string; readonly idempotencyKey: string }[];
+  }): Promise<readonly StorageDeleteResult[]> {
+    if (
+      typeof input !== "object" ||
+      input === null ||
+      Object.keys(input).join(",") !== "objects" ||
+      !Array.isArray(input.objects) ||
+      input.objects.length === 0 ||
+      input.objects.length > 4000
+    )
+      invalidRequest();
+    const objects = input.objects.map((item) => {
+      if (
+        typeof item !== "object" ||
+        item === null ||
+        Object.keys(item).sort().join(",") !== "idempotencyKey,objectKey" ||
+        typeof item.objectKey !== "string" ||
+        typeof item.idempotencyKey !== "string"
+      )
+        invalidRequest();
+      validateStorageMutation(item.idempotencyKey, item.objectKey);
+      return { objectKey: item.objectKey, idempotencyKey: item.idempotencyKey };
+    });
+    validateBatchKeys(objects.map((item) => item.objectKey));
+    if (new Set(objects.map((item) => item.idempotencyKey)).size !== objects.length)
+      invalidRequest();
+    const controller = new AbortController();
+    const deadline = Date.now() + connection.timeoutMilliseconds;
+    const timer = setTimeout(() => controller.abort(), connection.timeoutMilliseconds);
+    try {
+      const response = await batchRequest("/v1/deletions/batch", { objects }, controller.signal);
+      const value = await readStorageBatchJson(
+        response,
+        connection.timeoutMilliseconds,
+        controller.signal,
+      );
+      if (response.status !== 200) throwStorageError(response.status, value, environmentId);
+      const record = exactRecord(value, ["state", "results"]);
+      if (!Array.isArray(record["results"]) || record["results"].length !== objects.length)
+        invalidResponse();
+      const results = record["results"].map((value, index): StorageDeleteResult => {
+        const candidate = value as Record<string, unknown>;
+        const item = exactRecord(
+          value,
+          candidate?.["state"] === "completed"
+            ? ["object_key", "state", "deletion_id"]
+            : candidate?.["state"] === "pending"
+              ? ["object_key", "state", "retry_at"]
+              : ["object_key", "state", "code", "retryable"],
+        );
+        const requested = objects[index];
+        if (requested === undefined || item["object_key"] !== requested.objectKey)
+          invalidResponse();
+        const objectKey = requested.objectKey;
+        if (item["state"] === "completed")
+          return Object.freeze({
+            state: "completed",
+            objectKey,
+            deletionId: requireUlid(item["deletion_id"]),
+          });
+        if (item["state"] === "pending") {
+          return Object.freeze({
+            state: "pending",
+            objectKey,
+            retryAt: requireRetryAt(item["retry_at"]),
+          });
+        }
+        if (
+          item["state"] !== "failed" ||
+          (item["code"] !== "object_not_found" && item["code"] !== "storage_gateway_unavailable") ||
+          typeof item["retryable"] !== "boolean" ||
+          (item["code"] === "object_not_found" && item["retryable"] !== false)
+        )
+          invalidResponse();
+        return Object.freeze({
+          state: "failed",
+          objectKey,
+          code: item["code"],
+          retryable: item["retryable"],
+        });
+      });
+      if (
+        record["state"] !==
+        (results.every((item) => item.state === "completed") ? "completed" : "partial")
+      )
+        invalidResponse();
+      if (controller.signal.aborted || Date.now() >= deadline) {
+        controller.abort();
+        throw new CustomerRuntimeGatewayError("gateway_timeout", null, true);
+      }
+      return Object.freeze(results);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+  return Object.freeze({
+    reserveUpload,
+    completeUpload,
+    createSignedRead,
+    deleteObject,
+    upload,
+    readMany,
+    deleteMany,
+  });
+}
+
+function validateBatchKeys(value: readonly string[]): readonly string[] {
+  if (
+    !Array.isArray(value) ||
+    value.length === 0 ||
+    value.length > 4000 ||
+    new Set(value).size !== value.length
+  )
+    invalidRequest();
+  for (const key of value) {
+    if (typeof key !== "string") invalidRequest();
+    validateObjectKey(key);
+  }
+  return Object.freeze(Array.from(value));
 }
 
 function parseCapability(value: unknown, operation: "GET" | "PUT", objectKey: string) {
@@ -214,12 +395,18 @@ function exactHeaders(value: unknown): Readonly<Record<string, string>> {
   return Object.freeze(Object.fromEntries(entries) as Record<string, string>);
 }
 
-function throwStorageError(status: number, body: unknown): never {
+function throwStorageError(status: number, body: unknown, environmentId: string): never {
+  const candidate = body as Record<string, unknown>;
   const record = exactRecord(
     body,
-    typeof body === "object" && body !== null && "transfer_id" in body
-      ? ["code", "transfer_id"]
-      : ["code"],
+    candidate?.["code"] === "storage_delete_pending"
+      ? ["code", "retry_at"]
+      : candidate?.["code"] === "data_change_in_progress" ||
+          candidate?.["code"] === "storage_binding_data_changed"
+        ? ["code"]
+        : typeof body === "object" && body !== null && "transfer_id" in body
+          ? ["code", "transfer_id"]
+          : ["code"],
   );
   const code = record["code"];
   if (status === 401 && code === "unauthorized") {
@@ -232,6 +419,31 @@ function throwStorageError(status: number, body: unknown): never {
     requireUlid(record["transfer_id"]);
     throw new CustomerRuntimeGatewayError("storage_upload_expired", status, false);
   }
+  if (status === 409 && code === "data_change_in_progress") {
+    throw new CustomerRuntimeGatewayError(
+      "data_change_in_progress",
+      status,
+      true,
+      "a data change is running for this project; wait for that existing operation to finish, then repeat the same storage request with the same idempotency keys",
+    );
+  }
+  if (status === 409 && code === "storage_binding_data_changed") {
+    throw new CustomerRuntimeGatewayError(
+      "storage_binding_data_changed",
+      status,
+      false,
+      `environment ${environmentId} now uses a different data area; create a new deployment plan and redeploy that environment. Repeating this request will not update this deployment's binding`,
+    );
+  }
+  if (status === 409 && code === "storage_delete_pending") {
+    const retryAt = requireRetryAt(record["retry_at"]);
+    throw new CustomerRuntimeGatewayError(
+      "storage_delete_pending",
+      status,
+      true,
+      `a previously accepted upload or deletion is still settling; wait until ${retryAt}, then repeat deleteObject() with the same objectKey and idempotencyKey`,
+    );
+  }
   if (status === 409 && typeof code === "string") {
     throw new CustomerRuntimeGatewayError("storage_conflict", status, false);
   }
@@ -244,6 +456,17 @@ function throwStorageError(status: number, body: unknown): never {
   return invalidResponse();
 }
 
+function requireRetryAt(value: unknown): string {
+  if (
+    typeof value !== "string" ||
+    !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(value) ||
+    !Number.isFinite(Date.parse(value)) ||
+    new Date(value).toISOString() !== value
+  )
+    invalidResponse();
+  return value;
+}
+
 function validateStorageMutation(idempotencyKey: string, objectKey: string): void {
   if (!/^[A-Za-z0-9._:-]{1,128}$/u.test(idempotencyKey)) invalidRequest();
   validateObjectKey(objectKey);