@edgestore/sdk 1.0.0-next.1 → 1.0.0-next.2

package/src/internal/managementUploadOperations.ts

@@ -0,0 +1,99 @@
+import type { OperationBody, OperationResult } from './operationTypes';
+import type { Transport } from './transport';
+
+type ProjectUploadInput = {
+  project: string;
+  uploadId: string;
+  signal?: AbortSignal;
+};
+
+export type ManagementUploadOperations = ReturnType<
+  typeof createManagementUploadOperations
+>;
+
+export function createManagementUploadOperations(transport: Transport) {
+  return {
+    request: (
+      input: OperationBody<'v2.management.uploads.request'> & {
+        project: string;
+        bucket: string;
+        signal?: AbortSignal;
+      },
+    ): Promise<OperationResult<'v2.management.uploads.request'>> => {
+      const { project, bucket, signal, ...body } = input;
+      return transport.execute((client) =>
+        client.POST(
+          '/management/projects/{projectRef}/buckets/{bucketName}/uploads',
+          {
+            params: { path: { projectRef: project, bucketName: bucket } },
+            body,
+            signal,
+          },
+        ),
+      );
+    },
+    get: (
+      input: ProjectUploadInput,
+    ): Promise<OperationResult<'v2.management.uploads.get'>> =>
+      transport.execute((client) =>
+        client.GET('/management/projects/{projectRef}/uploads/{uploadId}', {
+          params: {
+            path: { projectRef: input.project, uploadId: input.uploadId },
+          },
+          signal: input.signal,
+        }),
+      ),
+    getWithResponse: (input: ProjectUploadInput) =>
+      transport.executeWithResponse((client) =>
+        client.GET('/management/projects/{projectRef}/uploads/{uploadId}', {
+          params: {
+            path: { projectRef: input.project, uploadId: input.uploadId },
+          },
+          signal: input.signal,
+        }),
+      ),
+    cancel: (
+      input: ProjectUploadInput,
+    ): Promise<OperationResult<'v2.management.uploads.cancel'>> =>
+      transport.execute((client) =>
+        client.DELETE('/management/projects/{projectRef}/uploads/{uploadId}', {
+          params: {
+            path: { projectRef: input.project, uploadId: input.uploadId },
+          },
+          signal: input.signal,
+        }),
+      ),
+    createParts: (
+      input: ProjectUploadInput &
+        OperationBody<'v2.management.uploads.parts.create'>,
+    ): Promise<OperationResult<'v2.management.uploads.parts.create'>> => {
+      const { project, uploadId, signal, ...body } = input;
+      return transport.execute((client) =>
+        client.POST(
+          '/management/projects/{projectRef}/uploads/{uploadId}/parts',
+          {
+            params: { path: { projectRef: project, uploadId } },
+            body,
+            signal,
+          },
+        ),
+      );
+    },
+    completeMultipart: (
+      input: ProjectUploadInput &
+        OperationBody<'v2.management.uploads.multipart.complete'>,
+    ): Promise<OperationResult<'v2.management.uploads.multipart.complete'>> => {
+      const { project, uploadId, signal, ...body } = input;
+      return transport.execute((client) =>
+        client.POST(
+          '/management/projects/{projectRef}/uploads/{uploadId}/complete',
+          {
+            params: { path: { projectRef: project, uploadId } },
+            body,
+            signal,
+          },
+        ),
+      );
+    },
+  };
+}

package/src/internal/multipartUpload.ts

@@ -5,8 +5,8 @@ import type { Transport } from './transport';
 import { throwIfAborted } from './uploadRetry';
 import { putWithRetry } from './uploadTransfer';
 
-type CompletedPart = { partNumber: number; eTag: string };
-type UploadPart = { partNumber: number; signedUrl: string };
+export type CompletedUploadPart = { partNumber: number; eTag: string };
+export type SignedUploadPart = { partNumber: number; signedUrl: string };
 type ProgressHandler = RuntimeUploadInput['onProgress'];
 
 export async function uploadParts(
@@ -14,14 +14,14 @@ export async function uploadParts(
   options: {
     body: Blob;
     uploadId: string;
-    parts: UploadPart[];
+    parts: SignedUploadPart[];
     partSizeBytes: number;
     concurrency: number;
     signal?: AbortSignal;
     onProgress?: ProgressHandler;
   },
-): Promise<CompletedPart[]> {
-  const completed: CompletedPart[] = Array(options.parts.length);
+): Promise<CompletedUploadPart[]> {
+  const completed: CompletedUploadPart[] = Array(options.parts.length);
   let nextIndex = 0;
   let transferredBytes = 0;
   const workerCount = Math.min(options.concurrency, options.parts.length);
@@ -70,14 +70,14 @@ export async function uploadStreamParts(
     stream: ReadableStream<Uint8Array>;
     totalBytes: number;
     uploadId: string;
-    parts: UploadPart[];
+    parts: SignedUploadPart[];
     partSizeBytes: number;
     signal?: AbortSignal;
     onProgress?: ProgressHandler;
   },
-): Promise<CompletedPart[]> {
+): Promise<CompletedUploadPart[]> {
   const reader = createSizedStreamReader(options.stream);
-  const completed: CompletedPart[] = [];
+  const completed: CompletedUploadPart[] = [];
   let transferredBytes = 0;
 
   try {

package/src/internal/transport.ts

@@ -8,6 +8,7 @@ import {
   EdgeStoreApiError,
   EdgeStoreError,
   EdgeStoreNetworkError,
+  EdgeStoreTimeoutError,
 } from '../errors';
 import type { paths } from '../generated/api-v2';
 import {
@@ -55,18 +56,26 @@ export function createTransport(options: TransportOptions): Transport {
   }
   const client = createClient<paths>({
     baseUrl: normalizeApiUrl(options.apiUrl ?? DEFAULT_API_URL),
-    fetch: (request) =>
-      fetch(
-        new Request(request, {
-          signal:
-            controlTimeoutMs === 0
-              ? request.signal
-              : AbortSignal.any([
-                  request.signal,
-                  AbortSignal.timeout(controlTimeoutMs),
-                ]),
-        }),
-      ),
+    fetch: async (request) => {
+      if (controlTimeoutMs === 0) return await fetch(request);
+
+      const timeoutSignal = AbortSignal.timeout(controlTimeoutMs);
+      try {
+        return await fetch(
+          new Request(request, {
+            signal: AbortSignal.any([request.signal, timeoutSignal]),
+          }),
+        );
+      } catch (error) {
+        if (timeoutSignal.aborted && !request.signal.aborted) {
+          throw new EdgeStoreTimeoutError(
+            `The EdgeStore API request timed out after ${controlTimeoutMs}ms.`,
+            { cause: error },
+          );
+        }
+        throw error;
+      }
+    },
     headers: {
       authorization,
       'user-agent': `${EDGE_STORE_PACKAGE_NAME}/${EDGE_STORE_PACKAGE_VERSION}`,

package/src/internal/uploadLifecycle.ts

@@ -0,0 +1,154 @@
+import {
+  EdgeStoreUploadCleanupError,
+  EdgeStoreUploadProcessingTimeoutError,
+} from '../errors';
+import type { MultipartUploadPlan } from '../multipartPlan';
+import {
+  DEFAULT_MULTIPART_CONCURRENCY,
+  type UploadProgress,
+} from '../uploadTypes';
+import {
+  uploadParts,
+  uploadStreamParts,
+  type CompletedUploadPart,
+  type SignedUploadPart,
+} from './multipartUpload';
+import type { Transport } from './transport';
+import { waitForUploadProcessing } from './uploadProcessing';
+import {
+  reportUploadProgress,
+  type PreparedUploadSource,
+} from './uploadSource';
+import { putWithRetry } from './uploadTransfer';
+import { getPositiveInteger } from './uploadValidation';
+
+type RequestedUpload = {
+  upload:
+    | { id: string; kind: 'single'; signedUrl: string }
+    | { id: string; kind: 'multipart'; parts: SignedUploadPart[] };
+};
+
+type UploadState = { upload: { status: string } };
+
+type UploadLifecycleOperations<TResult extends UploadState> = {
+  completeMultipart(input: {
+    uploadId: string;
+    parts: CompletedUploadPart[];
+    signal?: AbortSignal;
+  }): Promise<unknown>;
+  get(input: {
+    uploadId: string;
+    signal?: AbortSignal;
+  }): Promise<{ data: TResult; response: Response }>;
+  cancel(input: { uploadId: string }): Promise<unknown>;
+};
+
+export async function executeUploadLifecycle<
+  TResult extends UploadState,
+>(options: {
+  transport: Transport;
+  requested: RequestedUpload;
+  prepared: PreparedUploadSource;
+  multipartPlan: MultipartUploadPlan | null;
+  multipartConcurrency?: number;
+  defaultMultipartConcurrency?: number;
+  processingTimeoutMs: number;
+  signal?: AbortSignal;
+  onProgress?: (progress: UploadProgress) => void;
+  operations: UploadLifecycleOperations<TResult>;
+  cleanupPolicy: 'preserve-original' | 'report-failure';
+}): Promise<Extract<TResult, { upload: { status: 'completed' } }>> {
+  const {
+    transport,
+    requested,
+    prepared,
+    multipartPlan,
+    processingTimeoutMs,
+    signal,
+    onProgress,
+    operations,
+  } = options;
+  const uploadId = requested.upload.id;
+  const totalBytes = prepared.sizeBytes;
+
+  try {
+    reportUploadProgress(onProgress, {
+      transferredBytes: 0,
+      totalBytes,
+      phase: 'uploading',
+    });
+
+    if (prepared.kind === 'body' && requested.upload.kind === 'single') {
+      await putWithRetry(transport, {
+        uploadId,
+        url: requested.upload.signedUrl,
+        body: prepared.body,
+        signal,
+      });
+      reportUploadProgress(onProgress, {
+        transferredBytes: totalBytes,
+        totalBytes,
+        phase: 'uploading',
+      });
+    } else {
+      if (requested.upload.kind !== 'multipart' || !multipartPlan) {
+        throw new TypeError(
+          'The API returned an upload plan that does not match the request.',
+        );
+      }
+      const transfer = {
+        uploadId,
+        parts: requested.upload.parts,
+        partSizeBytes: multipartPlan.partSizeBytes,
+        signal,
+        onProgress,
+      };
+      const parts =
+        prepared.kind === 'stream'
+          ? await uploadStreamParts(transport, {
+              ...transfer,
+              stream: prepared.stream,
+              totalBytes,
+            })
+          : await uploadParts(transport, {
+              ...transfer,
+              body: prepared.body,
+              concurrency: getPositiveInteger(
+                options.multipartConcurrency,
+                options.defaultMultipartConcurrency ??
+                  DEFAULT_MULTIPART_CONCURRENCY,
+                'multipart.concurrency',
+              ),
+            });
+      await operations.completeMultipart({ uploadId, parts, signal });
+    }
+
+    reportUploadProgress(onProgress, {
+      transferredBytes: totalBytes,
+      totalBytes,
+      phase: 'processing',
+    });
+    return await waitForUploadProcessing<TResult>({
+      uploadId,
+      signal,
+      timeoutMs: processingTimeoutMs,
+      get: (requestSignal) =>
+        operations.get({ uploadId, signal: requestSignal }),
+    });
+  } catch (error) {
+    if (error instanceof EdgeStoreUploadProcessingTimeoutError) throw error;
+    try {
+      await operations.cancel({ uploadId });
+    } catch (cleanupError) {
+      if (options.cleanupPolicy === 'report-failure') {
+        throw new EdgeStoreUploadCleanupError({
+          message: `Automatic cancellation of upload ${uploadId} failed.`,
+          uploadId,
+          uploadCause: error,
+          cleanupCause: cleanupError,
+        });
+      }
+    }
+    throw error;
+  }
+}

package/src/internal/uploadProcessing.ts

@@ -0,0 +1,48 @@
+import {
+  EdgeStoreUploadCanceledError,
+  EdgeStoreUploadProcessingTimeoutError,
+} from '../errors';
+import { getRetryAfterMs, retryOperation, sleep } from './uploadRetry';
+
+type UploadState = { upload: { status: string } };
+
+export async function waitForUploadProcessing<
+  TResult extends UploadState,
+>(options: {
+  uploadId: string;
+  signal?: AbortSignal;
+  timeoutMs: number;
+  get(signal?: AbortSignal): Promise<{ data: TResult; response: Response }>;
+}): Promise<Extract<TResult, { upload: { status: 'completed' } }>> {
+  const deadline = Date.now() + options.timeoutMs;
+  const timeoutError = () =>
+    new EdgeStoreUploadProcessingTimeoutError(
+      'Timed out while EdgeStore was processing the upload.',
+      options.uploadId,
+    );
+
+  while (true) {
+    const { data, response } = await retryOperation(
+      (signal) => options.get(signal),
+      {
+        signal: options.signal,
+        deadline,
+        timeoutError,
+      },
+    );
+
+    if (data.upload.status === 'completed') {
+      return data as Extract<TResult, { upload: { status: 'completed' } }>;
+    }
+    if (data.upload.status === 'canceled') {
+      throw new EdgeStoreUploadCanceledError(
+        'The EdgeStore upload was canceled.',
+        options.uploadId,
+      );
+    }
+
+    const delayMs = getRetryAfterMs(response) ?? 1000;
+    if (Date.now() + delayMs > deadline) throw timeoutError();
+    await sleep(delayMs, options.signal);
+  }
+}

package/src/internal/uploadSource.ts

@@ -0,0 +1,104 @@
+import type {
+  UploadMetadataValue,
+  UploadProgress,
+  UploadSource,
+  UploadStreamSource,
+} from '../uploadTypes';
+
+type PreparedUploadDetails = {
+  sizeBytes: number;
+  fileName?: string;
+  mimeType?: string;
+};
+
+export type PreparedUploadSource =
+  | (PreparedUploadDetails & { kind: 'body'; body: Blob })
+  | (PreparedUploadDetails & {
+      kind: 'stream';
+      stream: ReadableStream<Uint8Array>;
+    });
+
+export function prepareUploadSource(
+  source: UploadSource,
+): PreparedUploadSource {
+  if (typeof source === 'string') {
+    const body = new Blob([source], { type: 'text/plain' });
+    return {
+      kind: 'body',
+      body,
+      sizeBytes: body.size,
+      mimeType: body.type,
+    };
+  }
+  if (isStreamSource(source)) {
+    if (!Number.isSafeInteger(source.sizeBytes) || source.sizeBytes < 0) {
+      throw new RangeError('source.sizeBytes must be a non-negative integer.');
+    }
+    return {
+      kind: 'stream',
+      stream: source.stream,
+      sizeBytes: source.sizeBytes,
+    };
+  }
+  if (source instanceof Blob) {
+    return {
+      kind: 'body',
+      body: source,
+      sizeBytes: source.size,
+      fileName:
+        'name' in source && typeof source.name === 'string'
+          ? source.name
+          : undefined,
+      mimeType: source.type || undefined,
+    };
+  }
+  if (source instanceof ArrayBuffer) {
+    const body = new Blob([source]);
+    return { kind: 'body', body, sizeBytes: body.size };
+  }
+  const body = new Blob([
+    Uint8Array.from(
+      new Uint8Array(source.buffer, source.byteOffset, source.byteLength),
+    ),
+  ]);
+  return { kind: 'body', body, sizeBytes: body.size };
+}
+
+export function normalizeUploadMetadata(
+  metadata?: Record<string, UploadMetadataValue>,
+): Record<string, string> | undefined {
+  if (!metadata) return undefined;
+  const entries = Object.entries(metadata).flatMap(([key, value]) =>
+    value === null || value === undefined ? [] : [[key, String(value)]],
+  );
+  return entries.length ? Object.fromEntries(entries) : undefined;
+}
+
+export function reportUploadProgress(
+  onProgress: ((progress: UploadProgress) => void) | undefined,
+  progress: Omit<UploadProgress, 'percentage'>,
+): void {
+  const { transferredBytes, totalBytes, phase } = progress;
+  onProgress?.({
+    ...progress,
+    percentage:
+      totalBytes === 0
+        ? phase === 'preparing'
+          ? 0
+          : 100
+        : Math.round((transferredBytes / totalBytes) * 10_000) / 100,
+  });
+}
+
+function isStreamSource(source: UploadSource): source is UploadStreamSource {
+  const stream =
+    typeof source === 'object' && source !== null && 'stream' in source
+      ? Reflect.get(source, 'stream')
+      : undefined;
+  return (
+    typeof stream === 'object' &&
+    stream !== null &&
+    'getReader' in stream &&
+    typeof Reflect.get(stream, 'getReader') === 'function'
+  );
+}

package/src/internal/uploadTransfer.ts

@@ -79,7 +79,8 @@ export async function putWithRetry(
         isRetryable: (error) =>
           error instanceof SignedUploadResponseError
             ? isRetryableStatus(error.status)
-            : !(error instanceof EdgeStoreUploadError),
+            : !findBlobReadError(error) &&
+              !(error instanceof EdgeStoreUploadError),
         getRetryDelayMs: (error) =>
           error instanceof SignedUploadResponseError
             ? error.retryAfterMs
@@ -89,6 +90,8 @@ export async function putWithRetry(
   } catch (error) {
     if (error instanceof EdgeStoreAbortError) throw error;
     if (error instanceof EdgeStoreUploadError) throw error;
+    const blobReadError = findBlobReadError(error);
+    if (blobReadError) throw blobReadError;
     if (error instanceof SignedUploadResponseError) {
       throw new EdgeStoreUploadError(error.message, options.uploadId);
     }
@@ -99,6 +102,22 @@ export async function putWithRetry(
   }
 }
 
+function findBlobReadError(error: unknown): DOMException | undefined {
+  const seen = new Set<unknown>();
+  let current = error;
+  while (current instanceof Error && !seen.has(current)) {
+    if (
+      current instanceof DOMException &&
+      current.name === 'NotReadableError'
+    ) {
+      return current;
+    }
+    seen.add(current);
+    current = current.cause;
+  }
+  return undefined;
+}
+
 async function cancelResponseBody(
   body: ReadableStream<Uint8Array> | null,
 ): Promise<void> {

package/src/managementClient.ts

@@ -8,6 +8,7 @@ import {
   createManagementResourceClient,
   type ManagementResourceClient,
 } from './managementResources';
+import type { UploadDefaults } from './uploadTypes';
 
 /** Complete administrative client available to Bearer credentials. */
 export type ManagementClient = ManagementResourceClient &
@@ -19,9 +20,12 @@ export type ManagementClient = ManagementResourceClient &
     }): Promise<OperationResult<'v2.whoami'>>;
   };
 
-export function createManagementClient(transport: Transport): ManagementClient {
+export function createManagementClient(
+  transport: Transport,
+  uploadDefaults?: UploadDefaults,
+): ManagementClient {
   return {
-    ...createManagementResourceClient(transport),
+    ...createManagementResourceClient(transport, uploadDefaults),
     ...createManagementAccessClient(transport),
     whoami: (options) =>
       transport.execute((client) =>

package/src/managementResources.ts

@@ -1,3 +1,4 @@
+import { createManagementUploadOperations } from './internal/managementUploadOperations';
 import type {
   OperationBody,
   OperationId,
@@ -5,6 +6,12 @@ import type {
   OperationResult,
 } from './internal/operationTypes';
 import type { Transport } from './internal/transport';
+import {
+  uploadManagementFile,
+  type ManagementUploadInput,
+  type ManagementUploadResult,
+} from './managementUpload';
+import type { UploadDefaults } from './uploadTypes';
 
 type CallOptions = {
   /** Cancels the request. */
@@ -126,6 +133,8 @@ export type ManagementResourceClient = {
     ): Promise<Result<'v2.management.files.delete'>>;
   };
   uploads: {
+    /** Transfers a file and waits for server-side processing to complete. */
+    upload(input: ManagementUploadInput): Promise<ManagementUploadResult>;
     /** Requests signed upload destination(s) without transferring data. */
     request(
       input: BucketInput &
@@ -157,7 +166,9 @@ export type ManagementResourceClient = {
 
 export function createManagementResourceClient(
   transport: Transport,
+  uploadDefaults: UploadDefaults = {},
 ): ManagementResourceClient {
+  const uploadOperations = createManagementUploadOperations(transport);
   return {
     projects: {
       list: ({ account, signal }) =>
@@ -323,58 +334,17 @@ export function createManagementResourceClient(
         ),
     },
     uploads: {
-      request: ({ project, bucket, signal, ...body }) =>
-        transport.execute((client) =>
-          client.POST(
-            '/management/projects/{projectRef}/buckets/{bucketName}/uploads',
-            {
-              params: {
-                path: { projectRef: project, bucketName: bucket },
-              },
-              body,
-              signal,
-            },
-          ),
-        ),
-      get: ({ project, uploadId, signal }) =>
-        transport.execute((client) =>
-          client.GET('/management/projects/{projectRef}/uploads/{uploadId}', {
-            params: { path: { projectRef: project, uploadId } },
-            signal,
-          }),
-        ),
-      cancel: ({ project, uploadId, signal }) =>
-        transport.execute((client) =>
-          client.DELETE(
-            '/management/projects/{projectRef}/uploads/{uploadId}',
-            {
-              params: { path: { projectRef: project, uploadId } },
-              signal,
-            },
-          ),
-        ),
-      createParts: ({ project, uploadId, signal, ...body }) =>
-        transport.execute((client) =>
-          client.POST(
-            '/management/projects/{projectRef}/uploads/{uploadId}/parts',
-            {
-              params: { path: { projectRef: project, uploadId } },
-              body,
-              signal,
-            },
-          ),
-        ),
-      completeMultipart: ({ project, uploadId, signal, ...body }) =>
-        transport.execute((client) =>
-          client.POST(
-            '/management/projects/{projectRef}/uploads/{uploadId}/complete',
-            {
-              params: { path: { projectRef: project, uploadId } },
-              body,
-              signal,
-            },
-          ),
+      upload: (input) =>
+        uploadManagementFile(
+          { transport, operations: uploadOperations },
+          input,
+          uploadDefaults,
         ),
+      request: uploadOperations.request,
+      get: uploadOperations.get,
+      cancel: uploadOperations.cancel,
+      createParts: uploadOperations.createParts,
+      completeMultipart: uploadOperations.completeMultipart,
     },
   };
 }