@eigenpal/sdk 0.16.3 → 0.16.5

package/src/lib/fetch-body.ts

@@ -0,0 +1,92 @@
+/** Request init for Node/Bun fetch when streaming a Node readable body. */
+export type StreamableRequestInit = RequestInit & { duplex?: 'half' };
+
+/** True when `body` is a Node.js readable stream (fs.createReadStream, etc.). */
+export function isNodeReadableStream(body: unknown): body is NodeJS.ReadableStream {
+  return typeof (body as { pipe?: unknown }).pipe === 'function';
+}
+
+/**
+ * Copy a byte view into a standalone ArrayBuffer accepted by DOM `BodyInit`.
+ * Avoids TS generic mismatches on `Uint8Array<ArrayBufferLike>`.
+ */
+export function byteViewToArrayBuffer(view: Uint8Array): ArrayBuffer {
+  if (
+    view.buffer instanceof ArrayBuffer &&
+    view.byteOffset === 0 &&
+    view.byteLength === view.buffer.byteLength
+  ) {
+    return view.buffer;
+  }
+  const copy = new Uint8Array(view.byteLength);
+  copy.set(view);
+  return copy.buffer;
+}
+
+/** Convert bytes or a Node readable into a fetch-compatible PUT body. */
+export function toPutBody(body: Uint8Array | NodeJS.ReadableStream): BodyInit {
+  if (isNodeReadableStream(body)) {
+    // Node/Bun fetch accepts Node readables with `duplex: 'half'`.
+    return body as unknown as BodyInit;
+  }
+  return byteViewToArrayBuffer(body);
+}
+
+export function putRequestInit(
+  body: Uint8Array | NodeJS.ReadableStream,
+  headers: HeadersInit,
+  signal?: AbortSignal
+): StreamableRequestInit {
+  const stream = isNodeReadableStream(body);
+  return {
+    method: 'PUT',
+    headers,
+    body: toPutBody(body),
+    signal,
+    ...(stream ? { duplex: 'half' as const } : {}),
+  };
+}
+
+type NodeReadableState = NodeJS.ReadableStream & {
+  readableEnded?: boolean;
+  destroyed?: boolean;
+  destroy?: (error?: Error) => void;
+};
+
+/** Drop a Node readable that never started sending (fetch failed before upload). */
+export function destroyUnreadNodeReadable(stream: NodeJS.ReadableStream): void {
+  const readable = stream as NodeReadableState;
+  if (!readable.readableEnded && !readable.destroyed && typeof readable.destroy === 'function') {
+    readable.destroy();
+  }
+}
+
+/**
+ * Block until every byte of a Node readable PUT body has been consumed.
+ * Real fetch/undici should drain the stream before resolving; mocks that
+ * return early leave bytes unread and must be drained here so callers do
+ * not destroy the stream while a retry or background upload is still running.
+ */
+export async function ensureNodeReadableFullySent(
+  stream: NodeJS.ReadableStream,
+  expectedByteLength?: number
+): Promise<number> {
+  const readable = stream as NodeReadableState;
+  if (readable.readableEnded) {
+    return expectedByteLength ?? 0;
+  }
+  if (readable.destroyed) {
+    throw new Error('Upload stream was destroyed before the request body finished sending');
+  }
+
+  let bytes = 0;
+  for await (const chunk of stream as AsyncIterable<Buffer | Uint8Array | string>) {
+    bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length;
+  }
+  if (expectedByteLength !== undefined && bytes !== expectedByteLength) {
+    throw new Error(
+      `Upload sent ${bytes} byte(s) but ${expectedByteLength} were expected for this part`
+    );
+  }
+  return bytes;
+}

package/src/lib/files.ts

@@ -51,9 +51,33 @@ export interface FileDescriptor {
   mimeType?: string;
 }
 
+/**
+ * Disk path that can be stat'd and re-opened per part for resumable multipart.
+ */
+export interface PathFileInput {
+  path: string;
+  filename?: string;
+  mimeType?: string;
+}
+
+/**
+ * Replayable byte-range factory for Node streams that are not a disk path.
+ * `open` must return a fresh body for `[start, start+length)` on every call.
+ */
+export interface StreamFactoryFileInput {
+  size: number;
+  filename: string;
+  mimeType?: string;
+  open: (
+    start: number,
+    length: number
+  ) => Blob | Uint8Array | ReadableStream | NodeJS.ReadableStream;
+}
+
 /**
  * A Node readable stream — typically `fs.createReadStream('contract.pdf')`.
- * The SDK drains it and infers the upload filename from `path`.
+ * Streams with a `path` are uploaded from disk without draining. A one-shot
+ * stream without a path cannot be multipart-resumed.
  */
 export interface NodeReadableStream extends AsyncIterable<unknown> {
   /** Source path; used to infer the upload filename. */
@@ -61,7 +85,12 @@ export interface NodeReadableStream extends AsyncIterable<unknown> {
 }
 
 /** Any value the SDK accepts as a "file" workflow input. */
-export type FileInput = Blob | FileDescriptor | NodeReadableStream;
+export type FileInput =
+  | Blob
+  | FileDescriptor
+  | PathFileInput
+  | StreamFactoryFileInput
+  | NodeReadableStream;
 
 /**
  * Attach a filename (and optional MIME type) to raw bytes. The escape hatch
@@ -83,7 +112,7 @@ export function toFile(
 }
 
 /** Detect a Node readable stream (`fs.createReadStream`, etc.). */
-function isReadStream(value: unknown): value is NodeReadableStream {
+export function isReadStream(value: unknown): value is NodeReadableStream {
   if (value === null || typeof value !== 'object') return false;
   const v = value as Record<PropertyKey, unknown>;
   return (
@@ -93,7 +122,7 @@ function isReadStream(value: unknown): value is NodeReadableStream {
 }
 
 /** Detect an explicit `{ content, filename }` descriptor. */
-function isFileDescriptor(value: unknown): value is FileDescriptor {
+export function isFileDescriptor(value: unknown): value is FileDescriptor {
   if (value === null || typeof value !== 'object') return false;
   const v = value as { content?: unknown; filename?: unknown };
   return (
@@ -105,9 +134,27 @@ function isFileDescriptor(value: unknown): value is FileDescriptor {
   );
 }
 
+export function isPathFileInput(value: unknown): value is PathFileInput {
+  if (value === null || typeof value !== 'object' || isReadStream(value)) return false;
+  const v = value as { path?: unknown; open?: unknown };
+  return typeof v.path === 'string' && typeof v.open !== 'function';
+}
+
+export function isStreamFactoryFileInput(value: unknown): value is StreamFactoryFileInput {
+  if (value === null || typeof value !== 'object') return false;
+  const v = value as { open?: unknown; size?: unknown; filename?: unknown };
+  return (
+    typeof v.open === 'function' &&
+    typeof v.filename === 'string' &&
+    typeof v.size === 'number' &&
+    Number.isFinite(v.size)
+  );
+}
+
 export function isFileInput(value: unknown): value is FileInput {
   if (typeof Blob !== 'undefined' && value instanceof Blob) return true;
   if (isReadStream(value)) return true;
+  if (isPathFileInput(value) || isStreamFactoryFileInput(value)) return true;
   return isFileDescriptor(value);
 }
 
@@ -128,9 +175,9 @@ function basename(path: string): string {
 /**
  * Resolve a `FileInput` to a `{ blob, filename }` pair for `FormData.append`.
  *
- * Streams are drained to bytes *here* — eagerly, before the request is sent —
- * so the body can be replayed if the SDK retries the request. (A consumed
- * stream cannot be re-read; a Blob can.)
+ * One-shot streams without a path are drained here so the body can be replayed
+ * on HTTP retries. Disk paths and stream factories stay unbuffered until the
+ * caller asks for bytes (small multipart leftovers).
  */
 export async function resolveFileBlob(file: FileInput): Promise<{ blob: Blob; filename: string }> {
   // `File` extends `Blob`, so this branch covers both.
@@ -138,7 +185,38 @@ export async function resolveFileBlob(file: FileInput): Promise<{ blob: Blob; fi
     return { blob: file, filename: (file as File).name || 'file' };
   }
 
-  // Node readable stream — drain it into a Blob now.
+  if (isPathFileInput(file) || (isReadStream(file) && typeof file.path === 'string')) {
+    const filePath = isPathFileInput(file) ? file.path : file.path!;
+    const filename = (isPathFileInput(file) ? file.filename : undefined) ?? basename(filePath);
+    if (await pathIsFile(filePath)) {
+      const { readFile } = await import('node:fs/promises');
+      const bytes = await readFile(filePath);
+      const type = (isPathFileInput(file) ? file.mimeType : undefined) ?? guessMimeType(filename);
+      return { blob: new Blob([bytes as BlobPart], { type }), filename };
+    }
+    if (isPathFileInput(file)) {
+      throw new Error(`Upload path is not a file: ${filePath}`);
+    }
+    // Named stream whose path is not readable — drain the in-memory stream.
+  }
+
+  if (isStreamFactoryFileInput(file)) {
+    const body = file.open(0, file.size);
+    if (typeof Blob !== 'undefined' && body instanceof Blob) {
+      return { blob: body, filename: file.filename };
+    }
+    if (body instanceof Uint8Array) {
+      return {
+        blob: new Blob([body as BlobPart], { type: file.mimeType ?? guessMimeType(file.filename) }),
+        filename: file.filename,
+      };
+    }
+    throw new Error(
+      'Stream factory returned a non-Blob body; pass a path or Blob for small multipart leftovers'
+    );
+  }
+
+  // Node readable stream without a path — drain it into a Blob now.
   if (isReadStream(file)) {
     const filename = typeof file.path === 'string' ? basename(file.path) : 'file';
     const parts: BlobPart[] = [];
@@ -161,6 +239,74 @@ export async function resolveFileBlob(file: FileInput): Promise<{ blob: Blob; fi
   return { blob, filename: desc.filename };
 }
 
+export async function statUploadSize(
+  file: FileInput
+): Promise<{ size: number; filename: string; contentType: string }> {
+  if (typeof Blob !== 'undefined' && file instanceof Blob) {
+    return {
+      size: file.size,
+      filename: (file as File).name || 'file',
+      contentType: file.type || DEFAULT_MIME,
+    };
+  }
+  if (isPathFileInput(file) || (isReadStream(file) && typeof file.path === 'string')) {
+    const filePath = isPathFileInput(file) ? file.path : file.path!;
+    const filename = (isPathFileInput(file) ? file.filename : undefined) ?? basename(filePath);
+    if (await pathIsFile(filePath)) {
+      const { stat } = await import('node:fs/promises');
+      const info = await stat(filePath);
+      return {
+        size: info.size,
+        filename,
+        contentType: (isPathFileInput(file) ? file.mimeType : undefined) ?? guessMimeType(filename),
+      };
+    }
+    if (isPathFileInput(file)) {
+      throw new Error(`Upload path is not a file: ${filePath}`);
+    }
+  }
+  if (isStreamFactoryFileInput(file)) {
+    return {
+      size: file.size,
+      filename: file.filename,
+      contentType: file.mimeType ?? guessMimeType(file.filename),
+    };
+  }
+  if (isFileDescriptor(file)) {
+    const size =
+      file.content instanceof Blob
+        ? file.content.size
+        : file.content instanceof ArrayBuffer
+          ? file.content.byteLength
+          : file.content.byteLength;
+    return {
+      size,
+      filename: file.filename,
+      contentType: file.mimeType ?? guessMimeType(file.filename),
+    };
+  }
+  throw new Error(
+    'Cannot determine size of a non-replayable stream. Pass a file path, Blob, or a stream factory with size.'
+  );
+}
+
+export function isReplayableUploadSource(file: FileInput | Uint8Array | ArrayBuffer): boolean {
+  if (file instanceof Uint8Array || file instanceof ArrayBuffer) return true;
+  if (typeof Blob !== 'undefined' && file instanceof Blob) return true;
+  if (isPathFileInput(file) || isStreamFactoryFileInput(file) || isFileDescriptor(file))
+    return true;
+  return isReadStream(file) && typeof file.path === 'string';
+}
+
+async function pathIsFile(filePath: string): Promise<boolean> {
+  try {
+    const { stat } = await import('node:fs/promises');
+    return (await stat(filePath)).isFile();
+  } catch {
+    return false;
+  }
+}
+
 export interface MultipartParts {
   /** FormData ready to send as the request body. */
   formData: FormData;

package/src/lib/upload-presigned-multipart.ts

@@ -0,0 +1,285 @@
+/**
+ * Client protocol for `presigned-multipart` file uploads.
+ * Part math matches the server session; ETags are never trusted locally.
+ */
+
+export const MULTIPART_UPLOAD_CONCURRENCY = 4;
+export const MULTIPART_PART_MAX_RETRIES = 4;
+export const MULTIPART_PART_RETRY_BASE_MS = 250;
+
+export type ListedUploadPart = {
+  partNumber: number;
+  size?: number;
+  etag: string;
+};
+
+export type PresignedUploadPart = {
+  url: string;
+  headers?: Record<string, string>;
+  partSizeBytes: number;
+};
+
+export function expectedPartByteLength(
+  totalSize: number,
+  partSizeBytes: number,
+  partNumber: number,
+  partCount: number
+): number {
+  if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > partCount) {
+    throw new Error(`partNumber must be an integer in [1, ${partCount}]`);
+  }
+  if (totalSize === 0) return 0;
+  if (partNumber < partCount) return partSizeBytes;
+  const remainder = totalSize % partSizeBytes;
+  return remainder === 0 ? partSizeBytes : remainder;
+}
+
+export function partByteOffset(partSizeBytes: number, partNumber: number): number {
+  return (partNumber - 1) * partSizeBytes;
+}
+
+export function isTransientHttpStatus(status: number): boolean {
+  return status === 408 || status === 429 || (status >= 500 && status <= 599);
+}
+
+export function filterStorageHeaders(
+  headers: Record<string, string> | undefined,
+  stripContentLength: boolean
+): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(headers ?? {}).filter(([name]) =>
+      stripContentLength ? name.toLowerCase() !== 'content-length' : true
+    )
+  );
+}
+
+export function partIsAuthoritativelyComplete(
+  listed: ReadonlyArray<ListedUploadPart>,
+  partNumber: number,
+  expectedSize: number
+): boolean {
+  const found = listed.find((part) => part.partNumber === partNumber);
+  if (!found) return false;
+  // Server complete rejects missing sizes. Treat them as incomplete so resume
+  // re-uploads instead of skipping a part the API cannot finalize.
+  return found.size === expectedSize;
+}
+
+export class PartUploadHttpError extends Error {
+  constructor(readonly status: number) {
+    super(`Storage part upload failed (${status})`);
+    this.name = 'PartUploadHttpError';
+  }
+}
+
+export function isAbortError(error: unknown): boolean {
+  return (
+    (error instanceof Error && error.name === 'AbortError') ||
+    (typeof DOMException !== 'undefined' &&
+      error instanceof DOMException &&
+      error.name === 'AbortError')
+  );
+}
+
+/**
+ * Abort leftover MPU state only while parts are not yet authoritative.
+ *
+ * After ListParts shows every expected part, POST complete is idempotent.
+ * 409/429/timeout/lost responses must not abort — that would delete GiB of
+ * uploaded parts and can race a complete that already succeeded server-side.
+ * Caller cancellation and unrecoverable part PUT/API failures still abort
+ * because `partsReady` is false in those cases.
+ */
+export function shouldAbortMultipartUploadSession(options: { partsReady: boolean }): boolean {
+  return !options.partsReady;
+}
+
+/**
+ * Abort leftover presigned-PUT state only while storage PUT is not yet authoritative.
+ *
+ * After a successful storage PUT, POST complete is idempotent. 409/429/timeout/lost
+ * responses must not abort — that would delete a recoverable up-to-4-GiB object and
+ * can race a complete that already succeeded server-side. Caller cancellation and
+ * unrecoverable PUT failures still abort because `putReady` is false in those cases.
+ */
+export function shouldAbortPresignedPutUploadSession(options: { putReady: boolean }): boolean {
+  return !options.putReady;
+}
+
+export function multipartCompleteRetryHint(uploadId: string): string {
+  return `Uploaded parts remain stored for ${uploadId}; retry complete and do not abort the session.`;
+}
+
+export function presignedPutCompleteRetryHint(uploadId: string): string {
+  return `Uploaded object remains stored for ${uploadId}; retry complete and do not abort the session.`;
+}
+
+export function annotateMultipartCompleteFailure(uploadId: string, error: unknown): unknown {
+  return annotateUploadCompleteFailure(multipartCompleteRetryHint(uploadId), error);
+}
+
+export function annotatePresignedPutCompleteFailure(uploadId: string, error: unknown): unknown {
+  return annotateUploadCompleteFailure(presignedPutCompleteRetryHint(uploadId), error);
+}
+
+function annotateUploadCompleteFailure(hint: string, error: unknown): unknown {
+  if (error instanceof Error) {
+    if (!error.message.includes('retry complete')) {
+      error.message = `${error.message} ${hint}`;
+    }
+    return error;
+  }
+  return new Error(`${String(error)} ${hint}`);
+}
+
+export function isRetryablePartError(error: unknown, signal?: AbortSignal): boolean {
+  if (signal?.aborted) return false;
+  if (isAbortError(error)) return false;
+  if (error instanceof PartUploadHttpError) return isTransientHttpStatus(error.status);
+  return true;
+}
+
+export async function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) throw abortError(signal);
+  await new Promise<void>((resolve, reject) => {
+    const timer = setTimeout(resolve, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(abortError(signal));
+    };
+    signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
+function abortError(signal?: AbortSignal): Error {
+  if (signal?.reason instanceof Error) return signal.reason;
+  return typeof DOMException !== 'undefined'
+    ? new DOMException('Upload aborted', 'AbortError')
+    : Object.assign(new Error('Upload aborted'), { name: 'AbortError' });
+}
+
+export async function mapPool<T>(
+  items: readonly T[],
+  concurrency: number,
+  signal: AbortSignal | undefined,
+  worker: (item: T) => Promise<void>
+): Promise<void> {
+  if (items.length === 0) return;
+  let cursor = 0;
+  let firstError: unknown;
+  await Promise.all(
+    Array.from({ length: Math.min(Math.max(1, concurrency), items.length) }, async () => {
+      while (firstError == null) {
+        if (signal?.aborted) throw abortError(signal);
+        const index = cursor++;
+        if (index >= items.length) return;
+        try {
+          await worker(items[index]!);
+        } catch (error) {
+          firstError ??= error;
+        }
+      }
+    })
+  );
+  if (firstError) throw firstError;
+}
+
+export async function uploadPresignedMultipartParts(options: {
+  partCount: number;
+  partSizeBytes: number;
+  totalSize: number;
+  signal?: AbortSignal;
+  concurrency?: number;
+  onProgress?: (uploadedBytes: number, totalBytes: number) => void;
+  listParts: () => Promise<ReadonlyArray<ListedUploadPart>>;
+  presignPart: (partNumber: number) => Promise<PresignedUploadPart>;
+  putPart: (args: {
+    url: string;
+    headers: Record<string, string>;
+    partNumber: number;
+    start: number;
+    length: number;
+  }) => Promise<void>;
+}): Promise<void> {
+  const concurrency = options.concurrency ?? MULTIPART_UPLOAD_CONCURRENCY;
+  let uploadedBytes = 0;
+  const credited = new Set<number>();
+
+  const credit = (partNumber: number, length: number) => {
+    if (credited.has(partNumber)) return;
+    credited.add(partNumber);
+    uploadedBytes += length;
+    options.onProgress?.(uploadedBytes, options.totalSize);
+  };
+
+  const pendingPartNumbers = async (): Promise<number[]> => {
+    const listed = await options.listParts();
+    const pending: number[] = [];
+    for (let partNumber = 1; partNumber <= options.partCount; partNumber++) {
+      const length = expectedPartByteLength(
+        options.totalSize,
+        options.partSizeBytes,
+        partNumber,
+        options.partCount
+      );
+      if (partIsAuthoritativelyComplete(listed, partNumber, length)) {
+        credit(partNumber, length);
+        continue;
+      }
+      pending.push(partNumber);
+    }
+    return pending;
+  };
+
+  const uploadOne = async (partNumber: number): Promise<void> => {
+    const length = expectedPartByteLength(
+      options.totalSize,
+      options.partSizeBytes,
+      partNumber,
+      options.partCount
+    );
+    const start = partByteOffset(options.partSizeBytes, partNumber);
+    let lastError: unknown;
+    for (let attempt = 0; attempt <= MULTIPART_PART_MAX_RETRIES; attempt++) {
+      if (options.signal?.aborted) throw abortError(options.signal);
+      try {
+        const signed = await options.presignPart(partNumber);
+        const partLength = Number.isInteger(signed.partSizeBytes) ? signed.partSizeBytes : length;
+        await options.putPart({
+          url: signed.url,
+          headers: signed.headers ?? {},
+          partNumber,
+          start,
+          length: partLength,
+        });
+        credit(partNumber, length);
+        return;
+      } catch (error) {
+        lastError = error;
+        if (
+          !isRetryablePartError(error, options.signal) ||
+          attempt === MULTIPART_PART_MAX_RETRIES
+        ) {
+          throw error;
+        }
+        await sleep(MULTIPART_PART_RETRY_BASE_MS * 2 ** attempt, options.signal);
+      }
+    }
+    throw lastError;
+  };
+
+  let pending = await pendingPartNumbers();
+  if (pending.length > 0) {
+    await mapPool(pending, concurrency, options.signal, uploadOne);
+  }
+  pending = await pendingPartNumbers();
+  if (pending.length > 0) {
+    await mapPool(pending, concurrency, options.signal, uploadOne);
+    pending = await pendingPartNumbers();
+  }
+  if (pending.length > 0) {
+    throw new Error(
+      `Upload incomplete: ${pending.length} part(s) missing from storage before complete`
+    );
+  }
+}