@flowscripter/pluggable-io-framework 1.1.5 → 1.2.0

package/src/copyMove.ts

@@ -4,10 +4,18 @@ import {
   type ChunkConverter,
   type ChunkRef,
   type IOProvider,
+  type ItemProperties,
   type Part,
+  type PartSizeConstraints,
   type TelemetryHooks,
 } from "@flowscripter/pluggable-io-framework-api";
 import { chunkLength } from "./chunkLength.ts";
+import {
+  type ConcurrencyLimiter,
+  defaultConcurrencyLimiter,
+  mapAsyncIterableConcurrently,
+} from "./ConcurrencyLimiter.ts";
+import { withRetry, type RetryOptions } from "./withRetry.ts";
 
 export interface TransferOptions {
   readonly telemetry?: TelemetryHooks;
@@ -20,9 +28,85 @@ export interface TransferOptions {
    * FFI-capable converter supplied by a runtime-specific package.
    */
   readonly chunkConverter?: ChunkConverter;
+  /**
+   * Bounds concurrent multipart parts and recursive-copy/move entries.
+   * Defaults to the shared {@link defaultConcurrencyLimiter} - so two
+   * `copy()`/`move()` calls in the same process share one cap unless a
+   * caller passes its own isolated instance.
+   */
+  readonly concurrencyLimiter?: ConcurrencyLimiter;
+  /** Retry policy for the non-direct transfer path. Defaults to `{ maxRetries: 3 }`. */
+  readonly retry?: RetryOptions;
 }
 
 const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
+const DEFAULT_PART_SIZE = 8 * 1024 * 1024;
+const DEFAULT_RETRY: RetryOptions = { maxRetries: 3 };
+
+const UNCONSTRAINED_PART_SIZE: PartSizeConstraints = {
+  minPartSize: 0,
+  maxPartSize: Infinity,
+  maxParts: Infinity,
+  defaultPartSize: DEFAULT_PART_SIZE,
+};
+
+/** Wraps `hooks` so any `onProgress` event passing through gets `parentOperationId` filled in, unless the emitter already set one (nested hierarchy). */
+function withParentOperationId(
+  hooks: TelemetryHooks | undefined,
+  parentOperationId: string | undefined,
+): TelemetryHooks {
+  if (!hooks || parentOperationId === undefined) {
+    return hooks ?? {};
+  }
+  return {
+    ...hooks,
+    onProgress: hooks.onProgress
+      ? (event) => hooks.onProgress?.({ parentOperationId, ...event })
+      : undefined,
+  };
+}
+
+function joinPath(base: string, relative: string): string {
+  if (relative === "") return base;
+  if (base === "") return relative;
+  return `${base.replace(/\/+$/, "")}/${relative}`;
+}
+
+/**
+ * Reconciles `source`'s and `sink`'s {@link PartSizeConstraints} for a
+ * transfer of `totalSize` bytes into a single part size satisfying both.
+ * Returns `undefined` when the bounds are mutually infeasible (source's
+ * minimum exceeds sink's maximum) - the caller should fall back to plain
+ * streaming rather than throwing.
+ */
+function negotiatePartSize(
+  source: IOProvider,
+  sink: IOProvider,
+  totalSize: number,
+): number | undefined {
+  const sourceConstraints = source.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
+  const sinkConstraints = sink.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
+  const minPartSize = Math.max(sourceConstraints.minPartSize, sinkConstraints.minPartSize);
+  const maxPartSize = Math.min(sourceConstraints.maxPartSize, sinkConstraints.maxPartSize);
+  const maxParts = Math.min(sourceConstraints.maxParts, sinkConstraints.maxParts);
+  if (minPartSize > maxPartSize) {
+    return undefined;
+  }
+  const byMaxParts = Math.ceil(totalSize / maxParts);
+  const candidate = Math.max(
+    minPartSize,
+    sourceConstraints.defaultPartSize,
+    sinkConstraints.defaultPartSize,
+    byMaxParts,
+  );
+  return Math.min(candidate, maxPartSize);
+}
+
+function canUseMultipart(source: IOProvider, sink: IOProvider): boolean {
+  return (
+    typeof source.getMultipartReader === "function" && typeof sink.getMultipartWriter === "function"
+  );
+}
 
 async function streamingTransfer(
   source: IOProvider,
@@ -30,9 +114,11 @@ async function streamingTransfer(
   sink: IOProvider,
   destPath: string,
   operationId: string,
+  parentOperationId: string | undefined,
   options: TransferOptions,
   totalBytes: number | undefined,
 ): Promise<void> {
+  const hooks = withParentOperationId(options.telemetry, parentOperationId);
   const readable = await source.getReadableStream(sourcePath);
   const writable = await sink.getWritableStream(destPath);
   // The source/sink kind mismatch is decided ONCE here, not per chunk.
@@ -51,7 +137,7 @@ async function streamingTransfer(
       if (done) break;
       await writer.write(value);
       bytesProcessed += chunkLength(value);
-      options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+      hooks.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
     }
     await writer.close();
   } catch (error) {
@@ -66,67 +152,270 @@ async function multipartTransfer(
   sink: IOProvider,
   destPath: string,
   operationId: string,
+  parentOperationId: string | undefined,
+  partSize: number,
+  limiter: ConcurrencyLimiter,
   options: TransferOptions,
   totalBytes: number | undefined,
 ): Promise<void> {
+  const hooks = withParentOperationId(options.telemetry, parentOperationId);
   let bytesProcessed = 0;
-  const writer = sink.getMultipartWriter(destPath);
-  async function* transferParts(): AsyncIterable<Part> {
-    for await (const part of source.getMultipartReader(sourcePath)) {
-      // Kind mismatch decided once per part, not once per chunk within it.
-      const adaptedStream = adaptReadableStream(
-        part.stream as ReadableStream<ChunkRef>,
-        part.kind,
-        sink.kind,
-        options.chunkConverter ?? identityChunkConverter,
+
+  async function transferPart(sourcePart: Part): Promise<Part> {
+    const partOperationId = crypto.randomUUID();
+    // Kind mismatch decided once per part, not once per chunk within it.
+    const adaptedStream = adaptReadableStream(
+      sourcePart.stream as ReadableStream<ChunkRef>,
+      sourcePart.kind,
+      sink.kind,
+      options.chunkConverter ?? identityChunkConverter,
+    );
+    const reader = adaptedStream.getReader();
+    const chunks: ChunkRef[] = [];
+    let partBytesProcessed = 0;
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+      const length = chunkLength(value);
+      partBytesProcessed += length;
+      bytesProcessed += length;
+      hooks.onProgress?.({
+        operationId: partOperationId,
+        parentOperationId: operationId,
+        type: "copy",
+        bytesProcessed: partBytesProcessed,
+      });
+      hooks.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+    }
+    return {
+      index: sourcePart.index,
+      offset: sourcePart.offset,
+      kind: sink.kind,
+      stream: new ReadableStream<ChunkRef>({
+        start(controller) {
+          for (const chunk of chunks) controller.enqueue(chunk);
+          controller.close();
+        },
+      }),
+      complete: () => sourcePart.complete(),
+    };
+  }
+
+  const writer = sink.getMultipartWriter(destPath, partSize);
+  const transferredParts = mapAsyncIterableConcurrently(
+    source.getMultipartReader(sourcePath, partSize),
+    transferPart,
+    limiter,
+  );
+  await writer.write(transferredParts);
+}
+
+async function copySingle(
+  source: IOProvider,
+  sourcePath: string,
+  sink: IOProvider,
+  destPath: string,
+  options: TransferOptions,
+  parentOperationId: string | undefined,
+  properties: ItemProperties,
+): Promise<void> {
+  const operationId = crypto.randomUUID();
+
+  if (source.canDirectTransfer?.(sink) && source.directCopy) {
+    await source.directCopy(sourcePath, destPath, {
+      operationId,
+      hooks: withParentOperationId(options.telemetry, parentOperationId),
+    });
+    return;
+  }
+
+  const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
+  const size = properties.size;
+  const partSize =
+    size !== undefined && size >= threshold && canUseMultipart(source, sink)
+      ? negotiatePartSize(source, sink, size)
+      : undefined;
+  const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+  const retryOptions = options.retry ?? DEFAULT_RETRY;
+
+  await withRetry(async () => {
+    if (partSize !== undefined) {
+      await multipartTransfer(
+        source,
+        sourcePath,
+        sink,
+        destPath,
+        operationId,
+        parentOperationId,
+        partSize,
+        limiter,
+        options,
+        size,
+      );
+    } else {
+      await streamingTransfer(
+        source,
+        sourcePath,
+        sink,
+        destPath,
+        operationId,
+        parentOperationId,
+        options,
+        size,
       );
-      const reader = adaptedStream.getReader();
-      const chunks: ChunkRef[] = [];
-      for (;;) {
-        const { done, value } = await reader.read();
-        if (done) break;
-        chunks.push(value);
-        bytesProcessed += chunkLength(value);
-        options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
-      }
-      yield {
-        index: part.index,
-        offset: part.offset,
-        kind: sink.kind,
-        stream: new ReadableStream<ChunkRef>({
-          start(controller) {
-            for (const chunk of chunks) controller.enqueue(chunk);
-            controller.close();
-          },
-        }),
-        complete: () => part.complete(),
-      };
     }
+  }, retryOptions);
+}
+
+async function moveSingle(
+  source: IOProvider,
+  sourcePath: string,
+  sink: IOProvider,
+  destPath: string,
+  options: TransferOptions,
+  parentOperationId: string | undefined,
+  properties: ItemProperties,
+): Promise<void> {
+  if (source.canDirectTransfer?.(sink) && source.directMove) {
+    const operationId = crypto.randomUUID();
+    await source.directMove(sourcePath, destPath, {
+      operationId,
+      hooks: withParentOperationId(options.telemetry, parentOperationId),
+    });
+    return;
   }
-  await writer.write(transferParts());
+  await copySingle(source, sourcePath, sink, destPath, options, parentOperationId, properties);
+  await withRetry(() => source.delete(sourcePath), options.retry ?? DEFAULT_RETRY);
 }
 
-function canUseMultipart(
+/**
+ * Resolves the effective destination root for a recursive transfer,
+ * following `cp -r`/`mv` semantics: a destination that doesn't exist becomes
+ * the copy itself; an existing folder gets the source nested inside it as
+ * `dest/<source-basename>`; an existing file is rejected.
+ */
+async function resolveRecursiveDestRoot(
+  sourcePath: string,
+  sink: IOProvider,
+  destPath: string,
+): Promise<string> {
+  let destProperties: ItemProperties | undefined;
+  try {
+    destProperties = await sink.getProperties(destPath);
+  } catch {
+    destProperties = undefined;
+  }
+  if (destProperties === undefined) {
+    return destPath;
+  }
+  if (!destProperties.isFolder) {
+    throw new Error(`Cannot copy/move folder "${sourcePath}" onto existing file "${destPath}"`);
+  }
+  const baseName = sourcePath.replace(/\/+$/, "").split("/").pop() || sourcePath;
+  return joinPath(destPath, baseName);
+}
+
+async function recursiveTransfer(
   source: IOProvider,
+  sourcePath: string,
   sink: IOProvider,
-  size: number | undefined,
-  threshold: number,
-): boolean {
-  return (
-    size !== undefined &&
-    size >= threshold &&
-    typeof source.getMultipartReader === "function" &&
-    typeof sink.getMultipartWriter === "function"
-  );
+  destPath: string,
+  options: TransferOptions,
+  mode: "copy" | "move",
+): Promise<void> {
+  if (source.canDirectTransfer?.(sink) && source.supportsRecursiveDirectTransfer) {
+    const directFn = mode === "copy" ? source.directCopy : source.directMove;
+    if (directFn) {
+      const operationId = crypto.randomUUID();
+      await directFn(sourcePath, destPath, { operationId, hooks: options.telemetry ?? {} });
+      return;
+    }
+  }
+
+  const resolvedDestRoot = await resolveRecursiveDestRoot(sourcePath, sink, destPath);
+  const recursiveOperationId = crypto.randomUUID();
+  const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+
+  let discovered = 0;
+  let completed = 0;
+  const reportProgress = (): void => {
+    options.telemetry?.onProgress?.({
+      operationId: recursiveOperationId,
+      type: mode,
+      bytesProcessed: 0,
+      itemsProcessed: completed,
+      totalItems: discovered,
+    });
+  };
+
+  async function processEntry(entry: { path: string; properties: ItemProperties }): Promise<void> {
+    discovered += 1;
+    reportProgress();
+    const entrySourcePath = joinPath(sourcePath, entry.path);
+    const entryDestPath = joinPath(resolvedDestRoot, entry.path);
+    if (entry.properties.isFolder) {
+      await sink.createFolder?.(entryDestPath);
+    } else if (mode === "copy") {
+      await copySingle(
+        source,
+        entrySourcePath,
+        sink,
+        entryDestPath,
+        options,
+        recursiveOperationId,
+        entry.properties,
+      );
+    } else if (source.canDirectTransfer?.(sink) && source.directMove) {
+      // A per-file directMove deletes its source atomically as part of the
+      // provider's own operation - safe to do immediately. The non-direct
+      // case below must NOT delete per entry: only after every entry
+      // succeeds does the caller delete the whole source root once.
+      const operationId = crypto.randomUUID();
+      await source.directMove(entrySourcePath, entryDestPath, {
+        operationId,
+        hooks: withParentOperationId(options.telemetry, recursiveOperationId),
+      });
+    } else {
+      await copySingle(
+        source,
+        entrySourcePath,
+        sink,
+        entryDestPath,
+        options,
+        recursiveOperationId,
+        entry.properties,
+      );
+    }
+    completed += 1;
+    reportProgress();
+  }
+
+  for await (const _entry of mapAsyncIterableConcurrently(
+    source.list(sourcePath, { recursive: true }),
+    processEntry,
+    limiter,
+  )) {
+    // work happens inside processEntry - nothing to do with the (void) result.
+  }
+
+  if (mode === "move") {
+    await withRetry(() => source.delete(sourcePath), options.retry ?? DEFAULT_RETRY);
+  }
 }
 
 /**
- * Copies `sourcePath` on `source` to `destPath` on `sink`.
+ * Copies `sourcePath` on `source` to `destPath` on `sink`. When `sourcePath`
+ * is a folder, recurses: uses a folder-aware `directCopy` when the source
+ * declares `supportsRecursiveDirectTransfer`, otherwise lists the folder and
+ * transfers each entry (bounded by `options.concurrencyLimiter`), following
+ * `cp -r` target semantics (see {@link resolveRecursiveDestRoot}).
  *
- * Uses `source.directCopy` when `source.canDirectTransfer?.(sink)` reports
- * eligibility (same-provider direct transfer). Otherwise falls back to
- * multipart transfer (when both sides support it and size crosses
- * `options.multipartThreshold`) or plain streaming.
+ * For a single file: uses `source.directCopy` when
+ * `source.canDirectTransfer?.(sink)` reports eligibility. Otherwise falls
+ * back to multipart transfer (when both sides support it, size crosses
+ * `options.multipartThreshold`, and part-size negotiation succeeds) or plain
+ * streaming - retried per `options.retry` on `TransientIOError`.
  */
 export async function copy(
   source: IOProvider,
@@ -135,40 +424,19 @@ export async function copy(
   destPath: string,
   options: TransferOptions = {},
 ): Promise<void> {
-  const operationId = crypto.randomUUID();
-  if (source.canDirectTransfer?.(sink) && source.directCopy) {
-    await source.directCopy(sourcePath, destPath);
-    return;
-  }
   const properties = await source.getProperties(sourcePath);
-  const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
-  if (canUseMultipart(source, sink, properties.size, threshold)) {
-    await multipartTransfer(
-      source,
-      sourcePath,
-      sink,
-      destPath,
-      operationId,
-      options,
-      properties.size,
-    );
+  if (properties.isFolder) {
+    await recursiveTransfer(source, sourcePath, sink, destPath, options, "copy");
     return;
   }
-  await streamingTransfer(
-    source,
-    sourcePath,
-    sink,
-    destPath,
-    operationId,
-    options,
-    properties.size,
-  );
+  await copySingle(source, sourcePath, sink, destPath, options, undefined, properties);
 }
 
 /**
- * Moves `sourcePath` on `source` to `destPath` on `sink`. Uses
- * `source.directMove` when eligible, otherwise performs a {@link copy}
- * followed by deleting the source item.
+ * Moves `sourcePath` on `source` to `destPath` on `sink`. Folder handling
+ * mirrors {@link copy}. For a single file, uses `source.directMove` when
+ * eligible, otherwise performs a {@link copy} followed by deleting the
+ * source item (each retried independently per `options.retry`).
  */
 export async function move(
   source: IOProvider,
@@ -177,10 +445,10 @@ export async function move(
   destPath: string,
   options: TransferOptions = {},
 ): Promise<void> {
-  if (source.canDirectTransfer?.(sink) && source.directMove) {
-    await source.directMove(sourcePath, destPath);
+  const properties = await source.getProperties(sourcePath);
+  if (properties.isFolder) {
+    await recursiveTransfer(source, sourcePath, sink, destPath, options, "move");
     return;
   }
-  await copy(source, sourcePath, sink, destPath, options);
-  await source.delete(sourcePath);
+  await moveSingle(source, sourcePath, sink, destPath, options, undefined, properties);
 }

package/src/withRetry.ts

@@ -0,0 +1,33 @@
+import { TransientIOError } from "@flowscripter/pluggable-io-framework-api";
+
+export interface RetryOptions {
+  readonly maxRetries: number;
+  /** Delay in ms before the given retry attempt (1-based). Defaults to `min(200 * 2^attempt, 5000)`. */
+  readonly backoffMs?: (attempt: number) => number;
+}
+
+function defaultBackoffMs(attempt: number): number {
+  return Math.min(200 * 2 ** attempt, 5000);
+}
+
+/**
+ * Runs `fn`, retrying only on `TransientIOError` up to `options.maxRetries`
+ * times with a delay between attempts. Any other error - including a plain
+ * `Error` a provider hasn't wrapped - is treated as non-retryable and
+ * rethrown immediately.
+ */
+export async function withRetry<T>(fn: () => Promise<T>, options: RetryOptions): Promise<T> {
+  const backoffMs = options.backoffMs ?? defaultBackoffMs;
+  let attempt = 0;
+  for (;;) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (!(error instanceof TransientIOError) || attempt >= options.maxRetries) {
+        throw error;
+      }
+      attempt += 1;
+      await new Promise((resolve) => setTimeout(resolve, backoffMs(attempt)));
+    }
+  }
+}