@flowscripter/pluggable-io-framework 1.1.5 → 1.2.1

package/dist/src/copyMove.js

@@ -1,7 +1,60 @@
 import { adaptReadableStream, identityChunkConverter, } from "@flowscripter/pluggable-io-framework-api";
 import { chunkLength } from "./chunkLength.js";
+import { defaultConcurrencyLimiter, mapAsyncIterableConcurrently, } from "./ConcurrencyLimiter.js";
+import { withRetry } from "./withRetry.js";
 const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
-async function streamingTransfer(source, sourcePath, sink, destPath, operationId, options, totalBytes) {
+const DEFAULT_PART_SIZE = 8 * 1024 * 1024;
+const DEFAULT_RETRY = { maxRetries: 3 };
+const UNCONSTRAINED_PART_SIZE = {
+    minPartSize: 0,
+    maxPartSize: Infinity,
+    maxParts: Infinity,
+    defaultPartSize: DEFAULT_PART_SIZE,
+};
+/** Wraps `hooks` so any `onProgress` event passing through gets `parentOperationId` filled in, unless the emitter already set one (nested hierarchy). */
+function withParentOperationId(hooks, parentOperationId) {
+    if (!hooks || parentOperationId === undefined) {
+        return hooks ?? {};
+    }
+    return {
+        ...hooks,
+        onProgress: hooks.onProgress
+            ? (event) => hooks.onProgress?.({ parentOperationId, ...event })
+            : undefined,
+    };
+}
+function joinPath(base, relative) {
+    if (relative === "")
+        return base;
+    if (base === "")
+        return relative;
+    return `${base.replace(/\/+$/, "")}/${relative}`;
+}
+/**
+ * Reconciles `source`'s and `sink`'s {@link PartSizeConstraints} for a
+ * transfer of `totalSize` bytes into a single part size satisfying both.
+ * Returns `undefined` when the bounds are mutually infeasible (source's
+ * minimum exceeds sink's maximum) - the caller should fall back to plain
+ * streaming rather than throwing.
+ */
+function negotiatePartSize(source, sink, totalSize) {
+    const sourceConstraints = source.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
+    const sinkConstraints = sink.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
+    const minPartSize = Math.max(sourceConstraints.minPartSize, sinkConstraints.minPartSize);
+    const maxPartSize = Math.min(sourceConstraints.maxPartSize, sinkConstraints.maxPartSize);
+    const maxParts = Math.min(sourceConstraints.maxParts, sinkConstraints.maxParts);
+    if (minPartSize > maxPartSize) {
+        return undefined;
+    }
+    const byMaxParts = Math.ceil(totalSize / maxParts);
+    const candidate = Math.max(minPartSize, sourceConstraints.defaultPartSize, sinkConstraints.defaultPartSize, byMaxParts);
+    return Math.min(candidate, maxPartSize);
+}
+function canUseMultipart(source, sink) {
+    return (typeof source.getMultipartReader === "function" && typeof sink.getMultipartWriter === "function");
+}
+async function streamingTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, options, totalBytes) {
+    const hooks = withParentOperationId(options.telemetry, parentOperationId);
     const readable = await source.getReadableStream(sourcePath);
     const writable = await sink.getWritableStream(destPath);
     // The source/sink kind mismatch is decided ONCE here, not per chunk.
@@ -16,7 +69,7 @@ async function streamingTransfer(source, sourcePath, sink, destPath, operationId
                 break;
             await writer.write(value);
             bytesProcessed += chunkLength(value);
-            options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+            hooks.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
         }
         await writer.close();
     }
@@ -25,78 +78,200 @@ async function streamingTransfer(source, sourcePath, sink, destPath, operationId
         throw error;
     }
 }
-async function multipartTransfer(source, sourcePath, sink, destPath, operationId, options, totalBytes) {
+async function multipartTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, partSize, limiter, options, totalBytes) {
+    const hooks = withParentOperationId(options.telemetry, parentOperationId);
     let bytesProcessed = 0;
-    const writer = sink.getMultipartWriter(destPath);
-    async function* transferParts() {
-        for await (const part of source.getMultipartReader(sourcePath)) {
-            // Kind mismatch decided once per part, not once per chunk within it.
-            const adaptedStream = adaptReadableStream(part.stream, part.kind, sink.kind, options.chunkConverter ?? identityChunkConverter);
-            const reader = adaptedStream.getReader();
-            const chunks = [];
-            for (;;) {
-                const { done, value } = await reader.read();
-                if (done)
-                    break;
-                chunks.push(value);
-                bytesProcessed += chunkLength(value);
-                options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
-            }
-            yield {
-                index: part.index,
-                offset: part.offset,
-                kind: sink.kind,
-                stream: new ReadableStream({
-                    start(controller) {
-                        for (const chunk of chunks)
-                            controller.enqueue(chunk);
-                        controller.close();
-                    },
-                }),
-                complete: () => part.complete(),
-            };
+    async function transferPart(sourcePart) {
+        const partOperationId = crypto.randomUUID();
+        // Kind mismatch decided once per part, not once per chunk within it.
+        const adaptedStream = adaptReadableStream(sourcePart.stream, sourcePart.kind, sink.kind, options.chunkConverter ?? identityChunkConverter);
+        const reader = adaptedStream.getReader();
+        const chunks = [];
+        let partBytesProcessed = 0;
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            chunks.push(value);
+            const length = chunkLength(value);
+            partBytesProcessed += length;
+            bytesProcessed += length;
+            hooks.onProgress?.({
+                operationId: partOperationId,
+                parentOperationId: operationId,
+                type: "copy",
+                bytesProcessed: partBytesProcessed,
+            });
+            hooks.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
         }
+        return {
+            index: sourcePart.index,
+            offset: sourcePart.offset,
+            kind: sink.kind,
+            stream: new ReadableStream({
+                start(controller) {
+                    for (const chunk of chunks)
+                        controller.enqueue(chunk);
+                    controller.close();
+                },
+            }),
+            complete: () => sourcePart.complete(),
+        };
+    }
+    const writer = sink.getMultipartWriter(destPath, partSize);
+    const transferredParts = mapAsyncIterableConcurrently(source.getMultipartReader(sourcePath, partSize), transferPart, limiter);
+    await writer.write(transferredParts);
+}
+async function copySingle(source, sourcePath, sink, destPath, options, parentOperationId, properties) {
+    const operationId = crypto.randomUUID();
+    if (source.canDirectTransfer?.(sink) && source.directCopy) {
+        await source.directCopy(sourcePath, destPath, {
+            operationId,
+            hooks: withParentOperationId(options.telemetry, parentOperationId),
+        });
+        return;
+    }
+    const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
+    const size = properties.size;
+    const partSize = size !== undefined && size >= threshold && canUseMultipart(source, sink)
+        ? negotiatePartSize(source, sink, size)
+        : undefined;
+    const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+    const retryOptions = options.retry ?? DEFAULT_RETRY;
+    await withRetry(async () => {
+        if (partSize !== undefined) {
+            await multipartTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, partSize, limiter, options, size);
+        }
+        else {
+            await streamingTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, options, size);
+        }
+    }, retryOptions);
+}
+async function moveSingle(source, sourcePath, sink, destPath, options, parentOperationId, properties) {
+    if (source.canDirectTransfer?.(sink) && source.directMove) {
+        const operationId = crypto.randomUUID();
+        await source.directMove(sourcePath, destPath, {
+            operationId,
+            hooks: withParentOperationId(options.telemetry, parentOperationId),
+        });
+        return;
     }
-    await writer.write(transferParts());
+    await copySingle(source, sourcePath, sink, destPath, options, parentOperationId, properties);
+    await withRetry(() => source.delete(sourcePath), options.retry ?? DEFAULT_RETRY);
 }
-function canUseMultipart(source, sink, size, threshold) {
-    return (size !== undefined &&
-        size >= threshold &&
-        typeof source.getMultipartReader === "function" &&
-        typeof sink.getMultipartWriter === "function");
+/**
+ * Resolves the effective destination root for a recursive transfer,
+ * following `cp -r`/`mv` semantics: a destination that doesn't exist becomes
+ * the copy itself; an existing folder gets the source nested inside it as
+ * `dest/<source-basename>`; an existing file is rejected.
+ */
+async function resolveRecursiveDestRoot(sourcePath, sink, destPath) {
+    let destProperties;
+    try {
+        destProperties = await sink.getProperties(destPath);
+    }
+    catch {
+        destProperties = undefined;
+    }
+    if (destProperties === undefined) {
+        return destPath;
+    }
+    if (!destProperties.isFolder) {
+        throw new Error(`Cannot copy/move folder "${sourcePath}" onto existing file "${destPath}"`);
+    }
+    const baseName = sourcePath.replace(/\/+$/, "").split("/").pop() || sourcePath;
+    return joinPath(destPath, baseName);
+}
+async function recursiveTransfer(source, sourcePath, sink, destPath, options, mode) {
+    if (source.canDirectTransfer?.(sink) && source.supportsRecursiveDirectTransfer) {
+        const directFn = mode === "copy" ? source.directCopy : source.directMove;
+        if (directFn) {
+            const operationId = crypto.randomUUID();
+            await directFn(sourcePath, destPath, { operationId, hooks: options.telemetry ?? {} });
+            return;
+        }
+    }
+    const resolvedDestRoot = await resolveRecursiveDestRoot(sourcePath, sink, destPath);
+    const recursiveOperationId = crypto.randomUUID();
+    const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+    let discovered = 0;
+    let completed = 0;
+    const reportProgress = () => {
+        options.telemetry?.onProgress?.({
+            operationId: recursiveOperationId,
+            type: mode,
+            bytesProcessed: 0,
+            itemsProcessed: completed,
+            totalItems: discovered,
+        });
+    };
+    async function processEntry(entry) {
+        discovered += 1;
+        reportProgress();
+        const entrySourcePath = joinPath(sourcePath, entry.path);
+        const entryDestPath = joinPath(resolvedDestRoot, entry.path);
+        if (entry.properties.isFolder) {
+            await sink.createFolder?.(entryDestPath);
+        }
+        else if (mode === "copy") {
+            await copySingle(source, entrySourcePath, sink, entryDestPath, options, recursiveOperationId, entry.properties);
+        }
+        else if (source.canDirectTransfer?.(sink) && source.directMove) {
+            // A per-file directMove deletes its source atomically as part of the
+            // provider's own operation - safe to do immediately. The non-direct
+            // case below must NOT delete per entry: only after every entry
+            // succeeds does the caller delete the whole source root once.
+            const operationId = crypto.randomUUID();
+            await source.directMove(entrySourcePath, entryDestPath, {
+                operationId,
+                hooks: withParentOperationId(options.telemetry, recursiveOperationId),
+            });
+        }
+        else {
+            await copySingle(source, entrySourcePath, sink, entryDestPath, options, recursiveOperationId, entry.properties);
+        }
+        completed += 1;
+        reportProgress();
+    }
+    for await (const _entry of mapAsyncIterableConcurrently(source.list(sourcePath, { recursive: true }), processEntry, limiter)) {
+        // work happens inside processEntry - nothing to do with the (void) result.
+    }
+    if (mode === "move") {
+        await withRetry(() => source.delete(sourcePath), options.retry ?? DEFAULT_RETRY);
+    }
 }
 /**
- * Copies `sourcePath` on `source` to `destPath` on `sink`.
+ * Copies `sourcePath` on `source` to `destPath` on `sink`. When `sourcePath`
+ * is a folder, recurses: uses a folder-aware `directCopy` when the source
+ * declares `supportsRecursiveDirectTransfer`, otherwise lists the folder and
+ * transfers each entry (bounded by `options.concurrencyLimiter`), following
+ * `cp -r` target semantics (see {@link resolveRecursiveDestRoot}).
  *
- * Uses `source.directCopy` when `source.canDirectTransfer?.(sink)` reports
- * eligibility (same-provider direct transfer). Otherwise falls back to
- * multipart transfer (when both sides support it and size crosses
- * `options.multipartThreshold`) or plain streaming.
+ * For a single file: uses `source.directCopy` when
+ * `source.canDirectTransfer?.(sink)` reports eligibility. Otherwise falls
+ * back to multipart transfer (when both sides support it, size crosses
+ * `options.multipartThreshold`, and part-size negotiation succeeds) or plain
+ * streaming - retried per `options.retry` on `TransientIOError`.
  */
 export async function copy(source, sourcePath, sink, destPath, options = {}) {
-    const operationId = crypto.randomUUID();
-    if (source.canDirectTransfer?.(sink) && source.directCopy) {
-        await source.directCopy(sourcePath, destPath);
-        return;
-    }
     const properties = await source.getProperties(sourcePath);
-    const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
-    if (canUseMultipart(source, sink, properties.size, threshold)) {
-        await multipartTransfer(source, sourcePath, sink, destPath, operationId, options, properties.size);
+    if (properties.isFolder) {
+        await recursiveTransfer(source, sourcePath, sink, destPath, options, "copy");
         return;
     }
-    await streamingTransfer(source, sourcePath, sink, destPath, operationId, options, properties.size);
+    await copySingle(source, sourcePath, sink, destPath, options, undefined, properties);
 }
 /**
- * Moves `sourcePath` on `source` to `destPath` on `sink`. Uses
- * `source.directMove` when eligible, otherwise performs a {@link copy}
- * followed by deleting the source item.
+ * Moves `sourcePath` on `source` to `destPath` on `sink`. Folder handling
+ * mirrors {@link copy}. For a single file, uses `source.directMove` when
+ * eligible, otherwise performs a {@link copy} followed by deleting the
+ * source item (each retried independently per `options.retry`).
  */
 export async function move(source, sourcePath, sink, destPath, options = {}) {
-    if (source.canDirectTransfer?.(sink) && source.directMove) {
-        await source.directMove(sourcePath, destPath);
+    const properties = await source.getProperties(sourcePath);
+    if (properties.isFolder) {
+        await recursiveTransfer(source, sourcePath, sink, destPath, options, "move");
         return;
     }
-    await copy(source, sourcePath, sink, destPath, options);
-    await source.delete(sourcePath);
+    await moveSingle(source, sourcePath, sink, destPath, options, undefined, properties);
 }

package/dist/src/withRetry.d.ts

@@ -0,0 +1,13 @@
+export interface RetryOptions {
+    readonly maxRetries: number;
+    /** Delay in ms before the given retry attempt (1-based). Defaults to `min(200 * 2^attempt, 5000)`. */
+    readonly backoffMs?: (attempt: number) => number;
+}
+/**
+ * Runs `fn`, retrying only on `TransientIOError` up to `options.maxRetries`
+ * times with a delay between attempts. Any other error - including a plain
+ * `Error` a provider hasn't wrapped - is treated as non-retryable and
+ * rethrown immediately.
+ */
+export declare function withRetry<T>(fn: () => Promise<T>, options: RetryOptions): Promise<T>;
+//# sourceMappingURL=withRetry.d.ts.map

package/dist/src/withRetry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"withRetry.d.ts","sourceRoot":"","sources":["../../src/withRetry.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,sGAAsG;IACtG,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAMD;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,CAc1F"}

package/dist/src/withRetry.js

@@ -0,0 +1,26 @@
+import { TransientIOError } from "@flowscripter/pluggable-io-framework-api";
+function defaultBackoffMs(attempt) {
+    return Math.min(200 * 2 ** attempt, 5000);
+}
+/**
+ * Runs `fn`, retrying only on `TransientIOError` up to `options.maxRetries`
+ * times with a delay between attempts. Any other error - including a plain
+ * `Error` a provider hasn't wrapped - is treated as non-retryable and
+ * rethrown immediately.
+ */
+export async function withRetry(fn, options) {
+    const backoffMs = options.backoffMs ?? defaultBackoffMs;
+    let attempt = 0;
+    for (;;) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            if (!(error instanceof TransientIOError) || attempt >= options.maxRetries) {
+                throw error;
+            }
+            attempt += 1;
+            await new Promise((resolve) => setTimeout(resolve, backoffMs(attempt)));
+        }
+    }
+}

package/index.ts

@@ -1,4 +1,6 @@
+export * from "./src/ConcurrencyLimiter.ts";
 export * from "./src/ProviderRegistry.ts";
 export * from "./src/copyMove.ts";
 export * from "./src/decorators/locallyCached.ts";
 export * from "./src/decorators/seekable.ts";
+export * from "./src/withRetry.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowscripter/pluggable-io-framework",
-  "version": "1.1.5",
+  "version": "1.2.1",
   "description": "A pluggable source/sink IO framework using https://github.com/flowscripter/dynamic-plugin-framework",
   "keywords": [
     "bun",
@@ -40,8 +40,8 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@flowscripter/dynamic-plugin-framework": "^2.2.6",
-    "@flowscripter/pluggable-io-framework-api": "^1.0.9"
+    "@flowscripter/dynamic-plugin-framework": "^2.2.8",
+    "@flowscripter/pluggable-io-framework-api": "^1.1.1"
   },
   "devDependencies": {
     "@types/bun": "^1.4.2",

package/src/ConcurrencyLimiter.ts

@@ -0,0 +1,178 @@
+const DEFAULT_MAX_CONCURRENCY = 4;
+
+/**
+ * A semaphore bounding how many `run()` callbacks execute at once. A single
+ * shared instance (see {@link defaultConcurrencyLimiter}) enforces its cap
+ * *globally* across concurrently invoked `copy()`/`move()` calls, not just
+ * within one call - two folder copies started at the same time and sharing
+ * a limiter will never together exceed its `maxConcurrency`.
+ */
+export class ConcurrencyLimiter {
+  #maxConcurrency: number;
+  #active = 0;
+  #queue: (() => void)[] = [];
+
+  public constructor(maxConcurrency: number) {
+    ConcurrencyLimiter.#validate(maxConcurrency);
+    this.#maxConcurrency = maxConcurrency;
+  }
+
+  public get maxConcurrency(): number {
+    return this.#maxConcurrency;
+  }
+
+  public setMaxConcurrency(maxConcurrency: number): void {
+    ConcurrencyLimiter.#validate(maxConcurrency);
+    this.#maxConcurrency = maxConcurrency;
+    this.#drain();
+  }
+
+  public async run<T>(fn: () => Promise<T>): Promise<T> {
+    await this.#acquire();
+    try {
+      return await fn();
+    } finally {
+      this.#active -= 1;
+      this.#drain();
+    }
+  }
+
+  #acquire(): Promise<void> {
+    if (this.#active < this.#maxConcurrency) {
+      this.#active += 1;
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      this.#queue.push(() => {
+        this.#active += 1;
+        resolve();
+      });
+    });
+  }
+
+  #drain(): void {
+    while (this.#active < this.#maxConcurrency && this.#queue.length > 0) {
+      this.#queue.shift()?.();
+    }
+  }
+
+  static #validate(maxConcurrency: number): void {
+    if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
+      throw new Error(`maxConcurrency must be a positive integer, got ${maxConcurrency}`);
+    }
+  }
+}
+
+/**
+ * Process-wide default limiter, used by `copy()`/`move()` whenever
+ * `TransferOptions.concurrencyLimiter` is omitted - so any two calls in the
+ * same process share one cap unless a caller explicitly opts out with its
+ * own instance (e.g. for test isolation).
+ */
+export const defaultConcurrencyLimiter = new ConcurrencyLimiter(DEFAULT_MAX_CONCURRENCY);
+
+/** Minimal single-consumer async FIFO queue backing {@link mapAsyncIterableConcurrently}. */
+class AsyncChannel<T> implements AsyncIterable<T> {
+  #items: T[] = [];
+  #waiters: { resolve: (result: IteratorResult<T>) => void; reject: (error: unknown) => void }[] =
+    [];
+  #closed = false;
+  #hasError = false;
+  #error: unknown;
+
+  push(item: T): void {
+    const waiter = this.#waiters.shift();
+    if (waiter) {
+      waiter.resolve({ value: item, done: false });
+    } else {
+      this.#items.push(item);
+    }
+  }
+
+  close(): void {
+    this.#closed = true;
+    for (const waiter of this.#waiters.splice(0)) {
+      if (this.#hasError) {
+        waiter.reject(this.#error);
+      } else {
+        waiter.resolve({ value: undefined as never, done: true });
+      }
+    }
+  }
+
+  fail(error: unknown): void {
+    this.#hasError = true;
+    this.#error = error;
+    this.close();
+  }
+
+  async #next(): Promise<IteratorResult<T>> {
+    if (this.#items.length > 0) {
+      return { value: this.#items.shift() as T, done: false };
+    }
+    if (this.#closed) {
+      if (this.#hasError) {
+        throw this.#error;
+      }
+      return { value: undefined as never, done: true };
+    }
+    return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    return { next: () => this.#next() };
+  }
+}
+
+/**
+ * Pulls items from `source` and runs `fn` on each, bounded by `limiter`:
+ * exactly `limiter.maxConcurrency` worker loops each pull-then-process one
+ * item at a time through `limiter.run`, so at most that many items are ever
+ * pulled/in-flight at once (important for sources like a multipart reader
+ * where pulling an item eagerly opens a file/network handle). Results are
+ * yielded in completion order, not source order. If `fn` throws, no new
+ * items are pulled but already-in-flight ones are allowed to finish before
+ * the error is rethrown from the returned iterable.
+ */
+export function mapAsyncIterableConcurrently<T, R>(
+  source: AsyncIterable<T>,
+  fn: (item: T) => Promise<R>,
+  limiter: ConcurrencyLimiter,
+): AsyncIterable<R> {
+  const channel = new AsyncChannel<R>();
+  const iterator = source[Symbol.asyncIterator]();
+  let stopped = false;
+  let firstError: unknown;
+  let errorRecorded = false;
+
+  async function worker(): Promise<void> {
+    for (;;) {
+      if (stopped) return;
+      const { done, value } = await iterator.next();
+      if (done) return;
+      try {
+        const result = await limiter.run(() => fn(value));
+        channel.push(result);
+      } catch (error) {
+        stopped = true;
+        if (!errorRecorded) {
+          errorRecorded = true;
+          firstError = error;
+        }
+        return;
+      }
+    }
+  }
+
+  void (async () => {
+    const workerCount = Math.max(1, limiter.maxConcurrency);
+    await Promise.all(Array.from({ length: workerCount }, () => worker()));
+    if (errorRecorded) {
+      channel.fail(firstError);
+    } else {
+      channel.close();
+    }
+  })();
+
+  return channel;
+}