@flowscripter/pluggable-io-framework 1.1.5 → 1.2.0

package/README.md

@@ -22,8 +22,20 @@
     bucket).
   - Otherwise transfers via multipart (when both sides support it and file
     size crosses a configurable threshold) or plain streaming.
+  - Multipart part size is negotiated between source and sink via
+    `getPartSizeConstraints` (e.g. reconciling S3's minimum part size and
+    10000-part cap on both ends) - see [Part-size negotiation](#part-size-negotiation).
+  - A folder `sourcePath` recurses: a folder-aware `directCopy`/`directMove`
+    when the source declares `supportsRecursiveDirectTransfer`, otherwise a
+    concurrency-bounded listing-driven transfer following `cp -r` target
+    semantics - see [Recursive copy/move](#recursive-copymove).
+  - Concurrent multipart parts and recursive entries are bounded by a shared
+    `ConcurrencyLimiter` - see [Concurrency](#concurrency).
+  - The non-direct transfer path is retried on `TransientIOError` - see
+    [Retries](#retries).
   - Reports progress via a global `TelemetryHooks` callback, tagged with a
-    per-operation correlation id.
+    per-operation correlation id; multipart parts and recursive entries
+    report their own child stream tagged with `parentOperationId`.
 - Stream decorators:
   - `seekable` wraps a handle that supports `RangeReadable` with a single
     logical stream whose read position can be jumped via `seek(offset)`,
@@ -69,6 +81,61 @@ The following example project is available:
 - [flowscripter-io-cli](https://github.com/flowscripter/flowscripter-io-cli) is
   an example CLI application based on this framework.
 
+## Part-size negotiation
+
+Before a multipart transfer, `copy()`/`move()` call `getPartSizeConstraints(totalSize)`
+on both `source` and `sink` (when implemented - a missing implementation is
+treated as unconstrained) and reconcile the two into a single part size:
+the larger of the two minimums, clamped to the smaller of the two maximums,
+bumped up if needed so the total number of parts never exceeds the smaller
+of the two `maxParts` (e.g. S3's 10000-part cap on a very large file). If
+the reconciled bounds are mutually infeasible, the transfer falls back to
+plain streaming rather than failing.
+
+## Recursive copy/move
+
+When `sourcePath` is a folder, `copy()`/`move()` follow `cp -r`/`mv`
+semantics for the destination: a `destPath` that doesn't exist becomes the
+copy itself; an existing folder gets the source nested inside it as
+`destPath/<source-basename>`; an existing file is rejected.
+
+If the source provider declares `supportsRecursiveDirectTransfer: true` and
+is direct-transfer-eligible with the sink, the whole folder is handed to a
+single `directCopy`/`directMove` call. Otherwise the folder is listed
+(`source.list(path, { recursive: true })`) and each entry is transferred
+individually - folders via the optional `createFolder` capability (so empty
+folders are preserved), files via the normal single-file path (which may
+still resolve to a per-file `directCopy`/`directMove`). For a recursive
+move without any direct-transfer capability, every entry is copied first;
+the source folder is deleted as a single recursive `delete()` call only
+after every entry has succeeded.
+
+## Concurrency
+
+Multipart parts and recursive-copy/move entries are bounded by a
+`ConcurrencyLimiter` (`TransferOptions.concurrencyLimiter`), defaulting to
+the exported `defaultConcurrencyLimiter` singleton - so two `copy()`/`move()`
+calls in the same process share one cap unless a caller passes its own
+isolated instance:
+
+```typescript
+import { defaultConcurrencyLimiter } from "@flowscripter/pluggable-io-framework";
+
+defaultConcurrencyLimiter.setMaxConcurrency(8);
+```
+
+## Retries
+
+The non-direct transfer path (plain streaming, multipart, and the
+copy-then-delete in a non-direct `move()`) is retried via `TransferOptions.retry`
+(`{ maxRetries, backoffMs? }`, defaulting to `{ maxRetries: 3 }`) whenever a
+provider throws `TransientIOError` from `pluggable-io-framework-api`. Any
+other error - including a plain, unwrapped `Error` - is treated as
+non-retryable. Direct provider calls (`directCopy`/`directMove`) are never
+retried by the framework, since backends like the AWS SDK already retry
+internally. The underlying `withRetry` utility is exported standalone for
+other call sites.
+
 ## Development
 
 Install dependencies:
@@ -114,12 +181,18 @@ sequenceDiagram
     ProviderRegistry-->>Host: IOProvider
 
     Host->>Source: copy(source, path, sink, path)
-    alt canDirectTransfer
+    alt path is a folder
+        alt supportsRecursiveDirectTransfer
+            Source->>Sink: directCopy(folderPath, folderPath)
+        else
+            Source-->>Sink: list + transfer entries (bounded by ConcurrencyLimiter)
+        end
+    else canDirectTransfer
         Source->>Sink: directCopy(path, path)
-    else multipart eligible
-        Source-->>Sink: transfer Parts concurrently
+    else multipart eligible (negotiated part size)
+        Source-->>Sink: transfer Parts concurrently (bounded, retried on TransientIOError)
     else
-        Source-->>Sink: stream ChunkRefs
+        Source-->>Sink: stream ChunkRefs (retried on TransientIOError)
     end
 ```
 

package/dist/index.d.ts

@@ -1,5 +1,7 @@
+export * from "./src/ConcurrencyLimiter.ts";
 export * from "./src/ProviderRegistry.ts";
 export * from "./src/copyMove.ts";
 export * from "./src/decorators/locallyCached.ts";
 export * from "./src/decorators/seekable.ts";
+export * from "./src/withRetry.ts";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mCAAmC,CAAC;AAClD,cAAc,8BAA8B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mCAAmC,CAAC;AAClD,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oBAAoB,CAAC"}

package/dist/index.js

@@ -1,4 +1,6 @@
+export * from "./src/ConcurrencyLimiter.js";
 export * from "./src/ProviderRegistry.js";
 export * from "./src/copyMove.js";
 export * from "./src/decorators/locallyCached.js";
 export * from "./src/decorators/seekable.js";
+export * from "./src/withRetry.js";

package/dist/src/ConcurrencyLimiter.d.ts

@@ -0,0 +1,33 @@
+/**
+ * A semaphore bounding how many `run()` callbacks execute at once. A single
+ * shared instance (see {@link defaultConcurrencyLimiter}) enforces its cap
+ * *globally* across concurrently invoked `copy()`/`move()` calls, not just
+ * within one call - two folder copies started at the same time and sharing
+ * a limiter will never together exceed its `maxConcurrency`.
+ */
+export declare class ConcurrencyLimiter {
+    #private;
+    constructor(maxConcurrency: number);
+    get maxConcurrency(): number;
+    setMaxConcurrency(maxConcurrency: number): void;
+    run<T>(fn: () => Promise<T>): Promise<T>;
+}
+/**
+ * Process-wide default limiter, used by `copy()`/`move()` whenever
+ * `TransferOptions.concurrencyLimiter` is omitted - so any two calls in the
+ * same process share one cap unless a caller explicitly opts out with its
+ * own instance (e.g. for test isolation).
+ */
+export declare const defaultConcurrencyLimiter: ConcurrencyLimiter;
+/**
+ * Pulls items from `source` and runs `fn` on each, bounded by `limiter`:
+ * exactly `limiter.maxConcurrency` worker loops each pull-then-process one
+ * item at a time through `limiter.run`, so at most that many items are ever
+ * pulled/in-flight at once (important for sources like a multipart reader
+ * where pulling an item eagerly opens a file/network handle). Results are
+ * yielded in completion order, not source order. If `fn` throws, no new
+ * items are pulled but already-in-flight ones are allowed to finish before
+ * the error is rethrown from the returned iterable.
+ */
+export declare function mapAsyncIterableConcurrently<T, R>(source: AsyncIterable<T>, fn: (item: T) => Promise<R>, limiter: ConcurrencyLimiter): AsyncIterable<R>;
+//# sourceMappingURL=ConcurrencyLimiter.d.ts.map

package/dist/src/ConcurrencyLimiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConcurrencyLimiter.d.ts","sourceRoot":"","sources":["../../src/ConcurrencyLimiter.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,qBAAa,kBAAkB;;IAK7B,YAAmB,cAAc,EAAE,MAAM,EAGxC;IAED,IAAW,cAAc,IAAI,MAAM,CAElC;IAEM,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAIrD;IAEY,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAQpD;CA0BF;AAED;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,oBAAkD,CAAC;AAuDzF;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CAAC,CAAC,EAAE,CAAC,EAC/C,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,EACxB,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,aAAa,CAAC,CAAC,CAAC,CAqClB"}

package/dist/src/ConcurrencyLimiter.js

@@ -0,0 +1,161 @@
+const DEFAULT_MAX_CONCURRENCY = 4;
+/**
+ * A semaphore bounding how many `run()` callbacks execute at once. A single
+ * shared instance (see {@link defaultConcurrencyLimiter}) enforces its cap
+ * *globally* across concurrently invoked `copy()`/`move()` calls, not just
+ * within one call - two folder copies started at the same time and sharing
+ * a limiter will never together exceed its `maxConcurrency`.
+ */
+export class ConcurrencyLimiter {
+    #maxConcurrency;
+    #active = 0;
+    #queue = [];
+    constructor(maxConcurrency) {
+        ConcurrencyLimiter.#validate(maxConcurrency);
+        this.#maxConcurrency = maxConcurrency;
+    }
+    get maxConcurrency() {
+        return this.#maxConcurrency;
+    }
+    setMaxConcurrency(maxConcurrency) {
+        ConcurrencyLimiter.#validate(maxConcurrency);
+        this.#maxConcurrency = maxConcurrency;
+        this.#drain();
+    }
+    async run(fn) {
+        await this.#acquire();
+        try {
+            return await fn();
+        }
+        finally {
+            this.#active -= 1;
+            this.#drain();
+        }
+    }
+    #acquire() {
+        if (this.#active < this.#maxConcurrency) {
+            this.#active += 1;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.#queue.push(() => {
+                this.#active += 1;
+                resolve();
+            });
+        });
+    }
+    #drain() {
+        while (this.#active < this.#maxConcurrency && this.#queue.length > 0) {
+            this.#queue.shift()?.();
+        }
+    }
+    static #validate(maxConcurrency) {
+        if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
+            throw new Error(`maxConcurrency must be a positive integer, got ${maxConcurrency}`);
+        }
+    }
+}
+/**
+ * Process-wide default limiter, used by `copy()`/`move()` whenever
+ * `TransferOptions.concurrencyLimiter` is omitted - so any two calls in the
+ * same process share one cap unless a caller explicitly opts out with its
+ * own instance (e.g. for test isolation).
+ */
+export const defaultConcurrencyLimiter = new ConcurrencyLimiter(DEFAULT_MAX_CONCURRENCY);
+/** Minimal single-consumer async FIFO queue backing {@link mapAsyncIterableConcurrently}. */
+class AsyncChannel {
+    #items = [];
+    #waiters = [];
+    #closed = false;
+    #hasError = false;
+    #error;
+    push(item) {
+        const waiter = this.#waiters.shift();
+        if (waiter) {
+            waiter.resolve({ value: item, done: false });
+        }
+        else {
+            this.#items.push(item);
+        }
+    }
+    close() {
+        this.#closed = true;
+        for (const waiter of this.#waiters.splice(0)) {
+            if (this.#hasError) {
+                waiter.reject(this.#error);
+            }
+            else {
+                waiter.resolve({ value: undefined, done: true });
+            }
+        }
+    }
+    fail(error) {
+        this.#hasError = true;
+        this.#error = error;
+        this.close();
+    }
+    async #next() {
+        if (this.#items.length > 0) {
+            return { value: this.#items.shift(), done: false };
+        }
+        if (this.#closed) {
+            if (this.#hasError) {
+                throw this.#error;
+            }
+            return { value: undefined, done: true };
+        }
+        return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
+    }
+    [Symbol.asyncIterator]() {
+        return { next: () => this.#next() };
+    }
+}
+/**
+ * Pulls items from `source` and runs `fn` on each, bounded by `limiter`:
+ * exactly `limiter.maxConcurrency` worker loops each pull-then-process one
+ * item at a time through `limiter.run`, so at most that many items are ever
+ * pulled/in-flight at once (important for sources like a multipart reader
+ * where pulling an item eagerly opens a file/network handle). Results are
+ * yielded in completion order, not source order. If `fn` throws, no new
+ * items are pulled but already-in-flight ones are allowed to finish before
+ * the error is rethrown from the returned iterable.
+ */
+export function mapAsyncIterableConcurrently(source, fn, limiter) {
+    const channel = new AsyncChannel();
+    const iterator = source[Symbol.asyncIterator]();
+    let stopped = false;
+    let firstError;
+    let errorRecorded = false;
+    async function worker() {
+        for (;;) {
+            if (stopped)
+                return;
+            const { done, value } = await iterator.next();
+            if (done)
+                return;
+            try {
+                const result = await limiter.run(() => fn(value));
+                channel.push(result);
+            }
+            catch (error) {
+                stopped = true;
+                if (!errorRecorded) {
+                    errorRecorded = true;
+                    firstError = error;
+                }
+                return;
+            }
+        }
+    }
+    void (async () => {
+        const workerCount = Math.max(1, limiter.maxConcurrency);
+        await Promise.all(Array.from({ length: workerCount }, () => worker()));
+        if (errorRecorded) {
+            channel.fail(firstError);
+        }
+        else {
+            channel.close();
+        }
+    })();
+    return channel;
+}

package/dist/src/copyMove.d.ts

@@ -1,4 +1,6 @@
 import { type ChunkConverter, type IOProvider, type TelemetryHooks } from "@flowscripter/pluggable-io-framework-api";
+import { type ConcurrencyLimiter } from "./ConcurrencyLimiter.ts";
+import { type RetryOptions } from "./withRetry.ts";
 export interface TransferOptions {
     readonly telemetry?: TelemetryHooks;
     /** Minimum file size (bytes) before multipart transfer is attempted over plain streaming. */
@@ -10,20 +12,35 @@ export interface TransferOptions {
      * FFI-capable converter supplied by a runtime-specific package.
      */
     readonly chunkConverter?: ChunkConverter;
+    /**
+     * Bounds concurrent multipart parts and recursive-copy/move entries.
+     * Defaults to the shared {@link defaultConcurrencyLimiter} - so two
+     * `copy()`/`move()` calls in the same process share one cap unless a
+     * caller passes its own isolated instance.
+     */
+    readonly concurrencyLimiter?: ConcurrencyLimiter;
+    /** Retry policy for the non-direct transfer path. Defaults to `{ maxRetries: 3 }`. */
+    readonly retry?: RetryOptions;
 }
 /**
- * Copies `sourcePath` on `source` to `destPath` on `sink`.
+ * Copies `sourcePath` on `source` to `destPath` on `sink`. When `sourcePath`
+ * is a folder, recurses: uses a folder-aware `directCopy` when the source
+ * declares `supportsRecursiveDirectTransfer`, otherwise lists the folder and
+ * transfers each entry (bounded by `options.concurrencyLimiter`), following
+ * `cp -r` target semantics (see {@link resolveRecursiveDestRoot}).
  *
- * Uses `source.directCopy` when `source.canDirectTransfer?.(sink)` reports
- * eligibility (same-provider direct transfer). Otherwise falls back to
- * multipart transfer (when both sides support it and size crosses
- * `options.multipartThreshold`) or plain streaming.
+ * For a single file: uses `source.directCopy` when
+ * `source.canDirectTransfer?.(sink)` reports eligibility. Otherwise falls
+ * back to multipart transfer (when both sides support it, size crosses
+ * `options.multipartThreshold`, and part-size negotiation succeeds) or plain
+ * streaming - retried per `options.retry` on `TransientIOError`.
  */
 export declare function copy(source: IOProvider, sourcePath: string, sink: IOProvider, destPath: string, options?: TransferOptions): Promise<void>;
 /**
- * Moves `sourcePath` on `source` to `destPath` on `sink`. Uses
- * `source.directMove` when eligible, otherwise performs a {@link copy}
- * followed by deleting the source item.
+ * Moves `sourcePath` on `source` to `destPath` on `sink`. Folder handling
+ * mirrors {@link copy}. For a single file, uses `source.directMove` when
+ * eligible, otherwise performs a {@link copy} followed by deleting the
+ * source item (each retried independently per `options.retry`).
  */
 export declare function move(source: IOProvider, sourcePath: string, sink: IOProvider, destPath: string, options?: TransferOptions): Promise<void>;
 //# sourceMappingURL=copyMove.d.ts.map

package/dist/src/copyMove.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"copyMove.d.ts","sourceRoot":"","sources":["../../src/copyMove.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EAEnB,KAAK,UAAU,EAEf,KAAK,cAAc,EACpB,MAAM,0CAA0C,CAAC;AAGlD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACpC,6FAA6F;IAC7F,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,CAAC;CAC1C;AAoGD;;;;;;;GAOG;AACH,wBAAsB,IAAI,CACxB,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;GAIG;AACH,wBAAsB,IAAI,CACxB,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAOf"}
+{"version":3,"file":"copyMove.d.ts","sourceRoot":"","sources":["../../src/copyMove.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EAEnB,KAAK,UAAU,EAIf,KAAK,cAAc,EACpB,MAAM,0CAA0C,CAAC;AAElD,OAAO,EACL,KAAK,kBAAkB,EAGxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAa,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACpC,6FAA6F;IAC7F,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACjD,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CAC/B;AA+WD;;;;;;;;;;;;GAYG;AACH,wBAAsB,IAAI,CACxB,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAOf;AAED;;;;;GAKG;AACH,wBAAsB,IAAI,CACxB,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAOf"}