@flowscripter/pluggable-io-framework 1.2.7 → 2.0.0

package/dist/src/transfer/copyMove.js

@@ -0,0 +1,68 @@
+import { PermanentIOError, } from "@flowscripter/pluggable-io-framework-api";
+import { baseName, joinKey } from "../util/keys.js";
+import { patternTransfer } from "./patternTransfer.js";
+import { recursiveTransfer } from "./recursiveTransfer.js";
+import { transferEntry } from "./transferEntry.js";
+async function checkSourceVariant(source, key, expectContainer, variant) {
+    const properties = await source.getProperties(key);
+    if (properties.isContainer !== expectContainer) {
+        const actual = properties.isContainer ? "a container" : "an entry";
+        throw new PermanentIOError(`Source "${key}" is declared as ${variant} but is ${actual}`);
+    }
+    return properties;
+}
+async function transfer(source, sourceTarget, sink, destTarget, options, type) {
+    if (destTarget.kind === "pattern") {
+        throw new PermanentIOError("A destination cannot be a pattern");
+    }
+    if (sourceTarget.kind === "entry") {
+        const properties = await checkSourceVariant(source, sourceTarget.key, false, "an entry");
+        const destKey = destTarget.kind === "entry"
+            ? destTarget.key
+            : joinKey(sink, destTarget.key, baseName(sourceTarget.key));
+        return transferEntry({
+            source,
+            sourceKey: sourceTarget.key,
+            sink,
+            destKey,
+            properties,
+            options,
+            type,
+            deleteSource: type === "move",
+        });
+    }
+    if (destTarget.kind === "entry") {
+        throw new PermanentIOError(sourceTarget.kind === "container"
+            ? "Cannot transfer a container onto an entry"
+            : "Cannot transfer a pattern of entries onto a single entry");
+    }
+    if (sourceTarget.kind === "container") {
+        await checkSourceVariant(source, sourceTarget.key, true, "a container");
+        return recursiveTransfer(source, sourceTarget.key, sink, destTarget.key, options, type);
+    }
+    await checkSourceVariant(source, sourceTarget.containerKey, true, "a pattern container");
+    return patternTransfer(source, sourceTarget.containerKey, sourceTarget.pattern, sink, destTarget.key, options, type);
+}
+/**
+ * Copies `sourceTarget` on `source` to `destTarget` on `sink`:
+ * - entry -> entry writes exactly the destination key;
+ * - entry -> container writes `<container>/<source basename>`;
+ * - container -> container copies recursively with `cp -r` semantics;
+ * - pattern -> container copies each matching entry;
+ * - container or pattern -> entry, and any pattern destination, are rejected.
+ *
+ * Each entry uses `directCopy` when eligible (and `options.directTransfer`
+ * is not `false`), otherwise multipart, the lease path or plain streaming.
+ */
+export function copy(source, sourceTarget, sink, destTarget, options = {}) {
+    return transfer(source, sourceTarget, sink, destTarget, options, "copy");
+}
+/**
+ * Moves `sourceTarget` on `source` to `destTarget` on `sink`, with the same
+ * target rules as {@link copy}. Non-direct moves copy and then delete the
+ * source. Unbounded sources are rejected, and a stopped move never deletes
+ * its source.
+ */
+export function move(source, sourceTarget, sink, destTarget, options = {}) {
+    return transfer(source, sourceTarget, sink, destTarget, options, "move");
+}

package/dist/src/transfer/leaseTransfer.d.ts

@@ -0,0 +1,14 @@
+import type { BufferProvider, FillReadable, StreamHandle } from "@flowscripter/pluggable-io-framework-api";
+import type { EntryContext, EntryOutcome } from "./EntryContext.ts";
+export declare const DEFAULT_LEASE_DEPTH = 2;
+/** The number of leases the engine keeps outstanding for `writable`. */
+export declare function leaseDepth(writable: BufferProvider, requested: number | undefined): number;
+/**
+ * Moves data by having the source fill buffers handed out by the sink:
+ * `acquire -> readInto -> commit`, with up to `depth` leases outstanding so
+ * the next read overlaps earlier commits. `readInto` calls are serialised
+ * and commits happen in acquisition order. Every lease not yet committed is
+ * released on abort, error or end of stream.
+ */
+export declare function leaseTransfer(readable: StreamHandle & FillReadable, writable: StreamHandle & BufferProvider, depth: number, context: EntryContext): Promise<EntryOutcome>;
+//# sourceMappingURL=leaseTransfer.d.ts.map

package/dist/src/transfer/leaseTransfer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"leaseTransfer.d.ts","sourceRoot":"","sources":["../../../src/transfer/leaseTransfer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,cAAc,EACd,YAAY,EAEZ,YAAY,EACb,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEpE,eAAO,MAAM,mBAAmB,IAAI,CAAC;AAErC,wEAAwE;AACxE,wBAAgB,UAAU,CAAC,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAK1F;AAID;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,YAAY,GAAG,YAAY,EACrC,QAAQ,EAAE,YAAY,GAAG,cAAc,EACvC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,YAAY,CAAC,CA0DvB"}

package/dist/src/transfer/leaseTransfer.js

@@ -0,0 +1,74 @@
+import { createAbortError } from "../util/abort.js";
+export const DEFAULT_LEASE_DEPTH = 2;
+/** The number of leases the engine keeps outstanding for `writable`. */
+export function leaseDepth(writable, requested) {
+    return Math.max(1, Math.min(requested ?? DEFAULT_LEASE_DEPTH, writable.maxOutstanding ?? Infinity));
+}
+function ignore() { }
+/**
+ * Moves data by having the source fill buffers handed out by the sink:
+ * `acquire -> readInto -> commit`, with up to `depth` leases outstanding so
+ * the next read overlaps earlier commits. `readInto` calls are serialised
+ * and commits happen in acquisition order. Every lease not yet committed is
+ * released on abort, error or end of stream.
+ */
+export async function leaseTransfer(readable, writable, depth, context) {
+    const { options, hooks, operationId, type } = context;
+    const outstanding = new Set();
+    const pending = [];
+    let commitChain = Promise.resolve();
+    let bytes = 0;
+    let items = 0;
+    let stopped = false;
+    const writer = writable.stream.getWriter();
+    try {
+        for (;;) {
+            if (options.signal?.aborted)
+                throw createAbortError();
+            if (options.stop?.aborted) {
+                stopped = true;
+                break;
+            }
+            while (pending.length >= depth) {
+                await pending.shift();
+            }
+            const lease = await writable.acquire();
+            outstanding.add(lease);
+            const length = await readable.readInto(lease);
+            if (length === null) {
+                outstanding.delete(lease);
+                lease.release();
+                break;
+            }
+            bytes += length;
+            items += 1;
+            const committed = commitChain.then(async () => {
+                await lease.commit(length);
+                outstanding.delete(lease);
+            });
+            committed.catch(ignore);
+            commitChain = committed;
+            pending.push(committed);
+            hooks.onProgress?.({
+                operationId,
+                type,
+                bytesProcessed: bytes,
+                totalBytes: undefined,
+            });
+        }
+        await commitChain;
+        await writer.close();
+        if (!stopped) {
+            await readable.stream.cancel().catch(ignore);
+        }
+        return { bytes, items, stopped };
+    }
+    catch (error) {
+        for (const lease of outstanding) {
+            lease.release();
+        }
+        await writer.abort(error).catch(ignore);
+        await readable.stream.cancel(error).catch(ignore);
+        throw error;
+    }
+}

package/dist/src/transfer/multipartTransfer.d.ts

@@ -0,0 +1,23 @@
+import { type IOProvider, type Part, type RangeReadable, type StreamHandle } from "@flowscripter/pluggable-io-framework-api";
+import type { ConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.ts";
+import type { EntryContext, EntryOutcome } from "./EntryContext.ts";
+export interface MultipartTransferInput {
+    readonly readable: StreamHandle & RangeReadable;
+    readonly sink: IOProvider;
+    readonly writer: {
+        write(parts: AsyncIterable<Part>): Promise<void>;
+    };
+    readonly totalBytes: number;
+    readonly partSize: number;
+    readonly limiter: ConcurrencyLimiter;
+    readonly context: EntryContext;
+}
+/**
+ * Reads parts with `readRange` and hands them to the sink's multipart
+ * writer, bounded by `limiter`. A part whose read fails with a
+ * `TransientIOError` is re-read on its own, up to `retry.maxRetries` times,
+ * within the same upload. `stop` ends the transfer after the parts already
+ * started; `signal` aborts it.
+ */
+export declare function multipartTransfer(input: MultipartTransferInput): Promise<EntryOutcome>;
+//# sourceMappingURL=multipartTransfer.d.ts.map

package/dist/src/transfer/multipartTransfer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multipartTransfer.d.ts","sourceRoot":"","sources":["../../../src/transfer/multipartTransfer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,UAAU,EAEf,KAAK,IAAI,EACT,KAAK,aAAa,EAClB,KAAK,YAAY,EAElB,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAM/E,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGpE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,aAAa,CAAC;IAChD,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE;QAAE,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC;IACtE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;CAChC;AAED;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAoF5F"}

package/dist/src/transfer/multipartTransfer.js

@@ -0,0 +1,94 @@
+import { TransientIOError, } from "@flowscripter/pluggable-io-framework-api";
+import { mapAsyncIterableConcurrently } from "../concurrency/mapAsyncIterableConcurrently.js";
+import { backoff, DEFAULT_RETRY } from "../retry/RetryOptions.js";
+import { createAbortError } from "../util/abort.js";
+import { applyPayloadConverter } from "../util/applyPayloadConverter.js";
+import { itemLength } from "../util/itemLength.js";
+import { rangeReadableMultipartReader } from "./rangeReadableMultipartReader.js";
+/**
+ * Reads parts with `readRange` and hands them to the sink's multipart
+ * writer, bounded by `limiter`. A part whose read fails with a
+ * `TransientIOError` is re-read on its own, up to `retry.maxRetries` times,
+ * within the same upload. `stop` ends the transfer after the parts already
+ * started; `signal` aborts it.
+ */
+export async function multipartTransfer(input) {
+    const { readable, sink, writer, totalBytes, partSize, limiter, context } = input;
+    const { options, hooks, operationId, type } = context;
+    const retry = options.retry ?? DEFAULT_RETRY;
+    let bytes = 0;
+    let items = 0;
+    let stopped = false;
+    async function readPart(sourcePart) {
+        const partOperationId = crypto.randomUUID();
+        const end = Math.min(sourcePart.offset + partSize, totalBytes);
+        let stream = sourcePart.stream;
+        let attempt = 0;
+        for (;;) {
+            const reader = applyPayloadConverter(stream, readable.kind, sink.kind, options.converter).getReader();
+            const partItems = [];
+            let partBytes = 0;
+            try {
+                for (;;) {
+                    if (options.signal?.aborted) {
+                        await reader.cancel().catch(() => { });
+                        throw createAbortError();
+                    }
+                    const { done, value } = await reader.read();
+                    if (done)
+                        break;
+                    partItems.push(value);
+                    partBytes += itemLength(value);
+                    hooks.onProgress?.({
+                        operationId: partOperationId,
+                        parentOperationId: operationId,
+                        type,
+                        bytesProcessed: partBytes,
+                    });
+                }
+                bytes += partBytes;
+                items += partItems.length;
+                hooks.onProgress?.({ operationId, type, bytesProcessed: bytes, totalBytes });
+                return partItems;
+            }
+            catch (error) {
+                if (!(error instanceof TransientIOError) || attempt >= retry.maxRetries) {
+                    throw error;
+                }
+                attempt += 1;
+                await backoff(retry, attempt);
+                stream = (await readable.readRange(sourcePart.offset, end));
+            }
+        }
+    }
+    async function transferPart(sourcePart) {
+        const partItems = await readPart(sourcePart);
+        return {
+            index: sourcePart.index,
+            offset: sourcePart.offset,
+            kind: sink.kind,
+            stream: new ReadableStream({
+                start(controller) {
+                    for (const item of partItems)
+                        controller.enqueue(item);
+                    controller.close();
+                },
+            }),
+            complete: () => sourcePart.complete(),
+        };
+    }
+    async function* sourceParts() {
+        for await (const part of rangeReadableMultipartReader(readable, totalBytes, partSize)) {
+            if (options.signal?.aborted)
+                throw createAbortError();
+            if (options.stop?.aborted) {
+                stopped = true;
+                await part.stream.cancel().catch(() => { });
+                return;
+            }
+            yield part;
+        }
+    }
+    await writer.write(mapAsyncIterableConcurrently(sourceParts(), transferPart, limiter));
+    return { bytes, items, stopped };
+}

package/dist/src/transfer/negotiatePartSize.d.ts

@@ -0,0 +1,10 @@
+import type { IOProvider } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Reconciles `source`'s and `sink`'s {@link PartSizeConstraints} for a
+ * transfer of `totalSize` bytes into a single part size satisfying both.
+ * Returns `undefined` when the bounds are mutually infeasible (source's
+ * minimum exceeds sink's maximum) - the caller should fall back to plain
+ * streaming rather than throwing.
+ */
+export declare function negotiatePartSize(source: IOProvider, sink: IOProvider, totalSize: number): number | undefined;
+//# sourceMappingURL=negotiatePartSize.d.ts.map

package/dist/src/transfer/negotiatePartSize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"negotiatePartSize.d.ts","sourceRoot":"","sources":["../../../src/transfer/negotiatePartSize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAuB,MAAM,0CAA0C,CAAC;AAWhG;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAiBpB"}

package/dist/src/transfer/negotiatePartSize.js

@@ -0,0 +1,27 @@
+const DEFAULT_PART_SIZE = 8 * 1024 * 1024;
+const UNCONSTRAINED_PART_SIZE = {
+    minPartSize: 0,
+    maxPartSize: Infinity,
+    maxParts: Infinity,
+    defaultPartSize: DEFAULT_PART_SIZE,
+};
+/**
+ * Reconciles `source`'s and `sink`'s {@link PartSizeConstraints} for a
+ * transfer of `totalSize` bytes into a single part size satisfying both.
+ * Returns `undefined` when the bounds are mutually infeasible (source's
+ * minimum exceeds sink's maximum) - the caller should fall back to plain
+ * streaming rather than throwing.
+ */
+export function negotiatePartSize(source, sink, totalSize) {
+    const sourceConstraints = source.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
+    const sinkConstraints = sink.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
+    const minPartSize = Math.max(sourceConstraints.minPartSize, sinkConstraints.minPartSize);
+    const maxPartSize = Math.min(sourceConstraints.maxPartSize, sinkConstraints.maxPartSize);
+    const maxParts = Math.min(sourceConstraints.maxParts, sinkConstraints.maxParts);
+    if (minPartSize > maxPartSize) {
+        return undefined;
+    }
+    const byMaxParts = Math.ceil(totalSize / maxParts);
+    const candidate = Math.max(minPartSize, sourceConstraints.defaultPartSize, sinkConstraints.defaultPartSize, byMaxParts);
+    return Math.min(candidate, maxPartSize);
+}

package/dist/src/transfer/patternTransfer.d.ts

@@ -0,0 +1,12 @@
+import { type IOProvider } from "@flowscripter/pluggable-io-framework-api";
+import type { TransferOptions } from "./TransferOptions.ts";
+import type { TransferResult } from "./TransferResult.ts";
+/**
+ * Transfers every entry directly inside `containerKey` whose name matches
+ * the glob `pattern` into the destination container, one single-entry
+ * transfer per match, bounded by the concurrency limiter. Matching
+ * containers are skipped. A move deletes each source entry after it is
+ * transferred.
+ */
+export declare function patternTransfer(source: IOProvider, containerKey: string, pattern: string, sink: IOProvider, destContainerKey: string, options: TransferOptions, type: "copy" | "move"): Promise<TransferResult>;
+//# sourceMappingURL=patternTransfer.d.ts.map

package/dist/src/transfer/patternTransfer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"patternTransfer.d.ts","sourceRoot":"","sources":["../../../src/transfer/patternTransfer.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,UAAU,EAEhB,MAAM,0CAA0C,CAAC;AAQlD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,UAAU,EAChB,gBAAgB,EAAE,MAAM,EACxB,OAAO,EAAE,eAAe,EACxB,IAAI,EAAE,MAAM,GAAG,MAAM,GACpB,OAAO,CAAC,cAAc,CAAC,CAqDzB"}

package/dist/src/transfer/patternTransfer.js

@@ -0,0 +1,61 @@
+import { PermanentIOError, } from "@flowscripter/pluggable-io-framework-api";
+import { defaultConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.js";
+import { mapAsyncIterableConcurrently } from "../concurrency/mapAsyncIterableConcurrently.js";
+import { throwIfAborted } from "../util/abort.js";
+import { globToRegex } from "../util/globToRegex.js";
+import { joinKey } from "../util/keys.js";
+import { aggregateResults } from "./aggregateResults.js";
+import { basePath, transferEntry } from "./transferEntry.js";
+/**
+ * Transfers every entry directly inside `containerKey` whose name matches
+ * the glob `pattern` into the destination container, one single-entry
+ * transfer per match, bounded by the concurrency limiter. Matching
+ * containers are skipped. A move deletes each source entry after it is
+ * transferred.
+ */
+export async function patternTransfer(source, containerKey, pattern, sink, destContainerKey, options, type) {
+    if (typeof source.list !== "function") {
+        throw new PermanentIOError("The source provider does not support listing containers");
+    }
+    const operationId = crypto.randomUUID();
+    const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+    let discovered = 0;
+    let completed = 0;
+    const reportProgress = () => {
+        options.telemetry?.onProgress?.({
+            operationId,
+            type,
+            bytesProcessed: 0,
+            entriesProcessed: completed,
+            totalEntries: discovered,
+        });
+    };
+    async function processEntry(entry) {
+        throwIfAborted(options.signal);
+        if (options.stop?.aborted || entry.properties.isContainer) {
+            return undefined;
+        }
+        discovered += 1;
+        reportProgress();
+        const result = await transferEntry({
+            source,
+            sourceKey: joinKey(source, containerKey, entry.path),
+            sink,
+            destKey: joinKey(sink, destContainerKey, entry.path),
+            properties: entry.properties,
+            options,
+            parentOperationId: operationId,
+            type,
+            deleteSource: type === "move",
+        });
+        completed += 1;
+        reportProgress();
+        return result;
+    }
+    const results = [];
+    for await (const result of mapAsyncIterableConcurrently(source.list(containerKey, { recursive: false, regex: globToRegex(pattern) }), processEntry, limiter)) {
+        if (result)
+            results.push(result);
+    }
+    return aggregateResults(results, basePath(source, sink, options), options.stop?.aborted ?? false);
+}

package/dist/src/transfer/rangeReadableMultipartReader.d.ts

@@ -0,0 +1,8 @@
+import type { Part, PayloadKind, RangeReadable, StreamHandle } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Splits a `RangeReadable` handle into parts of `partSize` bytes covering
+ * `totalSize`. Each part's stream is opened with `readRange(start, end)`
+ * (`end` exclusive) when the part is pulled.
+ */
+export declare function rangeReadableMultipartReader<K extends PayloadKind>(handle: StreamHandle<K> & RangeReadable<K>, totalSize: number, partSize: number): AsyncGenerator<Part<K>>;
+//# sourceMappingURL=rangeReadableMultipartReader.d.ts.map

package/dist/src/transfer/rangeReadableMultipartReader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rangeReadableMultipartReader.d.ts","sourceRoot":"","sources":["../../../src/transfer/rangeReadableMultipartReader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,EACJ,WAAW,EACX,aAAa,EACb,YAAY,EACb,MAAM,0CAA0C,CAAC;AAElD;;;;GAIG;AACH,wBAAuB,4BAA4B,CAAC,CAAC,SAAS,WAAW,EACvE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,EAC1C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAazB"}

package/dist/src/transfer/rangeReadableMultipartReader.js

@@ -0,0 +1,19 @@
+/**
+ * Splits a `RangeReadable` handle into parts of `partSize` bytes covering
+ * `totalSize`. Each part's stream is opened with `readRange(start, end)`
+ * (`end` exclusive) when the part is pulled.
+ */
+export async function* rangeReadableMultipartReader(handle, totalSize, partSize) {
+    const partCount = Math.max(1, Math.ceil(totalSize / partSize));
+    for (let index = 0; index < partCount; index += 1) {
+        const start = index * partSize;
+        const end = Math.min(start + partSize, totalSize);
+        yield {
+            index,
+            offset: start,
+            kind: handle.kind,
+            stream: await handle.readRange(start, end),
+            complete: async () => { },
+        };
+    }
+}

package/dist/src/transfer/recursiveTransfer.d.ts

@@ -0,0 +1,13 @@
+import { type IOProvider } from "@flowscripter/pluggable-io-framework-api";
+import type { TransferOptions } from "./TransferOptions.ts";
+import type { TransferResult } from "./TransferResult.ts";
+/**
+ * Transfers a whole container. Uses a container-aware
+ * `directCopy`/`directMove` when the source declares
+ * `supportsRecursiveDirectTransfer`; otherwise lists the container and
+ * transfers each entry, bounded by the concurrency limiter. A non-direct
+ * move deletes the source container once, after every entry succeeded and
+ * only if the transfer was not stopped.
+ */
+export declare function recursiveTransfer(source: IOProvider, sourceKey: string, sink: IOProvider, destKey: string, options: TransferOptions, type: "copy" | "move"): Promise<TransferResult>;
+//# sourceMappingURL=recursiveTransfer.d.ts.map

package/dist/src/transfer/recursiveTransfer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recursiveTransfer.d.ts","sourceRoot":"","sources":["../../../src/transfer/recursiveTransfer.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,UAAU,EAEhB,MAAM,0CAA0C,CAAC;AASlD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AA8B1D;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,EAChB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,eAAe,EACxB,IAAI,EAAE,MAAM,GAAG,MAAM,GACpB,OAAO,CAAC,cAAc,CAAC,CAuFzB"}

package/dist/src/transfer/recursiveTransfer.js

@@ -0,0 +1,112 @@
+import { PermanentIOError, } from "@flowscripter/pluggable-io-framework-api";
+import { defaultConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.js";
+import { mapAsyncIterableConcurrently } from "../concurrency/mapAsyncIterableConcurrently.js";
+import { DEFAULT_RETRY } from "../retry/RetryOptions.js";
+import { withRetry } from "../retry/withRetry.js";
+import { throwIfAborted } from "../util/abort.js";
+import { baseName, joinKey } from "../util/keys.js";
+import { aggregateResults } from "./aggregateResults.js";
+import { basePath, transferEntry } from "./transferEntry.js";
+/**
+ * Resolves the effective destination root for a recursive transfer,
+ * following `cp -r`/`mv` semantics: a destination that doesn't exist becomes
+ * the copy itself; an existing container gets the source nested inside it
+ * as `dest/<source-basename>`; an existing entry is rejected.
+ */
+async function resolveRecursiveDestRoot(sourceKey, sink, destKey) {
+    let destProperties;
+    try {
+        destProperties = await sink.getProperties(destKey);
+    }
+    catch {
+        destProperties = undefined;
+    }
+    if (destProperties === undefined) {
+        return destKey;
+    }
+    if (!destProperties.isContainer) {
+        throw new PermanentIOError(`Cannot copy/move container "${sourceKey}" onto existing entry "${destKey}"`);
+    }
+    return joinKey(sink, destKey, baseName(sourceKey));
+}
+/**
+ * Transfers a whole container. Uses a container-aware
+ * `directCopy`/`directMove` when the source declares
+ * `supportsRecursiveDirectTransfer`; otherwise lists the container and
+ * transfers each entry, bounded by the concurrency limiter. A non-direct
+ * move deletes the source container once, after every entry succeeded and
+ * only if the transfer was not stopped.
+ */
+export async function recursiveTransfer(source, sourceKey, sink, destKey, options, type) {
+    const path = basePath(source, sink, options);
+    const resolvedDestRoot = await resolveRecursiveDestRoot(sourceKey, sink, destKey);
+    if (options.directTransfer !== false &&
+        source.canDirectTransfer?.(sink) &&
+        source.supportsRecursiveDirectTransfer) {
+        const direct = type === "move" ? source.directMove : source.directCopy;
+        if (direct) {
+            throwIfAborted(options.signal);
+            const telemetry = { operationId: crypto.randomUUID(), hooks: options.telemetry ?? {} };
+            await direct.call(source, sourceKey, resolvedDestRoot, telemetry);
+            return { stopped: false, bytes: 0, items: 0, path: `${path}, direct` };
+        }
+    }
+    if (typeof source.list !== "function") {
+        throw new PermanentIOError("The source provider does not support listing containers");
+    }
+    if (type === "move" && typeof source.delete !== "function") {
+        throw new PermanentIOError("The source provider does not support delete, so it cannot move");
+    }
+    const recursiveOperationId = crypto.randomUUID();
+    const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+    let discovered = 0;
+    let completed = 0;
+    const reportProgress = () => {
+        options.telemetry?.onProgress?.({
+            operationId: recursiveOperationId,
+            type,
+            bytesProcessed: 0,
+            entriesProcessed: completed,
+            totalEntries: discovered,
+        });
+    };
+    async function processEntry(entry) {
+        throwIfAborted(options.signal);
+        if (options.stop?.aborted) {
+            return undefined;
+        }
+        discovered += 1;
+        reportProgress();
+        const entryDestKey = joinKey(sink, resolvedDestRoot, entry.path);
+        let result;
+        if (entry.properties.isContainer) {
+            await sink.createContainer?.(entryDestKey);
+        }
+        else {
+            result = await transferEntry({
+                source,
+                sourceKey: joinKey(source, sourceKey, entry.path),
+                sink,
+                destKey: entryDestKey,
+                properties: entry.properties,
+                options,
+                parentOperationId: recursiveOperationId,
+                type,
+                deleteSource: false,
+            });
+        }
+        completed += 1;
+        reportProgress();
+        return result;
+    }
+    const results = [];
+    for await (const result of mapAsyncIterableConcurrently(source.list(sourceKey, { recursive: true }), processEntry, limiter)) {
+        if (result)
+            results.push(result);
+    }
+    const outcome = aggregateResults(results, path, options.stop?.aborted ?? false);
+    if (type === "move" && !outcome.stopped) {
+        await withRetry(() => source.delete.call(source, sourceKey), options.retry ?? DEFAULT_RETRY);
+    }
+    return outcome;
+}

package/dist/src/transfer/streamTransfer.d.ts

@@ -0,0 +1,32 @@
+import { type IOProvider, type StreamHandle } from "@flowscripter/pluggable-io-framework-api";
+import type { EntryContext, EntryOutcome } from "./EntryContext.ts";
+export interface StreamTransferInput {
+    readonly source: IOProvider;
+    readonly sourceKey: string;
+    readonly readable: StreamHandle;
+    readonly sink: IOProvider;
+    readonly destKey: string;
+    readonly writable: StreamHandle;
+    /** Total bytes for a bounded source, when known. */
+    readonly totalBytes?: number;
+    readonly context: EntryContext;
+}
+/**
+ * Pipes a readable handle into a writable handle item by item, honouring
+ * `signal` (abort) and `stop` (graceful close).
+ *
+ * On a `TransientIOError`:
+ * - bounded source with `RangeReadable`, and a writable exposing a resume
+ *   token: the sink is reopened with the token and the source re-read from
+ *   the sink's `startOffset`;
+ * - other bounded transfers restart from scratch;
+ * - unbounded source failure: the source is reopened, the sink kept open,
+ *   the first new item flagged with `discontinuity`, and `onGap` reported;
+ * - unbounded sink failure: the sink is reopened with its resume token, or
+ *   the transfer fails.
+ *
+ * `retry.maxRetries` counts consecutive failures and resets once an item is
+ * written.
+ */
+export declare function streamTransfer(input: StreamTransferInput): Promise<EntryOutcome>;
+//# sourceMappingURL=streamTransfer.d.ts.map

package/dist/src/transfer/streamTransfer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamTransfer.d.ts","sourceRoot":"","sources":["../../../src/transfer/streamTransfer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,UAAU,EAIf,KAAK,YAAY,EAElB,MAAM,0CAA0C,CAAC;AAKlD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAYpE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAChC,oDAAoD;IACpD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;CAChC;AAID;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CA8JtF"}