@flowscripter/pluggable-io-framework 1.2.7 → 2.0.0

package/dist/src/transfer/streamTransfer.js

@@ -0,0 +1,186 @@
+import { isRangeReadable, isResumableWritable, TransientIOError, } from "@flowscripter/pluggable-io-framework-api";
+import { backoff, DEFAULT_RETRY } from "../retry/RetryOptions.js";
+import { createAbortError, watchSignals } from "../util/abort.js";
+import { applyPayloadConverter } from "../util/applyPayloadConverter.js";
+import { itemLength } from "../util/itemLength.js";
+/** Which side of the transfer a failure came from. */
+class SideError extends Error {
+    side;
+    constructor(side, cause) {
+        super("transfer side failed", { cause });
+        this.side = side;
+    }
+}
+function ignore() { }
+/**
+ * Pipes a readable handle into a writable handle item by item, honouring
+ * `signal` (abort) and `stop` (graceful close).
+ *
+ * On a `TransientIOError`:
+ * - bounded source with `RangeReadable`, and a writable exposing a resume
+ *   token: the sink is reopened with the token and the source re-read from
+ *   the sink's `startOffset`;
+ * - other bounded transfers restart from scratch;
+ * - unbounded source failure: the source is reopened, the sink kept open,
+ *   the first new item flagged with `discontinuity`, and `onGap` reported;
+ * - unbounded sink failure: the sink is reopened with its resume token, or
+ *   the transfer fails.
+ *
+ * `retry.maxRetries` counts consecutive failures and resets once an item is
+ * written.
+ */
+export async function streamTransfer(input) {
+    const { source, sourceKey, sink, destKey, totalBytes, context } = input;
+    const { options, hooks, operationId, type } = context;
+    const retry = options.retry ?? DEFAULT_RETRY;
+    const bounded = input.readable.bounded !== false;
+    const watch = watchSignals(options.signal, options.stop);
+    let readable = input.readable;
+    let writable = input.writable;
+    const convert = (stream) => applyPayloadConverter(stream, readable.kind, sink.kind, options.converter);
+    let reader = convert(readable.stream).getReader();
+    let writer = writable.stream.getWriter();
+    let bytes = 0;
+    let items = 0;
+    let failures = 0;
+    let markGap = false;
+    let recover;
+    async function pump() {
+        for (;;) {
+            let read;
+            try {
+                read = await Promise.race([reader.read(), watch.fired]);
+            }
+            catch (error) {
+                throw new SideError("source", error);
+            }
+            if (read === "aborted")
+                throw createAbortError();
+            if (read === "stopped")
+                return "stopped";
+            if (read.done)
+                return "done";
+            let item = read.value;
+            if (markGap) {
+                item = { ...item, attributes: { ...item.attributes, discontinuity: true } };
+                markGap = false;
+            }
+            let written;
+            try {
+                written = await Promise.race([writer.write(item), watch.fired]);
+            }
+            catch (error) {
+                throw new SideError("sink", error);
+            }
+            if (written === "aborted")
+                throw createAbortError();
+            bytes += itemLength(item);
+            items += 1;
+            failures = 0;
+            hooks.onProgress?.({
+                operationId,
+                type,
+                bytesProcessed: bytes,
+                totalBytes: bounded ? totalBytes : undefined,
+            });
+            if (written === "stopped")
+                return "stopped";
+        }
+    }
+    try {
+        for (;;) {
+            try {
+                if (recover) {
+                    const action = recover;
+                    recover = undefined;
+                    try {
+                        await action.run();
+                    }
+                    catch (error) {
+                        throw new SideError(action.side, error);
+                    }
+                }
+                const outcome = await pump();
+                if (outcome === "stopped") {
+                    await reader.cancel().catch(ignore);
+                }
+                await writer.close();
+                return { bytes, items, stopped: outcome === "stopped" };
+            }
+            catch (error) {
+                if (error instanceof DOMException && error.name === "AbortError") {
+                    await reader.cancel(error).catch(ignore);
+                    await writer.abort(error).catch(ignore);
+                    throw error;
+                }
+                const side = error instanceof SideError ? error.side : "sink";
+                const cause = error instanceof SideError ? error.cause : error;
+                failures += 1;
+                const retryable = cause instanceof TransientIOError && failures <= retry.maxRetries;
+                if (retryable && !bounded && side === "source" && retry.onGap !== "fail") {
+                    await reader.cancel().catch(ignore);
+                    const attempt = failures;
+                    recover = {
+                        side: "source",
+                        run: async () => {
+                            await backoff(retry, attempt);
+                            readable = await source.getReadableStream(sourceKey);
+                            reader = convert(readable.stream).getReader();
+                            markGap = true;
+                            hooks.onGap?.(operationId, { atBytes: bytes, attempt });
+                        },
+                    };
+                    continue;
+                }
+                const token = retryable && isResumableWritable(writable) ? writable.resumeToken() : undefined;
+                const canResume = token !== undefined && (!bounded || isRangeReadable(readable));
+                if (retryable && canResume) {
+                    // The failed sink is released, not aborted, so a provider that
+                    // discards partial output on abort keeps it for the resume.
+                    writer.releaseLock();
+                    if (bounded) {
+                        await reader.cancel().catch(ignore);
+                    }
+                    const attempt = failures;
+                    recover = {
+                        side: "sink",
+                        run: async () => {
+                            await backoff(retry, attempt);
+                            const resumed = await sink.getWritableStream(destKey, { resume: token });
+                            writable = resumed;
+                            writer = resumed.stream.getWriter();
+                            if (bounded && isRangeReadable(readable)) {
+                                const startOffset = resumed.startOffset ?? token.offset;
+                                const end = totalBytes ?? Number.MAX_SAFE_INTEGER;
+                                reader = convert((await readable.readRange(startOffset, end))).getReader();
+                                bytes = startOffset;
+                            }
+                        },
+                    };
+                    continue;
+                }
+                await reader.cancel(cause).catch(ignore);
+                await writer.abort(cause).catch(ignore);
+                if (!retryable || !bounded) {
+                    throw cause;
+                }
+                const attempt = failures;
+                recover = {
+                    side: "source",
+                    run: async () => {
+                        await backoff(retry, attempt);
+                        readable = await source.getReadableStream(sourceKey);
+                        reader = convert(readable.stream).getReader();
+                        writable = await sink.getWritableStream(destKey);
+                        writer = writable.stream.getWriter();
+                        bytes = 0;
+                        items = 0;
+                    },
+                };
+            }
+        }
+    }
+    finally {
+        watch.dispose();
+    }
+}

package/dist/src/transfer/transferEntry.d.ts

@@ -0,0 +1,25 @@
+import { type EntryProperties, type IOProvider } from "@flowscripter/pluggable-io-framework-api";
+import type { TransferOptions } from "./TransferOptions.ts";
+import type { TransferResult } from "./TransferResult.ts";
+export interface TransferEntryInput {
+    readonly source: IOProvider;
+    readonly sourceKey: string;
+    readonly sink: IOProvider;
+    readonly destKey: string;
+    readonly properties: EntryProperties;
+    readonly options: TransferOptions;
+    readonly parentOperationId?: string;
+    /** `"move"` rejects unbounded sources and prefers `directMove`. */
+    readonly type: "copy" | "move";
+    /** Whether a non-direct move deletes the source entry afterwards. */
+    readonly deleteSource: boolean;
+}
+/** The path description before the transfer strategy is known. */
+export declare function basePath(source: IOProvider, sink: IOProvider, options: TransferOptions): string;
+/**
+ * Transfers one entry: a direct provider transfer when eligible, otherwise
+ * multipart, the lease path or plain streaming, after checking the payload
+ * type and (for a move) that the source is bounded.
+ */
+export declare function transferEntry(input: TransferEntryInput): Promise<TransferResult>;
+//# sourceMappingURL=transferEntry.d.ts.map

package/dist/src/transfer/transferEntry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transferEntry.d.ts","sourceRoot":"","sources":["../../../src/transfer/transferEntry.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,UAAU,EAOhB,MAAM,0CAA0C,CAAC;AAYlD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,mEAAmE;IACnE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAC/B,qEAAqE;IACrE,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC;AAED,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAM/F;AAgBD;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,cAAc,CAAC,CAmHtF"}

package/dist/src/transfer/transferEntry.js

@@ -0,0 +1,136 @@
+import { BYTES_PAYLOAD_TYPE, isBufferProvider, isFillReadable, isRangeReadable, PermanentIOError, } from "@flowscripter/pluggable-io-framework-api";
+import { defaultConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.js";
+import { DEFAULT_RETRY } from "../retry/RetryOptions.js";
+import { withRetry } from "../retry/withRetry.js";
+import { throwIfAborted } from "../util/abort.js";
+import { describeConverter } from "../util/describePath.js";
+import { withParentOperationId } from "../util/withParentOperationId.js";
+import { leaseDepth, leaseTransfer } from "./leaseTransfer.js";
+import { multipartTransfer } from "./multipartTransfer.js";
+import { negotiatePartSize } from "./negotiatePartSize.js";
+import { streamTransfer } from "./streamTransfer.js";
+const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
+/** The path description before the transfer strategy is known. */
+export function basePath(source, sink, options) {
+    if (options.path !== undefined) {
+        return options.path;
+    }
+    const via = options.converter ? describeConverter(options.converter) : "";
+    return `${source.kind} -> ${sink.kind}${via}`;
+}
+function checkPayloadType(readable, options) {
+    const payloadType = readable.payloadType ?? BYTES_PAYLOAD_TYPE;
+    const accepted = options.writePayloadTypes ?? [BYTES_PAYLOAD_TYPE];
+    if (!accepted.includes(payloadType)) {
+        throw new PermanentIOError(`Source payload type "${payloadType}" is not accepted by the sink (sink accepts: ${accepted.join(", ")})`);
+    }
+}
+async function cancelStream(handle) {
+    await handle.stream.cancel().catch(() => { });
+}
+/**
+ * Transfers one entry: a direct provider transfer when eligible, otherwise
+ * multipart, the lease path or plain streaming, after checking the payload
+ * type and (for a move) that the source is bounded.
+ */
+export async function transferEntry(input) {
+    const { source, sourceKey, sink, destKey, properties, options, type } = input;
+    throwIfAborted(options.signal);
+    const path = basePath(source, sink, options);
+    if (options.stop?.aborted) {
+        return { stopped: true, bytes: 0, items: 0, path };
+    }
+    const operationId = crypto.randomUUID();
+    const hooks = withParentOperationId(options.telemetry, input.parentOperationId);
+    const context = { operationId, hooks, options, type };
+    if (options.directTransfer !== false && source.canDirectTransfer?.(sink)) {
+        const direct = type === "move" ? source.directMove : source.directCopy;
+        if (direct) {
+            await direct.call(source, sourceKey, destKey, { operationId, hooks });
+            return { stopped: false, bytes: properties.size ?? 0, items: 0, path: `${path}, direct` };
+        }
+    }
+    if (input.deleteSource && typeof source.delete !== "function") {
+        throw new PermanentIOError("The source provider does not support delete, so it cannot move");
+    }
+    const retry = options.retry ?? DEFAULT_RETRY;
+    let readable = await withRetry(() => source.getReadableStream(sourceKey), retry);
+    try {
+        checkPayloadType(readable, options);
+        if (type === "move" && readable.bounded === false) {
+            throw new PermanentIOError("cannot move a live source");
+        }
+    }
+    catch (error) {
+        await cancelStream(readable);
+        throw error;
+    }
+    let outcome;
+    let strategy;
+    const size = properties.size;
+    const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
+    const multipartWriter = readable.bounded !== false &&
+        size !== undefined &&
+        size >= threshold &&
+        isRangeReadable(readable) &&
+        typeof sink.getMultipartWriter === "function"
+        ? sink.getMultipartWriter.bind(sink)
+        : undefined;
+    const partSize = multipartWriter && size !== undefined ? negotiatePartSize(source, sink, size) : undefined;
+    if (multipartWriter &&
+        partSize !== undefined &&
+        size !== undefined &&
+        isRangeReadable(readable)) {
+        await cancelStream(readable);
+        const rangeReadable = readable;
+        outcome = await withRetry(() => multipartTransfer({
+            readable: rangeReadable,
+            sink,
+            writer: multipartWriter(destKey, partSize),
+            totalBytes: size,
+            partSize,
+            limiter: options.concurrencyLimiter ?? defaultConcurrencyLimiter,
+            context,
+        }), retry);
+        strategy = "multipart";
+    }
+    else {
+        let writable = await withRetry(() => sink.getWritableStream(destKey), retry);
+        if (!options.converter &&
+            isFillReadable(readable) &&
+            isBufferProvider(writable) &&
+            readable.domains.includes(writable.domain)) {
+            const depth = leaseDepth(writable, options.leaseDepth);
+            let attempt = 0;
+            outcome = await withRetry(async () => {
+                if (attempt > 0) {
+                    readable = await source.getReadableStream(sourceKey);
+                    writable = await sink.getWritableStream(destKey);
+                }
+                attempt += 1;
+                if (!isFillReadable(readable) || !isBufferProvider(writable)) {
+                    throw new PermanentIOError("Reopened handles no longer support the lease path");
+                }
+                return leaseTransfer(readable, writable, depth, context);
+            }, retry);
+            strategy = `lease (depth ${depth})`;
+        }
+        else {
+            outcome = await streamTransfer({
+                source,
+                sourceKey,
+                readable,
+                sink,
+                destKey,
+                writable,
+                totalBytes: size,
+                context,
+            });
+            strategy = "stream";
+        }
+    }
+    if (input.deleteSource && !outcome.stopped) {
+        await withRetry(() => source.delete.call(source, sourceKey), retry);
+    }
+    return { ...outcome, path: `${path}, ${strategy}` };
+}

package/dist/src/util/abort.d.ts

@@ -0,0 +1,12 @@
+/** The error a transfer rejects with when cancelled through `TransferOptions.signal`. */
+export declare function createAbortError(): DOMException;
+export declare function throwIfAborted(signal: AbortSignal | undefined): void;
+/**
+ * Resolves with `"aborted"` or `"stopped"` as soon as `signal` or `stop`
+ * fires, and never otherwise. `dispose` removes the listeners.
+ */
+export declare function watchSignals(signal: AbortSignal | undefined, stop: AbortSignal | undefined): {
+    readonly fired: Promise<"aborted" | "stopped">;
+    dispose(): void;
+};
+//# sourceMappingURL=abort.d.ts.map

package/dist/src/util/abort.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"abort.d.ts","sourceRoot":"","sources":["../../../src/util/abort.ts"],"names":[],"mappings":"AAAA,yFAAyF;AACzF,wBAAgB,gBAAgB,IAAI,YAAY,CAE/C;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI,CAIpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,IAAI,EAAE,WAAW,GAAG,SAAS,GAC5B;IAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;IAAC,OAAO,IAAI,IAAI,CAAA;CAAE,CAuBrE"}

package/dist/src/util/abort.js

@@ -0,0 +1,39 @@
+/** The error a transfer rejects with when cancelled through `TransferOptions.signal`. */
+export function createAbortError() {
+    return new DOMException("The transfer was aborted", "AbortError");
+}
+export function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw createAbortError();
+    }
+}
+/**
+ * Resolves with `"aborted"` or `"stopped"` as soon as `signal` or `stop`
+ * fires, and never otherwise. `dispose` removes the listeners.
+ */
+export function watchSignals(signal, stop) {
+    const cleanups = [];
+    const fired = new Promise((resolve) => {
+        for (const [target, outcome] of [
+            [signal, "aborted"],
+            [stop, "stopped"],
+        ]) {
+            if (!target)
+                continue;
+            if (target.aborted) {
+                resolve(outcome);
+                continue;
+            }
+            const listener = () => resolve(outcome);
+            target.addEventListener("abort", listener, { once: true });
+            cleanups.push(() => target.removeEventListener("abort", listener));
+        }
+    });
+    return {
+        fired,
+        dispose() {
+            for (const cleanup of cleanups)
+                cleanup();
+        },
+    };
+}

package/dist/src/util/applyPayloadConverter.d.ts

@@ -0,0 +1,9 @@
+import { type Item, type PayloadConverter, type PayloadKind } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Converts every item of `stream` for a sink of `toKind`. With a converter,
+ * each item goes through `converter.convert`, including when the kinds match
+ * and only the memory domain differs. Without one, a kind mismatch fails on
+ * the first item.
+ */
+export declare function applyPayloadConverter(stream: ReadableStream<Item>, fromKind: PayloadKind, toKind: PayloadKind, converter: PayloadConverter | undefined): ReadableStream<Item>;
+//# sourceMappingURL=applyPayloadConverter.d.ts.map

package/dist/src/util/applyPayloadConverter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"applyPayloadConverter.d.ts","sourceRoot":"","sources":["../../../src/util/applyPayloadConverter.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,IAAI,EACT,KAAK,gBAAgB,EACrB,KAAK,WAAW,EACjB,MAAM,0CAA0C,CAAC;AAElD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,EAC5B,QAAQ,EAAE,WAAW,EACrB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,gBAAgB,GAAG,SAAS,GACtC,cAAc,CAAC,IAAI,CAAC,CAkBtB"}

package/dist/src/util/applyPayloadConverter.js

@@ -0,0 +1,26 @@
+import { adaptReadableStream, } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Converts every item of `stream` for a sink of `toKind`. With a converter,
+ * each item goes through `converter.convert`, including when the kinds match
+ * and only the memory domain differs. Without one, a kind mismatch fails on
+ * the first item.
+ */
+export function applyPayloadConverter(stream, fromKind, toKind, converter) {
+    if (!converter) {
+        return adaptReadableStream(stream, fromKind, toKind);
+    }
+    const reader = stream.getReader();
+    return new ReadableStream({
+        async pull(controller) {
+            const { done, value } = await reader.read();
+            if (done) {
+                controller.close();
+                return;
+            }
+            controller.enqueue(converter.convert(value));
+        },
+        cancel(reason) {
+            return reader.cancel(reason);
+        },
+    });
+}

package/dist/src/util/describePath.d.ts

@@ -0,0 +1,6 @@
+import type { PayloadConverter, PayloadKind } from "@flowscripter/pluggable-io-framework-api";
+/** `kind` or `kind[domain]`. */
+export declare function describeKind(kind: PayloadKind, domain?: string): string;
+/** ` via from->to (zero-copy)` or ` via from->to (copy)`. */
+export declare function describeConverter(converter: PayloadConverter): string;
+//# sourceMappingURL=describePath.d.ts.map

package/dist/src/util/describePath.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"describePath.d.ts","sourceRoot":"","sources":["../../../src/util/describePath.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AAE9F,gCAAgC;AAChC,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAEvE;AAED,6DAA6D;AAC7D,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,gBAAgB,GAAG,MAAM,CAGrE"}

package/dist/src/util/describePath.js

@@ -0,0 +1,9 @@
+/** `kind` or `kind[domain]`. */
+export function describeKind(kind, domain) {
+    return domain === undefined ? kind : `${kind}[${domain}]`;
+}
+/** ` via from->to (zero-copy)` or ` via from->to (copy)`. */
+export function describeConverter(converter) {
+    const cost = converter.cost === 0 ? "zero-copy" : "copy";
+    return ` via ${describeKind(converter.from.kind, converter.from.domain)}->${describeKind(converter.to.kind, converter.to.domain)} (${cost})`;
+}

package/dist/src/util/globToRegex.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Translates a glob into an anchored regular expression matched against a
+ * single entry name: `*` matches any run of characters except `/`, `?`
+ * matches one such character, and `[...]` is a character class (`[!...]`
+ * negates it). Everything else matches literally.
+ */
+export declare function globToRegex(glob: string): RegExp;
+//# sourceMappingURL=globToRegex.d.ts.map

package/dist/src/util/globToRegex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"globToRegex.d.ts","sourceRoot":"","sources":["../../../src/util/globToRegex.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAyBhD"}

package/dist/src/util/globToRegex.js

@@ -0,0 +1,35 @@
+/**
+ * Translates a glob into an anchored regular expression matched against a
+ * single entry name: `*` matches any run of characters except `/`, `?`
+ * matches one such character, and `[...]` is a character class (`[!...]`
+ * negates it). Everything else matches literally.
+ */
+export function globToRegex(glob) {
+    let source = "";
+    for (let i = 0; i < glob.length; i += 1) {
+        const char = glob[i];
+        if (char === "*") {
+            source += "[^/]*";
+        }
+        else if (char === "?") {
+            source += "[^/]";
+        }
+        else if (char === "[") {
+            const close = glob.indexOf("]", i + 2);
+            if (close === -1) {
+                source += "\\[";
+                continue;
+            }
+            let body = glob.slice(i + 1, close);
+            if (body.startsWith("!")) {
+                body = `^${body.slice(1)}`;
+            }
+            source += `[${body.replace(/\\/g, "\\\\")}]`;
+            i = close;
+        }
+        else {
+            source += char.replace(/[.+^${}()|\\/\]]/g, "\\$&");
+        }
+    }
+    return new RegExp(`^${source}$`);
+}

package/dist/src/util/itemLength.d.ts

@@ -0,0 +1,4 @@
+import { type Item } from "@flowscripter/pluggable-io-framework-api";
+/** The payload size of an item in bytes. */
+export declare function itemLength(item: Item): number;
+//# sourceMappingURL=itemLength.d.ts.map

package/dist/src/util/itemLength.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"itemLength.d.ts","sourceRoot":"","sources":["../../../src/util/itemLength.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,EAAe,MAAM,0CAA0C,CAAC;AAElF,4CAA4C;AAC5C,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAE7C"}

package/dist/src/util/itemLength.js

@@ -0,0 +1,5 @@
+import { PayloadKind } from "@flowscripter/pluggable-io-framework-api";
+/** The payload size of an item in bytes. */
+export function itemLength(item) {
+    return item.payload.kind === PayloadKind.Js ? item.payload.data.byteLength : item.payload.length;
+}

package/dist/src/util/keys.d.ts

@@ -0,0 +1,8 @@
+import type { IOProvider } from "@flowscripter/pluggable-io-framework-api";
+/** Joins with `/` - the default when a provider has no `joinKey`. */
+export declare function defaultJoinKey(containerKey: string, name: string): string;
+/** Joins a child name onto a container key using the provider's own rule. */
+export declare function joinKey(provider: IOProvider, containerKey: string, name: string): string;
+/** The last segment of a key, treating both `/` and `\` as separators. */
+export declare function baseName(key: string): string;
+//# sourceMappingURL=keys.d.ts.map

package/dist/src/util/keys.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keys.d.ts","sourceRoot":"","sources":["../../../src/util/keys.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0CAA0C,CAAC;AAE3E,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAIzE;AAED,6EAA6E;AAC7E,wBAAgB,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxF;AAED,0EAA0E;AAC1E,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAG5C"}

package/dist/src/util/keys.js

@@ -0,0 +1,19 @@
+/** Joins with `/` - the default when a provider has no `joinKey`. */
+export function defaultJoinKey(containerKey, name) {
+    if (name === "")
+        return containerKey;
+    if (containerKey === "")
+        return name;
+    return `${containerKey.replace(/\/+$/, "")}/${name}`;
+}
+/** Joins a child name onto a container key using the provider's own rule. */
+export function joinKey(provider, containerKey, name) {
+    return provider.joinKey
+        ? provider.joinKey(containerKey, name)
+        : defaultJoinKey(containerKey, name);
+}
+/** The last segment of a key, treating both `/` and `\` as separators. */
+export function baseName(key) {
+    const segments = key.split(/[\\/]+/).filter((segment) => segment !== "");
+    return segments.at(-1) ?? key;
+}

package/dist/src/util/withParentOperationId.d.ts

@@ -0,0 +1,4 @@
+import type { TelemetryHooks } from "@flowscripter/pluggable-io-framework-api";
+/** Wraps `hooks` so any `onProgress` event passing through gets `parentOperationId` filled in, unless the emitter already set one (nested hierarchy). */
+export declare function withParentOperationId(hooks: TelemetryHooks | undefined, parentOperationId: string | undefined): TelemetryHooks;
+//# sourceMappingURL=withParentOperationId.d.ts.map

package/dist/src/util/withParentOperationId.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"withParentOperationId.d.ts","sourceRoot":"","sources":["../../../src/util/withParentOperationId.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0CAA0C,CAAC;AAE/E,yJAAyJ;AACzJ,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,cAAc,GAAG,SAAS,EACjC,iBAAiB,EAAE,MAAM,GAAG,SAAS,GACpC,cAAc,CAUhB"}

package/dist/src/util/withParentOperationId.js

@@ -0,0 +1,12 @@
+/** Wraps `hooks` so any `onProgress` event passing through gets `parentOperationId` filled in, unless the emitter already set one (nested hierarchy). */
+export function withParentOperationId(hooks, parentOperationId) {
+    if (!hooks || parentOperationId === undefined) {
+        return hooks ?? {};
+    }
+    return {
+        ...hooks,
+        onProgress: hooks.onProgress
+            ? (event) => hooks.onProgress?.({ parentOperationId, ...event })
+            : undefined,
+    };
+}

package/index.ts

@@ -1,6 +1,13 @@
-export * from "./src/ConcurrencyLimiter.ts";
-export * from "./src/ProviderRegistry.ts";
-export * from "./src/copyMove.ts";
+export * from "./src/concurrency/ConcurrencyLimiter.ts";
+export * from "./src/concurrency/mapAsyncIterableConcurrently.ts";
 export * from "./src/decorators/locallyCached.ts";
 export * from "./src/decorators/seekable.ts";
-export * from "./src/withRetry.ts";
+export * from "./src/registry/detectProtocol.ts";
+export * from "./src/registry/ProviderRegistry.ts";
+export * from "./src/retry/RetryOptions.ts";
+export * from "./src/retry/withRetry.ts";
+export * from "./src/transfer/copyMove.ts";
+export * from "./src/transfer/rangeReadableMultipartReader.ts";
+export * from "./src/transfer/TransferOptions.ts";
+export * from "./src/transfer/TransferResult.ts";
+export * from "./src/util/globToRegex.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowscripter/pluggable-io-framework",
-  "version": "1.2.7",
+  "version": "2.0.0",
   "description": "A pluggable source/sink IO framework using https://github.com/flowscripter/dynamic-plugin-framework",
   "keywords": [
     "bun",
@@ -41,12 +41,13 @@
   },
   "dependencies": {
     "@flowscripter/dynamic-plugin-framework": "^2.2.14",
-    "@flowscripter/pluggable-io-framework-api": "^1.1.6"
+    "@flowscripter/pluggable-io-framework-api": "^3.0.0"
   },
   "devDependencies": {
     "@types/bun": "^1.4.2",
     "oxfmt": "0.71.0",
-    "oxlint": "1.86.0"
+    "oxlint": "1.86.0",
+    "zod": "^4.6.5"
   },
   "peerDependencies": {
     "typescript": "^7.0.2"