@flowscripter/pluggable-io-framework 1.2.6 → 2.0.0

package/dist/src/copyMove.js

@@ -1,280 +0,0 @@
-import { adaptReadableStream, identityChunkConverter, } from "@flowscripter/pluggable-io-framework-api";
-import { chunkLength } from "./chunkLength.js";
-import { defaultConcurrencyLimiter, mapAsyncIterableConcurrently, } from "./ConcurrencyLimiter.js";
-import { withRetry } from "./withRetry.js";
-const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
-const DEFAULT_PART_SIZE = 8 * 1024 * 1024;
-const DEFAULT_RETRY = { maxRetries: 3 };
-const UNCONSTRAINED_PART_SIZE = {
-    minPartSize: 0,
-    maxPartSize: Infinity,
-    maxParts: Infinity,
-    defaultPartSize: DEFAULT_PART_SIZE,
-};
-/** Wraps `hooks` so any `onProgress` event passing through gets `parentOperationId` filled in, unless the emitter already set one (nested hierarchy). */
-function withParentOperationId(hooks, parentOperationId) {
-    if (!hooks || parentOperationId === undefined) {
-        return hooks ?? {};
-    }
-    return {
-        ...hooks,
-        onProgress: hooks.onProgress
-            ? (event) => hooks.onProgress?.({ parentOperationId, ...event })
-            : undefined,
-    };
-}
-function joinPath(base, relative) {
-    if (relative === "")
-        return base;
-    if (base === "")
-        return relative;
-    return `${base.replace(/\/+$/, "")}/${relative}`;
-}
-/**
- * Reconciles `source`'s and `sink`'s {@link PartSizeConstraints} for a
- * transfer of `totalSize` bytes into a single part size satisfying both.
- * Returns `undefined` when the bounds are mutually infeasible (source's
- * minimum exceeds sink's maximum) - the caller should fall back to plain
- * streaming rather than throwing.
- */
-function negotiatePartSize(source, sink, totalSize) {
-    const sourceConstraints = source.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
-    const sinkConstraints = sink.getPartSizeConstraints?.(totalSize) ?? UNCONSTRAINED_PART_SIZE;
-    const minPartSize = Math.max(sourceConstraints.minPartSize, sinkConstraints.minPartSize);
-    const maxPartSize = Math.min(sourceConstraints.maxPartSize, sinkConstraints.maxPartSize);
-    const maxParts = Math.min(sourceConstraints.maxParts, sinkConstraints.maxParts);
-    if (minPartSize > maxPartSize) {
-        return undefined;
-    }
-    const byMaxParts = Math.ceil(totalSize / maxParts);
-    const candidate = Math.max(minPartSize, sourceConstraints.defaultPartSize, sinkConstraints.defaultPartSize, byMaxParts);
-    return Math.min(candidate, maxPartSize);
-}
-function canUseMultipart(source, sink) {
-    return (typeof source.getMultipartReader === "function" && typeof sink.getMultipartWriter === "function");
-}
-async function streamingTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, options, totalBytes) {
-    const hooks = withParentOperationId(options.telemetry, parentOperationId);
-    const readable = await source.getReadableStream(sourcePath);
-    const writable = await sink.getWritableStream(destPath);
-    // The source/sink kind mismatch is decided ONCE here, not per chunk.
-    const adaptedStream = adaptReadableStream(readable.stream, readable.kind, writable.kind, options.chunkConverter ?? identityChunkConverter);
-    const reader = adaptedStream.getReader();
-    const writer = writable.stream.getWriter();
-    let bytesProcessed = 0;
-    try {
-        for (;;) {
-            const { done, value } = await reader.read();
-            if (done)
-                break;
-            await writer.write(value);
-            bytesProcessed += chunkLength(value);
-            hooks.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
-        }
-        await writer.close();
-    }
-    catch (error) {
-        await writer.abort(error);
-        throw error;
-    }
-}
-async function multipartTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, partSize, limiter, options, totalBytes) {
-    const hooks = withParentOperationId(options.telemetry, parentOperationId);
-    let bytesProcessed = 0;
-    async function transferPart(sourcePart) {
-        const partOperationId = crypto.randomUUID();
-        // Kind mismatch decided once per part, not once per chunk within it.
-        const adaptedStream = adaptReadableStream(sourcePart.stream, sourcePart.kind, sink.kind, options.chunkConverter ?? identityChunkConverter);
-        const reader = adaptedStream.getReader();
-        const chunks = [];
-        let partBytesProcessed = 0;
-        for (;;) {
-            const { done, value } = await reader.read();
-            if (done)
-                break;
-            chunks.push(value);
-            const length = chunkLength(value);
-            partBytesProcessed += length;
-            bytesProcessed += length;
-            hooks.onProgress?.({
-                operationId: partOperationId,
-                parentOperationId: operationId,
-                type: "copy",
-                bytesProcessed: partBytesProcessed,
-            });
-            hooks.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
-        }
-        return {
-            index: sourcePart.index,
-            offset: sourcePart.offset,
-            kind: sink.kind,
-            stream: new ReadableStream({
-                start(controller) {
-                    for (const chunk of chunks)
-                        controller.enqueue(chunk);
-                    controller.close();
-                },
-            }),
-            complete: () => sourcePart.complete(),
-        };
-    }
-    const writer = sink.getMultipartWriter(destPath, partSize);
-    const transferredParts = mapAsyncIterableConcurrently(source.getMultipartReader(sourcePath, partSize), transferPart, limiter);
-    await writer.write(transferredParts);
-}
-async function copySingle(source, sourcePath, sink, destPath, options, parentOperationId, properties) {
-    const operationId = crypto.randomUUID();
-    if (source.canDirectTransfer?.(sink) && source.directCopy) {
-        await source.directCopy(sourcePath, destPath, {
-            operationId,
-            hooks: withParentOperationId(options.telemetry, parentOperationId),
-        });
-        return;
-    }
-    const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
-    const size = properties.size;
-    const partSize = size !== undefined && size >= threshold && canUseMultipart(source, sink)
-        ? negotiatePartSize(source, sink, size)
-        : undefined;
-    const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
-    const retryOptions = options.retry ?? DEFAULT_RETRY;
-    await withRetry(async () => {
-        if (partSize !== undefined) {
-            await multipartTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, partSize, limiter, options, size);
-        }
-        else {
-            await streamingTransfer(source, sourcePath, sink, destPath, operationId, parentOperationId, options, size);
-        }
-    }, retryOptions);
-}
-async function moveSingle(source, sourcePath, sink, destPath, options, parentOperationId, properties) {
-    if (source.canDirectTransfer?.(sink) && source.directMove) {
-        const operationId = crypto.randomUUID();
-        await source.directMove(sourcePath, destPath, {
-            operationId,
-            hooks: withParentOperationId(options.telemetry, parentOperationId),
-        });
-        return;
-    }
-    await copySingle(source, sourcePath, sink, destPath, options, parentOperationId, properties);
-    await withRetry(() => source.delete(sourcePath), options.retry ?? DEFAULT_RETRY);
-}
-/**
- * Resolves the effective destination root for a recursive transfer,
- * following `cp -r`/`mv` semantics: a destination that doesn't exist becomes
- * the copy itself; an existing folder gets the source nested inside it as
- * `dest/<source-basename>`; an existing file is rejected.
- */
-async function resolveRecursiveDestRoot(sourcePath, sink, destPath) {
-    let destProperties;
-    try {
-        destProperties = await sink.getProperties(destPath);
-    }
-    catch {
-        destProperties = undefined;
-    }
-    if (destProperties === undefined) {
-        return destPath;
-    }
-    if (!destProperties.isFolder) {
-        throw new Error(`Cannot copy/move folder "${sourcePath}" onto existing file "${destPath}"`);
-    }
-    const baseName = sourcePath.replace(/\/+$/, "").split("/").pop() || sourcePath;
-    return joinPath(destPath, baseName);
-}
-async function recursiveTransfer(source, sourcePath, sink, destPath, options, mode) {
-    const resolvedDestRoot = await resolveRecursiveDestRoot(sourcePath, sink, destPath);
-    if (source.canDirectTransfer?.(sink) && source.supportsRecursiveDirectTransfer) {
-        const telemetry = { operationId: crypto.randomUUID(), hooks: options.telemetry ?? {} };
-        if (mode === "copy" && source.directCopy) {
-            await source.directCopy(sourcePath, resolvedDestRoot, telemetry);
-            return;
-        }
-        if (mode === "move" && source.directMove) {
-            await source.directMove(sourcePath, resolvedDestRoot, telemetry);
-            return;
-        }
-    }
-    const recursiveOperationId = crypto.randomUUID();
-    const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
-    let discovered = 0;
-    let completed = 0;
-    const reportProgress = () => {
-        options.telemetry?.onProgress?.({
-            operationId: recursiveOperationId,
-            type: mode,
-            bytesProcessed: 0,
-            itemsProcessed: completed,
-            totalItems: discovered,
-        });
-    };
-    async function processEntry(entry) {
-        discovered += 1;
-        reportProgress();
-        const entrySourcePath = joinPath(sourcePath, entry.path);
-        const entryDestPath = joinPath(resolvedDestRoot, entry.path);
-        if (entry.properties.isFolder) {
-            await sink.createFolder?.(entryDestPath);
-        }
-        else if (mode === "copy") {
-            await copySingle(source, entrySourcePath, sink, entryDestPath, options, recursiveOperationId, entry.properties);
-        }
-        else if (source.canDirectTransfer?.(sink) && source.directMove) {
-            // A per-file directMove deletes its source atomically as part of the
-            // provider's own operation - safe to do immediately. The non-direct
-            // case below must NOT delete per entry: only after every entry
-            // succeeds does the caller delete the whole source root once.
-            const operationId = crypto.randomUUID();
-            await source.directMove(entrySourcePath, entryDestPath, {
-                operationId,
-                hooks: withParentOperationId(options.telemetry, recursiveOperationId),
-            });
-        }
-        else {
-            await copySingle(source, entrySourcePath, sink, entryDestPath, options, recursiveOperationId, entry.properties);
-        }
-        completed += 1;
-        reportProgress();
-    }
-    for await (const _entry of mapAsyncIterableConcurrently(source.list(sourcePath, { recursive: true }), processEntry, limiter)) {
-        // work happens inside processEntry - nothing to do with the (void) result.
-    }
-    if (mode === "move") {
-        await withRetry(() => source.delete(sourcePath), options.retry ?? DEFAULT_RETRY);
-    }
-}
-/**
- * Copies `sourcePath` on `source` to `destPath` on `sink`. When `sourcePath`
- * is a folder, recurses: uses a folder-aware `directCopy` when the source
- * declares `supportsRecursiveDirectTransfer`, otherwise lists the folder and
- * transfers each entry (bounded by `options.concurrencyLimiter`), following
- * `cp -r` target semantics (see {@link resolveRecursiveDestRoot}).
- *
- * For a single file: uses `source.directCopy` when
- * `source.canDirectTransfer?.(sink)` reports eligibility. Otherwise falls
- * back to multipart transfer (when both sides support it, size crosses
- * `options.multipartThreshold`, and part-size negotiation succeeds) or plain
- * streaming - retried per `options.retry` on `TransientIOError`.
- */
-export async function copy(source, sourcePath, sink, destPath, options = {}) {
-    const properties = await source.getProperties(sourcePath);
-    if (properties.isFolder) {
-        await recursiveTransfer(source, sourcePath, sink, destPath, options, "copy");
-        return;
-    }
-    await copySingle(source, sourcePath, sink, destPath, options, undefined, properties);
-}
-/**
- * Moves `sourcePath` on `source` to `destPath` on `sink`. Folder handling
- * mirrors {@link copy}. For a single file, uses `source.directMove` when
- * eligible, otherwise performs a {@link copy} followed by deleting the
- * source item (each retried independently per `options.retry`).
- */
-export async function move(source, sourcePath, sink, destPath, options = {}) {
-    const properties = await source.getProperties(sourcePath);
-    if (properties.isFolder) {
-        await recursiveTransfer(source, sourcePath, sink, destPath, options, "move");
-        return;
-    }
-    await moveSingle(source, sourcePath, sink, destPath, options, undefined, properties);
-}

package/dist/src/withRetry.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"withRetry.d.ts","sourceRoot":"","sources":["../../src/withRetry.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,sGAAsG;IACtG,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAMD;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,CAc1F"}

package/src/ConcurrencyLimiter.ts

@@ -1,178 +0,0 @@
-const DEFAULT_MAX_CONCURRENCY = 4;
-
-/**
- * A semaphore bounding how many `run()` callbacks execute at once. A single
- * shared instance (see {@link defaultConcurrencyLimiter}) enforces its cap
- * *globally* across concurrently invoked `copy()`/`move()` calls, not just
- * within one call - two folder copies started at the same time and sharing
- * a limiter will never together exceed its `maxConcurrency`.
- */
-export class ConcurrencyLimiter {
-  #maxConcurrency: number;
-  #active = 0;
-  #queue: (() => void)[] = [];
-
-  public constructor(maxConcurrency: number) {
-    ConcurrencyLimiter.#validate(maxConcurrency);
-    this.#maxConcurrency = maxConcurrency;
-  }
-
-  public get maxConcurrency(): number {
-    return this.#maxConcurrency;
-  }
-
-  public setMaxConcurrency(maxConcurrency: number): void {
-    ConcurrencyLimiter.#validate(maxConcurrency);
-    this.#maxConcurrency = maxConcurrency;
-    this.#drain();
-  }
-
-  public async run<T>(fn: () => Promise<T>): Promise<T> {
-    await this.#acquire();
-    try {
-      return await fn();
-    } finally {
-      this.#active -= 1;
-      this.#drain();
-    }
-  }
-
-  #acquire(): Promise<void> {
-    if (this.#active < this.#maxConcurrency) {
-      this.#active += 1;
-      return Promise.resolve();
-    }
-    return new Promise((resolve) => {
-      this.#queue.push(() => {
-        this.#active += 1;
-        resolve();
-      });
-    });
-  }
-
-  #drain(): void {
-    while (this.#active < this.#maxConcurrency && this.#queue.length > 0) {
-      this.#queue.shift()?.();
-    }
-  }
-
-  static #validate(maxConcurrency: number): void {
-    if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
-      throw new Error(`maxConcurrency must be a positive integer, got ${maxConcurrency}`);
-    }
-  }
-}
-
-/**
- * Process-wide default limiter, used by `copy()`/`move()` whenever
- * `TransferOptions.concurrencyLimiter` is omitted - so any two calls in the
- * same process share one cap unless a caller explicitly opts out with its
- * own instance (e.g. for test isolation).
- */
-export const defaultConcurrencyLimiter = new ConcurrencyLimiter(DEFAULT_MAX_CONCURRENCY);
-
-/** Minimal single-consumer async FIFO queue backing {@link mapAsyncIterableConcurrently}. */
-class AsyncChannel<T> implements AsyncIterable<T> {
-  #items: T[] = [];
-  #waiters: { resolve: (result: IteratorResult<T>) => void; reject: (error: unknown) => void }[] =
-    [];
-  #closed = false;
-  #hasError = false;
-  #error: unknown;
-
-  push(item: T): void {
-    const waiter = this.#waiters.shift();
-    if (waiter) {
-      waiter.resolve({ value: item, done: false });
-    } else {
-      this.#items.push(item);
-    }
-  }
-
-  close(): void {
-    this.#closed = true;
-    for (const waiter of this.#waiters.splice(0)) {
-      if (this.#hasError) {
-        waiter.reject(this.#error);
-      } else {
-        waiter.resolve({ value: undefined as never, done: true });
-      }
-    }
-  }
-
-  fail(error: unknown): void {
-    this.#hasError = true;
-    this.#error = error;
-    this.close();
-  }
-
-  async #next(): Promise<IteratorResult<T>> {
-    if (this.#items.length > 0) {
-      return { value: this.#items.shift() as T, done: false };
-    }
-    if (this.#closed) {
-      if (this.#hasError) {
-        throw this.#error;
-      }
-      return { value: undefined as never, done: true };
-    }
-    return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
-  }
-
-  [Symbol.asyncIterator](): AsyncIterator<T> {
-    return { next: () => this.#next() };
-  }
-}
-
-/**
- * Pulls items from `source` and runs `fn` on each, bounded by `limiter`:
- * exactly `limiter.maxConcurrency` worker loops each pull-then-process one
- * item at a time through `limiter.run`, so at most that many items are ever
- * pulled/in-flight at once (important for sources like a multipart reader
- * where pulling an item eagerly opens a file/network handle). Results are
- * yielded in completion order, not source order. If `fn` throws, no new
- * items are pulled but already-in-flight ones are allowed to finish before
- * the error is rethrown from the returned iterable.
- */
-export function mapAsyncIterableConcurrently<T, R>(
-  source: AsyncIterable<T>,
-  fn: (item: T) => Promise<R>,
-  limiter: ConcurrencyLimiter,
-): AsyncIterable<R> {
-  const channel = new AsyncChannel<R>();
-  const iterator = source[Symbol.asyncIterator]();
-  let stopped = false;
-  let firstError: unknown;
-  let errorRecorded = false;
-
-  async function worker(): Promise<void> {
-    for (;;) {
-      if (stopped) return;
-      const { done, value } = await iterator.next();
-      if (done) return;
-      try {
-        const result = await limiter.run(() => fn(value));
-        channel.push(result);
-      } catch (error) {
-        stopped = true;
-        if (!errorRecorded) {
-          errorRecorded = true;
-          firstError = error;
-        }
-        return;
-      }
-    }
-  }
-
-  void (async () => {
-    const workerCount = Math.max(1, limiter.maxConcurrency);
-    await Promise.all(Array.from({ length: workerCount }, () => worker()));
-    if (errorRecorded) {
-      channel.fail(firstError);
-    } else {
-      channel.close();
-    }
-  })();
-
-  return channel;
-}

package/src/ProviderRegistry.ts

@@ -1,32 +0,0 @@
-import type { PluginManager } from "@flowscripter/dynamic-plugin-framework";
-import {
-  PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
-  type IOProvider,
-  type IOProviderFactory,
-} from "@flowscripter/pluggable-io-framework-api";
-
-/**
- * Wraps a `dynamic-plugin-framework` {@link PluginManager} to discover and
- * instantiate pluggable-io-framework source/sink provider plugins.
- */
-export class ProviderRegistry {
-  public constructor(private readonly pluginManager: PluginManager) {}
-
-  public async discover(): Promise<void> {
-    await this.pluginManager.registerExtensions(
-      PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
-    );
-  }
-
-  public async listAvailableProviders() {
-    return this.pluginManager.getRegisteredExtensions(
-      PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
-    );
-  }
-
-  public async createProvider(extensionHandle: string, config: unknown): Promise<IOProvider> {
-    const factory = (await this.pluginManager.instantiate(extensionHandle)) as IOProviderFactory;
-    const validatedConfig = factory.configSchema.parse(config);
-    return factory.createProvider(validatedConfig);
-  }
-}

package/src/chunkLength.ts

@@ -1,5 +0,0 @@
-import { ChunkKind, type ChunkRef } from "@flowscripter/pluggable-io-framework-api";
-
-export function chunkLength(chunk: ChunkRef): number {
-  return chunk.kind === ChunkKind.Js ? chunk.data.byteLength : chunk.length;
-}