@flowscripter/pluggable-io-framework 1.2.7 → 2.0.0

package/src/transfer/patternTransfer.ts

@@ -0,0 +1,84 @@
+import {
+  type EntryProperties,
+  type IOProvider,
+  PermanentIOError,
+} from "@flowscripter/pluggable-io-framework-api";
+import { defaultConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.ts";
+import { mapAsyncIterableConcurrently } from "../concurrency/mapAsyncIterableConcurrently.ts";
+import { throwIfAborted } from "../util/abort.ts";
+import { globToRegex } from "../util/globToRegex.ts";
+import { joinKey } from "../util/keys.ts";
+import { aggregateResults } from "./aggregateResults.ts";
+import { basePath, transferEntry } from "./transferEntry.ts";
+import type { TransferOptions } from "./TransferOptions.ts";
+import type { TransferResult } from "./TransferResult.ts";
+
+/**
+ * Transfers every entry directly inside `containerKey` whose name matches
+ * the glob `pattern` into the destination container, one single-entry
+ * transfer per match, bounded by the concurrency limiter. Matching
+ * containers are skipped. A move deletes each source entry after it is
+ * transferred.
+ */
+export async function patternTransfer(
+  source: IOProvider,
+  containerKey: string,
+  pattern: string,
+  sink: IOProvider,
+  destContainerKey: string,
+  options: TransferOptions,
+  type: "copy" | "move",
+): Promise<TransferResult> {
+  if (typeof source.list !== "function") {
+    throw new PermanentIOError("The source provider does not support listing containers");
+  }
+  const operationId = crypto.randomUUID();
+  const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+  let discovered = 0;
+  let completed = 0;
+  const reportProgress = (): void => {
+    options.telemetry?.onProgress?.({
+      operationId,
+      type,
+      bytesProcessed: 0,
+      entriesProcessed: completed,
+      totalEntries: discovered,
+    });
+  };
+
+  async function processEntry(entry: {
+    path: string;
+    properties: EntryProperties;
+  }): Promise<TransferResult | undefined> {
+    throwIfAborted(options.signal);
+    if (options.stop?.aborted || entry.properties.isContainer) {
+      return undefined;
+    }
+    discovered += 1;
+    reportProgress();
+    const result = await transferEntry({
+      source,
+      sourceKey: joinKey(source, containerKey, entry.path),
+      sink,
+      destKey: joinKey(sink, destContainerKey, entry.path),
+      properties: entry.properties,
+      options,
+      parentOperationId: operationId,
+      type,
+      deleteSource: type === "move",
+    });
+    completed += 1;
+    reportProgress();
+    return result;
+  }
+
+  const results: TransferResult[] = [];
+  for await (const result of mapAsyncIterableConcurrently(
+    source.list(containerKey, { recursive: false, regex: globToRegex(pattern) }),
+    processEntry,
+    limiter,
+  )) {
+    if (result) results.push(result);
+  }
+  return aggregateResults(results, basePath(source, sink, options), options.stop?.aborted ?? false);
+}

package/src/transfer/rangeReadableMultipartReader.ts

@@ -0,0 +1,30 @@
+import type {
+  Part,
+  PayloadKind,
+  RangeReadable,
+  StreamHandle,
+} from "@flowscripter/pluggable-io-framework-api";
+
+/**
+ * Splits a `RangeReadable` handle into parts of `partSize` bytes covering
+ * `totalSize`. Each part's stream is opened with `readRange(start, end)`
+ * (`end` exclusive) when the part is pulled.
+ */
+export async function* rangeReadableMultipartReader<K extends PayloadKind>(
+  handle: StreamHandle<K> & RangeReadable<K>,
+  totalSize: number,
+  partSize: number,
+): AsyncGenerator<Part<K>> {
+  const partCount = Math.max(1, Math.ceil(totalSize / partSize));
+  for (let index = 0; index < partCount; index += 1) {
+    const start = index * partSize;
+    const end = Math.min(start + partSize, totalSize);
+    yield {
+      index,
+      offset: start,
+      kind: handle.kind,
+      stream: await handle.readRange(start, end),
+      complete: async () => {},
+    };
+  }
+}

package/src/transfer/recursiveTransfer.ts

@@ -0,0 +1,147 @@
+import {
+  type EntryProperties,
+  type IOProvider,
+  PermanentIOError,
+} from "@flowscripter/pluggable-io-framework-api";
+import { defaultConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.ts";
+import { mapAsyncIterableConcurrently } from "../concurrency/mapAsyncIterableConcurrently.ts";
+import { DEFAULT_RETRY } from "../retry/RetryOptions.ts";
+import { withRetry } from "../retry/withRetry.ts";
+import { throwIfAborted } from "../util/abort.ts";
+import { baseName, joinKey } from "../util/keys.ts";
+import { aggregateResults } from "./aggregateResults.ts";
+import { basePath, transferEntry } from "./transferEntry.ts";
+import type { TransferOptions } from "./TransferOptions.ts";
+import type { TransferResult } from "./TransferResult.ts";
+
+/**
+ * Resolves the effective destination root for a recursive transfer,
+ * following `cp -r`/`mv` semantics: a destination that doesn't exist becomes
+ * the copy itself; an existing container gets the source nested inside it
+ * as `dest/<source-basename>`; an existing entry is rejected.
+ */
+async function resolveRecursiveDestRoot(
+  sourceKey: string,
+  sink: IOProvider,
+  destKey: string,
+): Promise<string> {
+  let destProperties: EntryProperties | undefined;
+  try {
+    destProperties = await sink.getProperties(destKey);
+  } catch {
+    destProperties = undefined;
+  }
+  if (destProperties === undefined) {
+    return destKey;
+  }
+  if (!destProperties.isContainer) {
+    throw new PermanentIOError(
+      `Cannot copy/move container "${sourceKey}" onto existing entry "${destKey}"`,
+    );
+  }
+  return joinKey(sink, destKey, baseName(sourceKey));
+}
+
+/**
+ * Transfers a whole container. Uses a container-aware
+ * `directCopy`/`directMove` when the source declares
+ * `supportsRecursiveDirectTransfer`; otherwise lists the container and
+ * transfers each entry, bounded by the concurrency limiter. A non-direct
+ * move deletes the source container once, after every entry succeeded and
+ * only if the transfer was not stopped.
+ */
+export async function recursiveTransfer(
+  source: IOProvider,
+  sourceKey: string,
+  sink: IOProvider,
+  destKey: string,
+  options: TransferOptions,
+  type: "copy" | "move",
+): Promise<TransferResult> {
+  const path = basePath(source, sink, options);
+  const resolvedDestRoot = await resolveRecursiveDestRoot(sourceKey, sink, destKey);
+
+  if (
+    options.directTransfer !== false &&
+    source.canDirectTransfer?.(sink) &&
+    source.supportsRecursiveDirectTransfer
+  ) {
+    const direct = type === "move" ? source.directMove : source.directCopy;
+    if (direct) {
+      throwIfAborted(options.signal);
+      const telemetry = { operationId: crypto.randomUUID(), hooks: options.telemetry ?? {} };
+      await direct.call(source, sourceKey, resolvedDestRoot, telemetry);
+      return { stopped: false, bytes: 0, items: 0, path: `${path}, direct` };
+    }
+  }
+  if (typeof source.list !== "function") {
+    throw new PermanentIOError("The source provider does not support listing containers");
+  }
+  if (type === "move" && typeof source.delete !== "function") {
+    throw new PermanentIOError("The source provider does not support delete, so it cannot move");
+  }
+
+  const recursiveOperationId = crypto.randomUUID();
+  const limiter = options.concurrencyLimiter ?? defaultConcurrencyLimiter;
+  let discovered = 0;
+  let completed = 0;
+  const reportProgress = (): void => {
+    options.telemetry?.onProgress?.({
+      operationId: recursiveOperationId,
+      type,
+      bytesProcessed: 0,
+      entriesProcessed: completed,
+      totalEntries: discovered,
+    });
+  };
+
+  async function processEntry(entry: {
+    path: string;
+    properties: EntryProperties;
+  }): Promise<TransferResult | undefined> {
+    throwIfAborted(options.signal);
+    if (options.stop?.aborted) {
+      return undefined;
+    }
+    discovered += 1;
+    reportProgress();
+    const entryDestKey = joinKey(sink, resolvedDestRoot, entry.path);
+    let result: TransferResult | undefined;
+    if (entry.properties.isContainer) {
+      await sink.createContainer?.(entryDestKey);
+    } else {
+      result = await transferEntry({
+        source,
+        sourceKey: joinKey(source, sourceKey, entry.path),
+        sink,
+        destKey: entryDestKey,
+        properties: entry.properties,
+        options,
+        parentOperationId: recursiveOperationId,
+        type,
+        deleteSource: false,
+      });
+    }
+    completed += 1;
+    reportProgress();
+    return result;
+  }
+
+  const results: TransferResult[] = [];
+  for await (const result of mapAsyncIterableConcurrently(
+    source.list(sourceKey, { recursive: true }),
+    processEntry,
+    limiter,
+  )) {
+    if (result) results.push(result);
+  }
+
+  const outcome = aggregateResults(results, path, options.stop?.aborted ?? false);
+  if (type === "move" && !outcome.stopped) {
+    await withRetry(
+      () => (source.delete as (key: string) => Promise<void>).call(source, sourceKey),
+      options.retry ?? DEFAULT_RETRY,
+    );
+  }
+  return outcome;
+}

package/src/transfer/streamTransfer.ts

@@ -0,0 +1,214 @@
+import {
+  type IOProvider,
+  type Item,
+  isRangeReadable,
+  isResumableWritable,
+  type StreamHandle,
+  TransientIOError,
+} from "@flowscripter/pluggable-io-framework-api";
+import { backoff, DEFAULT_RETRY } from "../retry/RetryOptions.ts";
+import { createAbortError, watchSignals } from "../util/abort.ts";
+import { applyPayloadConverter } from "../util/applyPayloadConverter.ts";
+import { itemLength } from "../util/itemLength.ts";
+import type { EntryContext, EntryOutcome } from "./EntryContext.ts";
+
+/** Which side of the transfer a failure came from. */
+class SideError extends Error {
+  public constructor(
+    public readonly side: "source" | "sink",
+    cause: unknown,
+  ) {
+    super("transfer side failed", { cause });
+  }
+}
+
+export interface StreamTransferInput {
+  readonly source: IOProvider;
+  readonly sourceKey: string;
+  readonly readable: StreamHandle;
+  readonly sink: IOProvider;
+  readonly destKey: string;
+  readonly writable: StreamHandle;
+  /** Total bytes for a bounded source, when known. */
+  readonly totalBytes?: number;
+  readonly context: EntryContext;
+}
+
+function ignore(): void {}
+
+/**
+ * Pipes a readable handle into a writable handle item by item, honouring
+ * `signal` (abort) and `stop` (graceful close).
+ *
+ * On a `TransientIOError`:
+ * - bounded source with `RangeReadable`, and a writable exposing a resume
+ *   token: the sink is reopened with the token and the source re-read from
+ *   the sink's `startOffset`;
+ * - other bounded transfers restart from scratch;
+ * - unbounded source failure: the source is reopened, the sink kept open,
+ *   the first new item flagged with `discontinuity`, and `onGap` reported;
+ * - unbounded sink failure: the sink is reopened with its resume token, or
+ *   the transfer fails.
+ *
+ * `retry.maxRetries` counts consecutive failures and resets once an item is
+ * written.
+ */
+export async function streamTransfer(input: StreamTransferInput): Promise<EntryOutcome> {
+  const { source, sourceKey, sink, destKey, totalBytes, context } = input;
+  const { options, hooks, operationId, type } = context;
+  const retry = options.retry ?? DEFAULT_RETRY;
+  const bounded = input.readable.bounded !== false;
+  const watch = watchSignals(options.signal, options.stop);
+
+  let readable = input.readable;
+  let writable = input.writable;
+  const convert = (stream: ReadableStream<Item>) =>
+    applyPayloadConverter(stream, readable.kind, sink.kind, options.converter);
+  let reader = convert(readable.stream as ReadableStream<Item>).getReader();
+  let writer = (writable.stream as WritableStream<Item>).getWriter();
+  let bytes = 0;
+  let items = 0;
+  let failures = 0;
+  let markGap = false;
+  let recover: { side: "source" | "sink"; run: () => Promise<void> } | undefined;
+
+  async function pump(): Promise<"done" | "stopped"> {
+    for (;;) {
+      let read:
+        | Awaited<ReturnType<ReadableStreamDefaultReader<Item>["read"]>>
+        | "aborted"
+        | "stopped";
+      try {
+        read = await Promise.race([reader.read(), watch.fired]);
+      } catch (error) {
+        throw new SideError("source", error);
+      }
+      if (read === "aborted") throw createAbortError();
+      if (read === "stopped") return "stopped";
+      if (read.done) return "done";
+
+      let item = read.value;
+      if (markGap) {
+        item = { ...item, attributes: { ...item.attributes, discontinuity: true } };
+        markGap = false;
+      }
+      let written: void | "aborted" | "stopped";
+      try {
+        written = await Promise.race([writer.write(item), watch.fired]);
+      } catch (error) {
+        throw new SideError("sink", error);
+      }
+      if (written === "aborted") throw createAbortError();
+      bytes += itemLength(item);
+      items += 1;
+      failures = 0;
+      hooks.onProgress?.({
+        operationId,
+        type,
+        bytesProcessed: bytes,
+        totalBytes: bounded ? totalBytes : undefined,
+      });
+      if (written === "stopped") return "stopped";
+    }
+  }
+
+  try {
+    for (;;) {
+      try {
+        if (recover) {
+          const action = recover;
+          recover = undefined;
+          try {
+            await action.run();
+          } catch (error) {
+            throw new SideError(action.side, error);
+          }
+        }
+        const outcome = await pump();
+        if (outcome === "stopped") {
+          await reader.cancel().catch(ignore);
+        }
+        await writer.close();
+        return { bytes, items, stopped: outcome === "stopped" };
+      } catch (error) {
+        if (error instanceof DOMException && error.name === "AbortError") {
+          await reader.cancel(error).catch(ignore);
+          await writer.abort(error).catch(ignore);
+          throw error;
+        }
+        const side = error instanceof SideError ? error.side : "sink";
+        const cause = error instanceof SideError ? error.cause : error;
+        failures += 1;
+        const retryable = cause instanceof TransientIOError && failures <= retry.maxRetries;
+
+        if (retryable && !bounded && side === "source" && retry.onGap !== "fail") {
+          await reader.cancel().catch(ignore);
+          const attempt = failures;
+          recover = {
+            side: "source",
+            run: async () => {
+              await backoff(retry, attempt);
+              readable = await source.getReadableStream(sourceKey);
+              reader = convert(readable.stream as ReadableStream<Item>).getReader();
+              markGap = true;
+              hooks.onGap?.(operationId, { atBytes: bytes, attempt });
+            },
+          };
+          continue;
+        }
+
+        const token =
+          retryable && isResumableWritable(writable) ? writable.resumeToken() : undefined;
+        const canResume = token !== undefined && (!bounded || isRangeReadable(readable));
+        if (retryable && canResume) {
+          // The failed sink is released, not aborted, so a provider that
+          // discards partial output on abort keeps it for the resume.
+          writer.releaseLock();
+          if (bounded) {
+            await reader.cancel().catch(ignore);
+          }
+          const attempt = failures;
+          recover = {
+            side: "sink",
+            run: async () => {
+              await backoff(retry, attempt);
+              const resumed = await sink.getWritableStream(destKey, { resume: token });
+              writable = resumed;
+              writer = (resumed.stream as WritableStream<Item>).getWriter();
+              if (bounded && isRangeReadable(readable)) {
+                const startOffset = resumed.startOffset ?? token.offset;
+                const end = totalBytes ?? Number.MAX_SAFE_INTEGER;
+                reader = convert(
+                  (await readable.readRange(startOffset, end)) as ReadableStream<Item>,
+                ).getReader();
+                bytes = startOffset;
+              }
+            },
+          };
+          continue;
+        }
+
+        await reader.cancel(cause).catch(ignore);
+        await writer.abort(cause).catch(ignore);
+        if (!retryable || !bounded) {
+          throw cause;
+        }
+        const attempt = failures;
+        recover = {
+          side: "source",
+          run: async () => {
+            await backoff(retry, attempt);
+            readable = await source.getReadableStream(sourceKey);
+            reader = convert(readable.stream as ReadableStream<Item>).getReader();
+            writable = await sink.getWritableStream(destKey);
+            writer = (writable.stream as WritableStream<Item>).getWriter();
+            bytes = 0;
+            items = 0;
+          },
+        };
+      }
+    }
+  } finally {
+    watch.dispose();
+  }
+}

package/src/transfer/transferEntry.ts

@@ -0,0 +1,185 @@
+import {
+  BYTES_PAYLOAD_TYPE,
+  type EntryProperties,
+  type IOProvider,
+  isBufferProvider,
+  isFillReadable,
+  isRangeReadable,
+  type Item,
+  PermanentIOError,
+  type StreamHandle,
+} from "@flowscripter/pluggable-io-framework-api";
+import { defaultConcurrencyLimiter } from "../concurrency/ConcurrencyLimiter.ts";
+import { DEFAULT_RETRY } from "../retry/RetryOptions.ts";
+import { withRetry } from "../retry/withRetry.ts";
+import { throwIfAborted } from "../util/abort.ts";
+import { describeConverter } from "../util/describePath.ts";
+import { withParentOperationId } from "../util/withParentOperationId.ts";
+import type { EntryContext, EntryOutcome } from "./EntryContext.ts";
+import { leaseDepth, leaseTransfer } from "./leaseTransfer.ts";
+import { multipartTransfer } from "./multipartTransfer.ts";
+import { negotiatePartSize } from "./negotiatePartSize.ts";
+import { streamTransfer } from "./streamTransfer.ts";
+import type { TransferOptions } from "./TransferOptions.ts";
+import type { TransferResult } from "./TransferResult.ts";
+
+const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
+
+export interface TransferEntryInput {
+  readonly source: IOProvider;
+  readonly sourceKey: string;
+  readonly sink: IOProvider;
+  readonly destKey: string;
+  readonly properties: EntryProperties;
+  readonly options: TransferOptions;
+  readonly parentOperationId?: string;
+  /** `"move"` rejects unbounded sources and prefers `directMove`. */
+  readonly type: "copy" | "move";
+  /** Whether a non-direct move deletes the source entry afterwards. */
+  readonly deleteSource: boolean;
+}
+
+/** The path description before the transfer strategy is known. */
+export function basePath(source: IOProvider, sink: IOProvider, options: TransferOptions): string {
+  if (options.path !== undefined) {
+    return options.path;
+  }
+  const via = options.converter ? describeConverter(options.converter) : "";
+  return `${source.kind} -> ${sink.kind}${via}`;
+}
+
+function checkPayloadType(readable: StreamHandle, options: TransferOptions): void {
+  const payloadType = readable.payloadType ?? BYTES_PAYLOAD_TYPE;
+  const accepted = options.writePayloadTypes ?? [BYTES_PAYLOAD_TYPE];
+  if (!accepted.includes(payloadType)) {
+    throw new PermanentIOError(
+      `Source payload type "${payloadType}" is not accepted by the sink (sink accepts: ${accepted.join(", ")})`,
+    );
+  }
+}
+
+async function cancelStream(handle: StreamHandle): Promise<void> {
+  await (handle.stream as ReadableStream<Item>).cancel().catch(() => {});
+}
+
+/**
+ * Transfers one entry: a direct provider transfer when eligible, otherwise
+ * multipart, the lease path or plain streaming, after checking the payload
+ * type and (for a move) that the source is bounded.
+ */
+export async function transferEntry(input: TransferEntryInput): Promise<TransferResult> {
+  const { source, sourceKey, sink, destKey, properties, options, type } = input;
+  throwIfAborted(options.signal);
+  const path = basePath(source, sink, options);
+  if (options.stop?.aborted) {
+    return { stopped: true, bytes: 0, items: 0, path };
+  }
+  const operationId = crypto.randomUUID();
+  const hooks = withParentOperationId(options.telemetry, input.parentOperationId);
+  const context: EntryContext = { operationId, hooks, options, type };
+
+  if (options.directTransfer !== false && source.canDirectTransfer?.(sink)) {
+    const direct = type === "move" ? source.directMove : source.directCopy;
+    if (direct) {
+      await direct.call(source, sourceKey, destKey, { operationId, hooks });
+      return { stopped: false, bytes: properties.size ?? 0, items: 0, path: `${path}, direct` };
+    }
+  }
+  if (input.deleteSource && typeof source.delete !== "function") {
+    throw new PermanentIOError("The source provider does not support delete, so it cannot move");
+  }
+
+  const retry = options.retry ?? DEFAULT_RETRY;
+  let readable = await withRetry(() => source.getReadableStream(sourceKey), retry);
+  try {
+    checkPayloadType(readable, options);
+    if (type === "move" && readable.bounded === false) {
+      throw new PermanentIOError("cannot move a live source");
+    }
+  } catch (error) {
+    await cancelStream(readable);
+    throw error;
+  }
+
+  let outcome: EntryOutcome;
+  let strategy: string;
+  const size = properties.size;
+  const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
+  const multipartWriter =
+    readable.bounded !== false &&
+    size !== undefined &&
+    size >= threshold &&
+    isRangeReadable(readable) &&
+    typeof sink.getMultipartWriter === "function"
+      ? sink.getMultipartWriter.bind(sink)
+      : undefined;
+  const partSize =
+    multipartWriter && size !== undefined ? negotiatePartSize(source, sink, size) : undefined;
+
+  if (
+    multipartWriter &&
+    partSize !== undefined &&
+    size !== undefined &&
+    isRangeReadable(readable)
+  ) {
+    await cancelStream(readable);
+    const rangeReadable = readable;
+    outcome = await withRetry(
+      () =>
+        multipartTransfer({
+          readable: rangeReadable,
+          sink,
+          writer: multipartWriter(destKey, partSize),
+          totalBytes: size,
+          partSize,
+          limiter: options.concurrencyLimiter ?? defaultConcurrencyLimiter,
+          context,
+        }),
+      retry,
+    );
+    strategy = "multipart";
+  } else {
+    let writable = await withRetry(() => sink.getWritableStream(destKey), retry);
+    if (
+      !options.converter &&
+      isFillReadable(readable) &&
+      isBufferProvider(writable) &&
+      readable.domains.includes(writable.domain)
+    ) {
+      const depth = leaseDepth(writable, options.leaseDepth);
+      let attempt = 0;
+      outcome = await withRetry(async () => {
+        if (attempt > 0) {
+          readable = await source.getReadableStream(sourceKey);
+          writable = await sink.getWritableStream(destKey);
+        }
+        attempt += 1;
+        if (!isFillReadable(readable) || !isBufferProvider(writable)) {
+          throw new PermanentIOError("Reopened handles no longer support the lease path");
+        }
+        return leaseTransfer(readable, writable, depth, context);
+      }, retry);
+      strategy = `lease (depth ${depth})`;
+    } else {
+      outcome = await streamTransfer({
+        source,
+        sourceKey,
+        readable,
+        sink,
+        destKey,
+        writable,
+        totalBytes: size,
+        context,
+      });
+      strategy = "stream";
+    }
+  }
+
+  if (input.deleteSource && !outcome.stopped) {
+    await withRetry(
+      () => (source.delete as (key: string) => Promise<void>).call(source, sourceKey),
+      retry,
+    );
+  }
+  return { ...outcome, path: `${path}, ${strategy}` };
+}