@flowscripter/pluggable-io-framework 1.2.7 → 2.0.0

package/src/concurrency/AsyncChannel.ts

@@ -0,0 +1,52 @@
+/** Minimal single-consumer async FIFO queue. */
+export class AsyncChannel<T> implements AsyncIterable<T> {
+  #items: T[] = [];
+  #waiters: { resolve: (result: IteratorResult<T>) => void; reject: (error: unknown) => void }[] =
+    [];
+  #closed = false;
+  #hasError = false;
+  #error: unknown;
+
+  push(item: T): void {
+    const waiter = this.#waiters.shift();
+    if (waiter) {
+      waiter.resolve({ value: item, done: false });
+    } else {
+      this.#items.push(item);
+    }
+  }
+
+  close(): void {
+    this.#closed = true;
+    for (const waiter of this.#waiters.splice(0)) {
+      if (this.#hasError) {
+        waiter.reject(this.#error);
+      } else {
+        waiter.resolve({ value: undefined as never, done: true });
+      }
+    }
+  }
+
+  fail(error: unknown): void {
+    this.#hasError = true;
+    this.#error = error;
+    this.close();
+  }
+
+  async #next(): Promise<IteratorResult<T>> {
+    if (this.#items.length > 0) {
+      return { value: this.#items.shift() as T, done: false };
+    }
+    if (this.#closed) {
+      if (this.#hasError) {
+        throw this.#error;
+      }
+      return { value: undefined as never, done: true };
+    }
+    return new Promise((resolve, reject) => this.#waiters.push({ resolve, reject }));
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    return { next: () => this.#next() };
+  }
+}

package/src/concurrency/ConcurrencyLimiter.ts

@@ -0,0 +1,72 @@
+const DEFAULT_MAX_CONCURRENCY = 4;
+
+/**
+ * A semaphore bounding how many `run()` callbacks execute at once. A single
+ * shared instance (see {@link defaultConcurrencyLimiter}) enforces its cap
+ * *globally* across concurrently invoked `copy()`/`move()` calls, not just
+ * within one call - two folder copies started at the same time and sharing
+ * a limiter will never together exceed its `maxConcurrency`.
+ */
+export class ConcurrencyLimiter {
+  #maxConcurrency: number;
+  #active = 0;
+  #queue: (() => void)[] = [];
+
+  public constructor(maxConcurrency: number) {
+    ConcurrencyLimiter.#validate(maxConcurrency);
+    this.#maxConcurrency = maxConcurrency;
+  }
+
+  public get maxConcurrency(): number {
+    return this.#maxConcurrency;
+  }
+
+  public setMaxConcurrency(maxConcurrency: number): void {
+    ConcurrencyLimiter.#validate(maxConcurrency);
+    this.#maxConcurrency = maxConcurrency;
+    this.#drain();
+  }
+
+  public async run<T>(fn: () => Promise<T>): Promise<T> {
+    await this.#acquire();
+    try {
+      return await fn();
+    } finally {
+      this.#active -= 1;
+      this.#drain();
+    }
+  }
+
+  #acquire(): Promise<void> {
+    if (this.#active < this.#maxConcurrency) {
+      this.#active += 1;
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      this.#queue.push(() => {
+        this.#active += 1;
+        resolve();
+      });
+    });
+  }
+
+  #drain(): void {
+    while (this.#active < this.#maxConcurrency && this.#queue.length > 0) {
+      this.#queue.shift()?.();
+    }
+  }
+
+  static #validate(maxConcurrency: number): void {
+    if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
+      throw new Error(`maxConcurrency must be a positive integer, got ${maxConcurrency}`);
+    }
+  }
+}
+
+/**
+ * Process-wide default limiter, used by `copy()`/`move()` whenever
+ * `TransferOptions.concurrencyLimiter` is omitted - so any two calls in the
+ * same process share one cap unless a caller explicitly opts out with its
+ * own instance.
+ */
+export const defaultConcurrencyLimiter = new ConcurrencyLimiter(DEFAULT_MAX_CONCURRENCY);

package/src/concurrency/mapAsyncIterableConcurrently.ts

@@ -0,0 +1,66 @@
+import { AsyncChannel } from "./AsyncChannel.ts";
+import type { ConcurrencyLimiter } from "./ConcurrencyLimiter.ts";
+
+/**
+ * Pulls items from `source` and runs `fn` on each, bounded by `limiter`:
+ * exactly `limiter.maxConcurrency` worker loops each pull-then-process one
+ * item at a time through `limiter.run`, so at most that many items are ever
+ * pulled/in-flight at once (important for sources like a multipart reader
+ * where pulling an item eagerly opens a file/network handle). Results are
+ * yielded in completion order, not source order. If `fn` throws, no new
+ * items are pulled but already-in-flight ones are allowed to finish before
+ * the error is rethrown from the returned iterable.
+ */
+export function mapAsyncIterableConcurrently<T, R>(
+  source: AsyncIterable<T>,
+  fn: (item: T) => Promise<R>,
+  limiter: ConcurrencyLimiter,
+): AsyncIterable<R> {
+  const channel = new AsyncChannel<R>();
+  const iterator = source[Symbol.asyncIterator]();
+  let stopped = false;
+  let firstError: unknown;
+  let errorRecorded = false;
+
+  function recordError(error: unknown): void {
+    stopped = true;
+    if (!errorRecorded) {
+      errorRecorded = true;
+      firstError = error;
+    }
+  }
+
+  async function worker(): Promise<void> {
+    for (;;) {
+      if (stopped) return;
+      let next: IteratorResult<T>;
+      try {
+        next = await iterator.next();
+      } catch (error) {
+        recordError(error);
+        return;
+      }
+      if (next.done) return;
+      const value = next.value;
+      try {
+        const result = await limiter.run(() => fn(value));
+        channel.push(result);
+      } catch (error) {
+        recordError(error);
+        return;
+      }
+    }
+  }
+
+  void (async () => {
+    const workerCount = Math.max(1, limiter.maxConcurrency);
+    await Promise.all(Array.from({ length: workerCount }, () => worker()));
+    if (errorRecorded) {
+      channel.fail(firstError);
+    } else {
+      channel.close();
+    }
+  })();
+
+  return channel;
+}

package/src/decorators/locallyCached.ts

@@ -1,53 +1,61 @@
 import type {
-  ChunkKind,
-  ChunkOfKind,
+  ItemOfKind,
+  PayloadKind,
   StreamHandle,
 } from "@flowscripter/pluggable-io-framework-api";
 
 /**
- * Wraps a `StreamHandle` factory (e.g. `() => provider.getReadableStream(path)`)
- * so the underlying source is read at most once - the first call drains the
- * source while caching every chunk in memory; every subsequent call replays
- * the cached chunks without touching the source again.
+ * A `StreamOpenerDecorator`: wraps a `StreamHandle` opener so the
+ * underlying source is read at most once - the first call drains the
+ * source while caching every item in memory; every subsequent call replays
+ * the cached items without touching the source again.
  *
  * A plain `StreamHandle` only exposes a single one-shot `ReadableStream`, so
- * caching can't be a `StreamDecorator<K, C>` operating on an already-open
- * handle (there would be nothing left to re-read on a second call) - it has
- * to intercept the *open* operation itself, provider-agnostic regardless of
- * what kind of source is behind it.
+ * caching can't be a `StreamDecorator` operating on an already-open handle
+ * (there would be nothing left to re-read on a second call) - it has to
+ * intercept the *open* operation itself.
  */
-export function locallyCached<K extends ChunkKind>(
+export function locallyCached<K extends PayloadKind>(
   open: () => Promise<StreamHandle<K>>,
 ): () => Promise<StreamHandle<K>> {
-  let cache: { kind: K; chunks: ChunkOfKind<K>[] } | undefined;
+  let cache: { handle: StreamHandle<K>; items: ItemOfKind<K>[] } | undefined;
 
-  function replay(chunks: ChunkOfKind<K>[]): ReadableStream<ChunkOfKind<K>> {
-    return new ReadableStream<ChunkOfKind<K>>({
+  function replay(items: ItemOfKind<K>[]): ReadableStream<ItemOfKind<K>> {
+    return new ReadableStream<ItemOfKind<K>>({
       start(controller) {
-        for (const chunk of chunks) controller.enqueue(chunk);
+        for (const item of items) controller.enqueue(item);
         controller.close();
       },
     });
   }
 
+  function describe(handle: StreamHandle<K>, stream: ReadableStream<ItemOfKind<K>>) {
+    return {
+      kind: handle.kind,
+      stream,
+      bounded: handle.bounded,
+      payloadType: handle.payloadType,
+    };
+  }
+
   return async () => {
     if (cache) {
-      return { kind: cache.kind, stream: replay(cache.chunks) };
+      return describe(cache.handle, replay(cache.items));
     }
 
     const handle = await open();
-    const reader = (handle.stream as ReadableStream<ChunkOfKind<K>>).getReader();
-    const chunks: ChunkOfKind<K>[] = [];
+    const reader = (handle.stream as ReadableStream<ItemOfKind<K>>).getReader();
+    const items: ItemOfKind<K>[] = [];
 
-    const stream = new ReadableStream<ChunkOfKind<K>>({
+    const stream = new ReadableStream<ItemOfKind<K>>({
       async pull(controller) {
         const { done, value } = await reader.read();
         if (done) {
-          cache = { kind: handle.kind, chunks };
+          cache = { handle, items };
           controller.close();
           return;
         }
-        chunks.push(value);
+        items.push(value);
         controller.enqueue(value);
       },
       cancel(reason) {
@@ -55,6 +63,6 @@ export function locallyCached<K extends ChunkKind>(
       },
     });
 
-    return { kind: handle.kind, stream };
+    return describe(handle, stream);
   };
 }

package/src/decorators/seekable.ts

@@ -1,14 +1,14 @@
 import type {
-  ChunkKind,
-  ChunkOfKind,
+  PayloadKind,
+  ItemOfKind,
   RangeReadable,
   Seekable,
   StreamHandle,
 } from "@flowscripter/pluggable-io-framework-api";
 
 /**
- * Wraps a handle that already supports {@link RangeReadable} (arbitrary
- * byte-range reads) with a {@link Seekable} capability: a single logical
+ * Wraps a handle that already supports `RangeReadable` (arbitrary
+ * byte-range reads) with a `Seekable` capability: a single logical
  * stream whose read position can be jumped via `seek(offset)`, rather than
  * requiring the caller to open a fresh stream per range.
  *
@@ -16,12 +16,12 @@ import type {
  * in flight - like any seekable stream, seeking and reading are sequential,
  * not concurrent, operations on the same handle.
  */
-export function seekable<K extends ChunkKind>(
+export function seekable<K extends PayloadKind>(
   handle: StreamHandle<K> & RangeReadable<K>,
 ): StreamHandle<K> & Seekable {
-  let reader = (handle.stream as ReadableStream<ChunkOfKind<K>>).getReader();
+  let reader = (handle.stream as ReadableStream<ItemOfKind<K>>).getReader();
 
-  const stream = new ReadableStream<ChunkOfKind<K>>(
+  const stream = new ReadableStream<ItemOfKind<K>>(
     {
       async pull(controller) {
         const { done, value } = await reader.read();

package/src/registry/ProviderRegistry.ts

@@ -0,0 +1,241 @@
+import type { PluginManager } from "@flowscripter/dynamic-plugin-framework";
+import {
+  BYTES_PAYLOAD_TYPE,
+  type IOProvider,
+  type IOProviderFactory,
+  type LocationTarget,
+  PayloadKind,
+  type PayloadConverter,
+  PermanentIOError,
+  PLUGGABLE_IO_FRAMEWORK_PAYLOAD_CONVERTER_EXTENSION_POINT,
+  PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
+  type ProviderResolver,
+} from "@flowscripter/pluggable-io-framework-api";
+import type { TransferOptions } from "../transfer/TransferOptions.ts";
+import { detectProtocol } from "./detectProtocol.ts";
+import { DEFAULT_NATIVE_DOMAINS, negotiateTransfer } from "./negotiateTransfer.ts";
+
+/** Nested provider resolutions allowed before resolution fails. */
+export const MAX_RESOLUTION_DEPTH = 4;
+
+export interface ResolvedProvider {
+  readonly provider: IOProvider;
+  readonly target: LocationTarget;
+}
+
+export interface TransferProviders {
+  readonly source: ResolvedProvider;
+  readonly dest: ResolvedProvider;
+  /**
+   * The negotiated `converter`, `path` and `writePayloadTypes`, to pass to
+   * `copy`/`move` as is or spread together with caller options.
+   */
+  readonly options: TransferOptions;
+}
+
+function locationFieldNames(factory: IOProviderFactory): string[] {
+  const shape = (factory.locationSchema as { shape?: Record<string, unknown> }).shape;
+  return Object.keys(shape ?? {}).sort();
+}
+
+/**
+ * Discovers provider factories and payload converters through a
+ * `dynamic-plugin-framework` `PluginManager`, keyed by protocol and
+ * payload kind, and creates providers for location strings. It is also the
+ * `ProviderResolver` handed to every provider it creates.
+ */
+export class ProviderRegistry implements ProviderResolver {
+  #factories = new Map<string, Map<PayloadKind, IOProviderFactory>>();
+  #converters: PayloadConverter[] = [];
+
+  public constructor(private readonly pluginManager: PluginManager) {}
+
+  /**
+   * Instantiates every registered factory and converter once. Fails, naming
+   * both factories, if two share a (protocol, kind) pair or if factories for
+   * the same protocol have different top-level location fields.
+   */
+  public async discover(): Promise<void> {
+    const factories = new Map<string, Map<PayloadKind, IOProviderFactory>>();
+    const handles = new Map<IOProviderFactory, string>();
+    for (const extension of await this.#extensions(
+      PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
+    )) {
+      const factory = (await this.pluginManager.instantiate(
+        extension.extensionHandle,
+      )) as IOProviderFactory;
+      handles.set(factory, extension.extensionHandle);
+      const byKind = factories.get(factory.protocol) ?? new Map<PayloadKind, IOProviderFactory>();
+      const clash = byKind.get(factory.kind);
+      if (clash) {
+        throw new Error(
+          `Factories "${handles.get(clash)}" and "${extension.extensionHandle}" both provide protocol "${factory.protocol}" with payload kind "${factory.kind}"`,
+        );
+      }
+      for (const other of byKind.values()) {
+        if (locationFieldNames(other).join() !== locationFieldNames(factory).join()) {
+          throw new Error(
+            `Factories "${handles.get(other)}" and "${extension.extensionHandle}" for protocol "${factory.protocol}" have different location fields`,
+          );
+        }
+      }
+      byKind.set(factory.kind, factory);
+      factories.set(factory.protocol, byKind);
+    }
+    const converters: PayloadConverter[] = [];
+    for (const extension of await this.#extensions(
+      PLUGGABLE_IO_FRAMEWORK_PAYLOAD_CONVERTER_EXTENSION_POINT,
+    )) {
+      converters.push(
+        (await this.pluginManager.instantiate(extension.extensionHandle)) as PayloadConverter,
+      );
+    }
+    this.#factories = factories;
+    this.#converters = converters;
+  }
+
+  public getProtocols(): string[] {
+    return [...this.#factories.keys()].sort();
+  }
+
+  public getKinds(protocol: string): PayloadKind[] {
+    return [...(this.#factories.get(protocol)?.keys() ?? [])].sort();
+  }
+
+  /** The factory for `protocol` and `kind`; without `kind`, the `js` one, else the only one. */
+  public getFactory(protocol: string, kind?: PayloadKind): IOProviderFactory | undefined {
+    const byKind = this.#factories.get(protocol);
+    if (!byKind) return undefined;
+    if (kind !== undefined) return byKind.get(kind);
+    return byKind.get(PayloadKind.Js) ?? byKind.values().next().value;
+  }
+
+  public getConverters(): readonly PayloadConverter[] {
+    return this.#converters;
+  }
+
+  public createProviderForLocation(
+    location: string,
+    options?: { kind?: PayloadKind; domain?: string },
+  ): Promise<ResolvedProvider> {
+    return this.#resolve(location, options, 0);
+  }
+
+  /**
+   * Creates providers for both sides of a transfer, negotiating a common
+   * (kind, domain) and payload type, or a converter.
+   */
+  public async createProvidersForTransfer(
+    source: string,
+    dest: string,
+    options?: { kind?: PayloadKind },
+  ): Promise<TransferProviders> {
+    const sourceProtocol = detectProtocol(source);
+    const destProtocol = detectProtocol(dest);
+    const sourceFactories = this.#factoriesFor(sourceProtocol, options?.kind);
+    const destFactories = this.#factoriesFor(destProtocol, options?.kind);
+    const negotiation = negotiateTransfer(
+      sourceProtocol,
+      sourceFactories,
+      destProtocol,
+      destFactories,
+      this.#converters,
+      options?.kind,
+    );
+    const sourceResolved = await this.#instantiate(
+      negotiation.source.factory,
+      source,
+      negotiation.source.domain,
+      0,
+    );
+    let destResolved: ResolvedProvider;
+    try {
+      destResolved = await this.#instantiate(
+        negotiation.dest.factory,
+        dest,
+        negotiation.dest.domain,
+        0,
+      );
+    } catch (error) {
+      await sourceResolved.provider[Symbol.asyncDispose]();
+      throw error;
+    }
+    return {
+      source: sourceResolved,
+      dest: destResolved,
+      options: {
+        converter: negotiation.converter,
+        path: negotiation.path,
+        writePayloadTypes: negotiation.dest.factory.writePayloadTypes ?? [BYTES_PAYLOAD_TYPE],
+      },
+    };
+  }
+
+  #extensions(extensionPoint: string) {
+    return this.pluginManager
+      .registerExtensions(extensionPoint)
+      .then(() => this.pluginManager.getRegisteredExtensions(extensionPoint));
+  }
+
+  #factoriesFor(protocol: string, kind?: PayloadKind): IOProviderFactory[] {
+    const byKind = this.#factories.get(protocol);
+    if (!byKind) {
+      throw new Error(
+        `No provider for protocol "${protocol}" (available: ${this.getProtocols().join(", ")})`,
+      );
+    }
+    if (kind !== undefined && !byKind.has(kind)) {
+      throw new Error(
+        `No provider for protocol "${protocol}" with payload kind "${kind}" (available: ${this.getKinds(protocol).join(", ")})`,
+      );
+    }
+    return [...byKind.values()];
+  }
+
+  async #resolve(
+    location: string,
+    options: { kind?: PayloadKind; domain?: string } | undefined,
+    depth: number,
+  ): Promise<ResolvedProvider> {
+    if (depth > MAX_RESOLUTION_DEPTH) {
+      throw new PermanentIOError("provider resolution too deep");
+    }
+    const protocol = detectProtocol(location);
+    this.#factoriesFor(protocol, options?.kind);
+    const factory = this.getFactory(protocol, options?.kind) as IOProviderFactory;
+    let domain: string | undefined;
+    if (factory.kind === PayloadKind.Native) {
+      const domains = factory.domains ?? DEFAULT_NATIVE_DOMAINS;
+      domain = options?.domain ?? domains[0];
+      if (domain === undefined || !domains.includes(domain)) {
+        throw new Error(
+          `No provider for protocol "${protocol}" supports domain "${domain}" (available: ${domains.join(", ")})`,
+        );
+      }
+    }
+    return this.#instantiate(factory, location, domain, depth);
+  }
+
+  async #instantiate(
+    factory: IOProviderFactory,
+    location: string,
+    domain: string | undefined,
+    depth: number,
+  ): Promise<ResolvedProvider> {
+    const parsed = factory.locationSchema.parse(factory.parseLocationString(location));
+    const { config, target } = factory.toProviderInputs(parsed);
+    const resolver: ProviderResolver = {
+      createProviderForLocation: (inner, opts) =>
+        this.#resolve(
+          inner,
+          { kind: opts?.kind ?? factory.kind, domain: opts?.domain ?? domain },
+          depth + 1,
+        ),
+    };
+    const provider = await factory.createProvider(factory.configSchema.parse(config), {
+      domain,
+      resolver,
+    });
+    return { provider, target };
+  }
+}

package/src/registry/detectProtocol.ts

@@ -0,0 +1,16 @@
+const SCHEME = /^([a-z][a-z0-9+.-]+):/i;
+
+/**
+ * Returns the protocol of a location string: its URI scheme, lower-cased,
+ * or `"file"` when there is none. A scheme must be at least two characters,
+ * so a Windows drive letter (`C:\foo`) is not read as one. For a composite
+ * scheme the protocol is the part before the first `+`.
+ */
+export function detectProtocol(input: string): string {
+  const match = SCHEME.exec(input);
+  if (!match) {
+    return "file";
+  }
+  const scheme = (match[1] as string).toLowerCase();
+  return scheme.split("+")[0] as string;
+}