@flowscripter/pluggable-io-framework 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Flowscripter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,129 @@
+# pluggable-io-framework
+
+[![version](https://img.shields.io/github/v/release/flowscripter/pluggable-io-framework?sort=semver)](https://github.com/flowscripter/pluggable-io-framework/releases)
+[![build](https://img.shields.io/github/actions/workflow/status/flowscripter/pluggable-io-framework/release-bun-library.yml)](https://github.com/flowscripter/pluggable-io-framework/actions/workflows/release-bun-library.yml)
+[![docs](https://img.shields.io/badge/docs-API-blue)](https://flowscripter.github.io/pluggable-io-framework/index.html)
+[![license: MIT](https://img.shields.io/github/license/flowscripter/pluggable-io-framework)](https://github.com/flowscripter/pluggable-io-framework/blob/main/LICENSE)
+
+> A pluggable source/sink IO framework - provider discovery, copy/move
+> orchestration and telemetry, for files and folders across different
+> backends (local filesystem, object storage, HTTP, etc.)
+
+## Key Features
+
+- Discovers and instantiates source/sink provider plugins (implementing the
+  `IOProviderFactory`/`IOProvider` contract from
+  [pluggable-io-framework-api](https://github.com/flowscripter/pluggable-io-framework-api))
+  via
+  [dynamic-plugin-framework](https://github.com/flowscripter/dynamic-plugin-framework),
+  including config validation against each plugin's Zod schema.
+- Copy/move orchestration:
+  - Uses a provider's `directCopy`/`directMove` when
+    `canDirectTransfer` reports the source and sink are the same
+    underlying provider (e.g. same filesystem mount, same object storage
+    bucket).
+  - Otherwise transfers via multipart (when both sides support it and file
+    size crosses a configurable threshold) or plain streaming.
+  - Reports progress via a global `TelemetryHooks` callback, tagged with a
+    per-operation correlation id.
+- Standalone today - usable directly or via the
+  [flowscripter-io-cli](https://github.com/flowscripter/flowscripter-io-cli)
+  built on
+  [dynamic-cli-framework](https://github.com/flowscripter/dynamic-cli-framework).
+  When the full Flowscripter graph runtime exists, an `adapt` operator will
+  wrap these providers to bridge them into the processing graph - that
+  wrapping is out of scope for this repo.
+- See
+  [pluggable-io-framework-plugin-filesystem](https://github.com/flowscripter/pluggable-io-framework-plugin-filesystem)
+  for a reference local filesystem source/sink plugin.
+
+## Bun Module Usage
+
+Add the module:
+
+`bun add @flowscripter/pluggable-io-framework`
+
+Discover and use a provider:
+
+```typescript
+import {
+  DefaultPluginManager,
+  LocalFolderPluginRepository,
+} from "@flowscripter/dynamic-plugin-framework";
+import { ProviderRegistry, copy } from "@flowscripter/pluggable-io-framework";
+
+const pluginManager = new DefaultPluginManager([new LocalFolderPluginRepository("./plugins")]);
+const registry = new ProviderRegistry(pluginManager);
+await registry.discover();
+
+const [extension] = await registry.listAvailableProviders();
+const provider = await registry.createProvider(extension.extensionHandle, { rootPath: "/data" });
+
+await copy(provider, "a.txt", provider, "b.txt", {
+  telemetry: { onProgress: (event) => console.log(event) },
+});
+```
+
+## Development
+
+Install dependencies:
+
+`bun install`
+
+Build (produces `dist/` for Node.js and TypeScript consumers; Bun uses raw source directly):
+
+`bun run build`
+
+Test:
+
+`bun test`
+
+Format:
+
+`bunx oxfmt`
+
+Lint:
+
+`bunx oxlint index.ts src/ tests/`
+
+Generate HTML API Documentation:
+
+`bunx typedoc index.ts`
+
+## Documentation
+
+### Overview
+
+```mermaid
+sequenceDiagram
+    participant Host
+    participant ProviderRegistry
+    participant PluginManager
+    participant Source as IOProvider (source)
+    participant Sink as IOProvider (sink)
+
+    Host->>ProviderRegistry: discover()
+    ProviderRegistry->>PluginManager: registerExtensions(extensionPoint)
+    Host->>ProviderRegistry: createProvider(handle, config)
+    ProviderRegistry->>PluginManager: instantiate(handle)
+    ProviderRegistry-->>Host: IOProvider
+
+    Host->>Source: copy(source, path, sink, path)
+    alt canDirectTransfer
+        Source->>Sink: directCopy(path, path)
+    else multipart eligible
+        Source-->>Sink: transfer Parts concurrently
+    else
+        Source-->>Sink: stream ChunkRefs
+    end
+```
+
+### API
+
+Link to auto-generated API docs:
+
+[API Documentation](https://flowscripter.github.io/pluggable-io-framework/index.html)
+
+## License
+
+MIT © Flowscripter

package/dist/src/ProviderRegistry.d.ts

@@ -0,0 +1,14 @@
+import type { PluginManager } from "@flowscripter/dynamic-plugin-framework";
+import { type IOProvider } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Wraps a `dynamic-plugin-framework` {@link PluginManager} to discover and
+ * instantiate pluggable-io-framework source/sink provider plugins.
+ */
+export declare class ProviderRegistry {
+    private readonly pluginManager;
+    constructor(pluginManager: PluginManager);
+    discover(): Promise<void>;
+    listAvailableProviders(): Promise<readonly import("@flowscripter/dynamic-plugin-framework").ExtensionInfo[]>;
+    createProvider(extensionHandle: string, config: unknown): Promise<IOProvider>;
+}
+//# sourceMappingURL=ProviderRegistry.d.ts.map

package/dist/src/ProviderRegistry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ProviderRegistry.d.ts","sourceRoot":"","sources":["../../src/ProviderRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAEL,KAAK,UAAU,EAEhB,MAAM,0CAA0C,CAAC;AAElD;;;GAGG;AACH,qBAAa,gBAAgB;IACR,OAAO,CAAC,QAAQ,CAAC,aAAa;gBAAb,aAAa,EAAE,aAAa;IAEnD,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAIzB,sBAAsB;IAItB,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC;CAK3F"}

package/dist/src/ProviderRegistry.js

@@ -0,0 +1,22 @@
+import { PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT, } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Wraps a `dynamic-plugin-framework` {@link PluginManager} to discover and
+ * instantiate pluggable-io-framework source/sink provider plugins.
+ */
+export class ProviderRegistry {
+    pluginManager;
+    constructor(pluginManager) {
+        this.pluginManager = pluginManager;
+    }
+    async discover() {
+        await this.pluginManager.registerExtensions(PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT);
+    }
+    async listAvailableProviders() {
+        return this.pluginManager.getRegisteredExtensions(PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT);
+    }
+    async createProvider(extensionHandle, config) {
+        const factory = (await this.pluginManager.instantiate(extensionHandle));
+        const validatedConfig = factory.configSchema.parse(config);
+        return factory.createProvider(validatedConfig);
+    }
+}

package/dist/src/chunkLength.d.ts

@@ -0,0 +1,3 @@
+import type { ChunkRef } from "@flowscripter/pluggable-io-framework-api";
+export declare function chunkLength(chunk: ChunkRef): number;
+//# sourceMappingURL=chunkLength.d.ts.map

package/dist/src/chunkLength.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkLength.d.ts","sourceRoot":"","sources":["../../src/chunkLength.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0CAA0C,CAAC;AAEzE,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,CAEnD"}

package/dist/src/chunkLength.js

@@ -0,0 +1,3 @@
+export function chunkLength(chunk) {
+    return chunk.kind === "js" ? chunk.data.byteLength : chunk.length;
+}

package/index.ts

@@ -0,0 +1,2 @@
+export * from "./src/ProviderRegistry.ts";
+export * from "./src/copyMove.ts";

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@flowscripter/pluggable-io-framework",
+  "version": "0.1.0",
+  "description": "Pluggable source/sink IO framework - provider discovery, copy/move orchestration, decorators, telemetry",
+  "keywords": [
+    "bun",
+    "io",
+    "plugin",
+    "streaming"
+  ],
+  "homepage": "https://github.com/flowscripter/pluggable-io-framework#readme",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/flowscripter/pluggable-io-framework.git"
+  },
+  "files": [
+    "dist",
+    "src",
+    "index.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "type": "module",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "bun": "./index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "test": "bun test"
+  },
+  "dependencies": {
+    "@flowscripter/dynamic-plugin-framework": "file:../dynamic-plugin-framework",
+    "@flowscripter/pluggable-io-framework-api": "file:../pluggable-io-framework-api"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.14",
+    "oxfmt": "0.57.0",
+    "oxlint": "1.72.0"
+  },
+  "peerDependencies": {
+    "typescript": "^6.0.3"
+  },
+  "typedocOptions": {
+    "readme": "none",
+    "exclude": [
+      "tests/**"
+    ]
+  }
+}

package/src/ProviderRegistry.ts

@@ -0,0 +1,32 @@
+import type { PluginManager } from "@flowscripter/dynamic-plugin-framework";
+import {
+  PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
+  type IOProvider,
+  type IOProviderFactory,
+} from "@flowscripter/pluggable-io-framework-api";
+
+/**
+ * Wraps a `dynamic-plugin-framework` {@link PluginManager} to discover and
+ * instantiate pluggable-io-framework source/sink provider plugins.
+ */
+export class ProviderRegistry {
+  public constructor(private readonly pluginManager: PluginManager) {}
+
+  public async discover(): Promise<void> {
+    await this.pluginManager.registerExtensions(
+      PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
+    );
+  }
+
+  public async listAvailableProviders() {
+    return this.pluginManager.getRegisteredExtensions(
+      PLUGGABLE_IO_FRAMEWORK_PROVIDER_FACTORY_EXTENSION_POINT,
+    );
+  }
+
+  public async createProvider(extensionHandle: string, config: unknown): Promise<IOProvider> {
+    const factory = (await this.pluginManager.instantiate(extensionHandle)) as IOProviderFactory;
+    const validatedConfig = factory.configSchema.parse(config);
+    return factory.createProvider(validatedConfig);
+  }
+}

package/src/chunkLength.ts

@@ -0,0 +1,5 @@
+import type { ChunkRef } from "@flowscripter/pluggable-io-framework-api";
+
+export function chunkLength(chunk: ChunkRef): number {
+  return chunk.kind === "js" ? chunk.data.byteLength : chunk.length;
+}

package/src/copyMove.ts

@@ -0,0 +1,161 @@
+import type {
+  ChunkRef,
+  IOProvider,
+  Part,
+  TelemetryHooks,
+} from "@flowscripter/pluggable-io-framework-api";
+import { chunkLength } from "./chunkLength.ts";
+
+export interface TransferOptions {
+  readonly telemetry?: TelemetryHooks;
+  /** Minimum file size (bytes) before multipart transfer is attempted over plain streaming. */
+  readonly multipartThreshold?: number;
+}
+
+const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
+
+async function streamingTransfer(
+  source: IOProvider,
+  sourcePath: string,
+  sink: IOProvider,
+  destPath: string,
+  operationId: string,
+  telemetry: TelemetryHooks | undefined,
+  totalBytes: number | undefined,
+): Promise<void> {
+  const readable = await source.getReadableStream(sourcePath);
+  const writable = await sink.getWritableStream(destPath);
+  const reader = (readable.stream as ReadableStream<ChunkRef>).getReader();
+  const writer = (writable.stream as WritableStream<ChunkRef>).getWriter();
+  let bytesProcessed = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      await writer.write(value);
+      bytesProcessed += chunkLength(value);
+      telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+    }
+    await writer.close();
+  } catch (error) {
+    await writer.abort(error);
+    throw error;
+  }
+}
+
+async function multipartTransfer(
+  source: IOProvider,
+  sourcePath: string,
+  sink: IOProvider,
+  destPath: string,
+  operationId: string,
+  telemetry: TelemetryHooks | undefined,
+  totalBytes: number | undefined,
+): Promise<void> {
+  let bytesProcessed = 0;
+  const writer = sink.getMultipartWriter(destPath);
+  async function* transferParts(): AsyncIterable<Part> {
+    for await (const part of source.getMultipartReader(sourcePath)) {
+      const reader = (part.stream as ReadableStream<ChunkRef>).getReader();
+      const chunks: ChunkRef[] = [];
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        chunks.push(value);
+        bytesProcessed += chunkLength(value);
+        telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+      }
+      yield {
+        index: part.index,
+        offset: part.offset,
+        stream: new ReadableStream<ChunkRef>({
+          start(controller) {
+            for (const chunk of chunks) controller.enqueue(chunk);
+            controller.close();
+          },
+        }),
+        complete: () => part.complete(),
+      };
+    }
+  }
+  await writer.write(transferParts());
+}
+
+function canUseMultipart(
+  source: IOProvider,
+  sink: IOProvider,
+  size: number | undefined,
+  threshold: number,
+): boolean {
+  return (
+    size !== undefined &&
+    size >= threshold &&
+    typeof source.getMultipartReader === "function" &&
+    typeof sink.getMultipartWriter === "function"
+  );
+}
+
+/**
+ * Copies `sourcePath` on `source` to `destPath` on `sink`.
+ *
+ * Uses `source.directCopy` when `source.canDirectTransfer?.(sink)` reports
+ * eligibility (same-provider direct transfer). Otherwise falls back to
+ * multipart transfer (when both sides support it and size crosses
+ * `options.multipartThreshold`) or plain streaming.
+ */
+export async function copy(
+  source: IOProvider,
+  sourcePath: string,
+  sink: IOProvider,
+  destPath: string,
+  options: TransferOptions = {},
+): Promise<void> {
+  const operationId = crypto.randomUUID();
+  if (source.canDirectTransfer?.(sink) && source.directCopy) {
+    await source.directCopy(sourcePath, destPath);
+    return;
+  }
+  const properties = await source.getProperties(sourcePath);
+  const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
+  if (canUseMultipart(source, sink, properties.size, threshold)) {
+    await multipartTransfer(
+      source,
+      sourcePath,
+      sink,
+      destPath,
+      operationId,
+      options.telemetry,
+      properties.size,
+    );
+    return;
+  }
+  await streamingTransfer(
+    source,
+    sourcePath,
+    sink,
+    destPath,
+    operationId,
+    options.telemetry,
+    properties.size,
+  );
+}
+
+/**
+ * Moves `sourcePath` on `source` to `destPath` on `sink`. Uses
+ * `source.directMove` when eligible, otherwise performs a {@link copy}
+ * followed by deleting the source item.
+ */
+export async function move(
+  source: IOProvider,
+  sourcePath: string,
+  sink: IOProvider,
+  destPath: string,
+  options: TransferOptions = {},
+): Promise<void> {
+  if (source.canDirectTransfer?.(sink) && source.directMove) {
+    await source.directMove(sourcePath, destPath);
+    return;
+  }
+  await copy(source, sourcePath, sink, destPath, options);
+  await source.delete(sourcePath);
+}