@flowscripter/pluggable-io-framework 0.1.0 → 1.0.0

package/README.md

@@ -5,9 +5,7 @@
 [![docs](https://img.shields.io/badge/docs-API-blue)](https://flowscripter.github.io/pluggable-io-framework/index.html)
 [![license: MIT](https://img.shields.io/github/license/flowscripter/pluggable-io-framework)](https://github.com/flowscripter/pluggable-io-framework/blob/main/LICENSE)
 
-> A pluggable source/sink IO framework - provider discovery, copy/move
-> orchestration and telemetry, for files and folders across different
-> backends (local filesystem, object storage, HTTP, etc.)
+> A pluggable source/sink IO framework using https://github.com/flowscripter/dynamic-plugin-framework
 
 ## Key Features
 
@@ -26,13 +24,6 @@
     size crosses a configurable threshold) or plain streaming.
   - Reports progress via a global `TelemetryHooks` callback, tagged with a
     per-operation correlation id.
-- Standalone today - usable directly or via the
-  [flowscripter-io-cli](https://github.com/flowscripter/flowscripter-io-cli)
-  built on
-  [dynamic-cli-framework](https://github.com/flowscripter/dynamic-cli-framework).
-  When the full Flowscripter graph runtime exists, an `adapt` operator will
-  wrap these providers to bridge them into the processing graph - that
-  wrapping is out of scope for this repo.
 - See
   [pluggable-io-framework-plugin-filesystem](https://github.com/flowscripter/pluggable-io-framework-plugin-filesystem)
   for a reference local filesystem source/sink plugin.
@@ -64,6 +55,13 @@ await copy(provider, "a.txt", provider, "b.txt", {
 });
 ```
 
+## Usage Example
+
+The following example project is available:
+
+- [flowscripter-io-cli](https://github.com/flowscripter/flowscripter-io-cli) is
+  an example CLI application based on this framework.
+
 ## Development
 
 Install dependencies:

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./src/ProviderRegistry.ts";
+export * from "./src/copyMove.ts";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from "./src/ProviderRegistry.js";
+export * from "./src/copyMove.js";

package/dist/src/ProviderRegistry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ProviderRegistry.d.ts","sourceRoot":"","sources":["../../src/ProviderRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAEL,KAAK,UAAU,EAEhB,MAAM,0CAA0C,CAAC;AAElD;;;GAGG;AACH,qBAAa,gBAAgB;IACR,OAAO,CAAC,QAAQ,CAAC,aAAa;gBAAb,aAAa,EAAE,aAAa;IAEnD,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAIzB,sBAAsB;IAItB,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC;CAK3F"}
+{"version":3,"file":"ProviderRegistry.d.ts","sourceRoot":"","sources":["../../src/ProviderRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAEL,KAAK,UAAU,EAEhB,MAAM,0CAA0C,CAAC;AAElD;;;GAGG;AACH,qBAAa,gBAAgB;IACR,OAAO,CAAC,QAAQ,CAAC,aAAa;gBAAb,aAAa,EAAE,aAAa;IAEnD,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAMzB,sBAAsB;IAMtB,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC;CAK3F"}

package/dist/src/chunkLength.d.ts

@@ -1,3 +1,3 @@
-import type { ChunkRef } from "@flowscripter/pluggable-io-framework-api";
+import { type ChunkRef } from "@flowscripter/pluggable-io-framework-api";
 export declare function chunkLength(chunk: ChunkRef): number;
 //# sourceMappingURL=chunkLength.d.ts.map

package/dist/src/chunkLength.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunkLength.d.ts","sourceRoot":"","sources":["../../src/chunkLength.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0CAA0C,CAAC;AAEzE,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,CAEnD"}
+{"version":3,"file":"chunkLength.d.ts","sourceRoot":"","sources":["../../src/chunkLength.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,0CAA0C,CAAC;AAEpF,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,CAEnD"}

package/dist/src/chunkLength.js

@@ -1,3 +1,4 @@
+import { ChunkKind } from "@flowscripter/pluggable-io-framework-api";
 export function chunkLength(chunk) {
-    return chunk.kind === "js" ? chunk.data.byteLength : chunk.length;
+    return chunk.kind === ChunkKind.Js ? chunk.data.byteLength : chunk.length;
 }

package/dist/src/copyMove.d.ts

@@ -0,0 +1,29 @@
+import { type ChunkConverter, type IOProvider, type TelemetryHooks } from "@flowscripter/pluggable-io-framework-api";
+export interface TransferOptions {
+    readonly telemetry?: TelemetryHooks;
+    /** Minimum file size (bytes) before multipart transfer is attempted over plain streaming. */
+    readonly multipartThreshold?: number;
+    /**
+     * Required only when `source.kind !== sink.kind` - pure-TS code can only
+     * convert chunks already of the target kind (see
+     * `identityChunkConverter`); a real js<->native conversion needs an
+     * FFI-capable converter supplied by a runtime-specific package.
+     */
+    readonly chunkConverter?: ChunkConverter;
+}
+/**
+ * Copies `sourcePath` on `source` to `destPath` on `sink`.
+ *
+ * Uses `source.directCopy` when `source.canDirectTransfer?.(sink)` reports
+ * eligibility (same-provider direct transfer). Otherwise falls back to
+ * multipart transfer (when both sides support it and size crosses
+ * `options.multipartThreshold`) or plain streaming.
+ */
+export declare function copy(source: IOProvider, sourcePath: string, sink: IOProvider, destPath: string, options?: TransferOptions): Promise<void>;
+/**
+ * Moves `sourcePath` on `source` to `destPath` on `sink`. Uses
+ * `source.directMove` when eligible, otherwise performs a {@link copy}
+ * followed by deleting the source item.
+ */
+export declare function move(source: IOProvider, sourcePath: string, sink: IOProvider, destPath: string, options?: TransferOptions): Promise<void>;
+//# sourceMappingURL=copyMove.d.ts.map

package/dist/src/copyMove.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"copyMove.d.ts","sourceRoot":"","sources":["../../src/copyMove.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EAEnB,KAAK,UAAU,EAEf,KAAK,cAAc,EACpB,MAAM,0CAA0C,CAAC;AAGlD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACpC,6FAA6F;IAC7F,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,CAAC;CAC1C;AAoGD;;;;;;;GAOG;AACH,wBAAsB,IAAI,CACxB,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;GAIG;AACH,wBAAsB,IAAI,CACxB,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAOf"}

package/dist/src/copyMove.js

@@ -0,0 +1,102 @@
+import { adaptReadableStream, identityChunkConverter, } from "@flowscripter/pluggable-io-framework-api";
+import { chunkLength } from "./chunkLength.js";
+const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
+async function streamingTransfer(source, sourcePath, sink, destPath, operationId, options, totalBytes) {
+    const readable = await source.getReadableStream(sourcePath);
+    const writable = await sink.getWritableStream(destPath);
+    // The source/sink kind mismatch is decided ONCE here, not per chunk.
+    const adaptedStream = adaptReadableStream(readable.stream, readable.kind, writable.kind, options.chunkConverter ?? identityChunkConverter);
+    const reader = adaptedStream.getReader();
+    const writer = writable.stream.getWriter();
+    let bytesProcessed = 0;
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            await writer.write(value);
+            bytesProcessed += chunkLength(value);
+            options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+        }
+        await writer.close();
+    }
+    catch (error) {
+        await writer.abort(error);
+        throw error;
+    }
+}
+async function multipartTransfer(source, sourcePath, sink, destPath, operationId, options, totalBytes) {
+    let bytesProcessed = 0;
+    const writer = sink.getMultipartWriter(destPath);
+    async function* transferParts() {
+        for await (const part of source.getMultipartReader(sourcePath)) {
+            // Kind mismatch decided once per part, not once per chunk within it.
+            const adaptedStream = adaptReadableStream(part.stream, part.kind, sink.kind, options.chunkConverter ?? identityChunkConverter);
+            const reader = adaptedStream.getReader();
+            const chunks = [];
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                chunks.push(value);
+                bytesProcessed += chunkLength(value);
+                options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+            }
+            yield {
+                index: part.index,
+                offset: part.offset,
+                kind: sink.kind,
+                stream: new ReadableStream({
+                    start(controller) {
+                        for (const chunk of chunks)
+                            controller.enqueue(chunk);
+                        controller.close();
+                    },
+                }),
+                complete: () => part.complete(),
+            };
+        }
+    }
+    await writer.write(transferParts());
+}
+function canUseMultipart(source, sink, size, threshold) {
+    return (size !== undefined &&
+        size >= threshold &&
+        typeof source.getMultipartReader === "function" &&
+        typeof sink.getMultipartWriter === "function");
+}
+/**
+ * Copies `sourcePath` on `source` to `destPath` on `sink`.
+ *
+ * Uses `source.directCopy` when `source.canDirectTransfer?.(sink)` reports
+ * eligibility (same-provider direct transfer). Otherwise falls back to
+ * multipart transfer (when both sides support it and size crosses
+ * `options.multipartThreshold`) or plain streaming.
+ */
+export async function copy(source, sourcePath, sink, destPath, options = {}) {
+    const operationId = crypto.randomUUID();
+    if (source.canDirectTransfer?.(sink) && source.directCopy) {
+        await source.directCopy(sourcePath, destPath);
+        return;
+    }
+    const properties = await source.getProperties(sourcePath);
+    const threshold = options.multipartThreshold ?? DEFAULT_MULTIPART_THRESHOLD;
+    if (canUseMultipart(source, sink, properties.size, threshold)) {
+        await multipartTransfer(source, sourcePath, sink, destPath, operationId, options, properties.size);
+        return;
+    }
+    await streamingTransfer(source, sourcePath, sink, destPath, operationId, options, properties.size);
+}
+/**
+ * Moves `sourcePath` on `source` to `destPath` on `sink`. Uses
+ * `source.directMove` when eligible, otherwise performs a {@link copy}
+ * followed by deleting the source item.
+ */
+export async function move(source, sourcePath, sink, destPath, options = {}) {
+    if (source.canDirectTransfer?.(sink) && source.directMove) {
+        await source.directMove(sourcePath, destPath);
+        return;
+    }
+    await copy(source, sourcePath, sink, destPath, options);
+    await source.delete(sourcePath);
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@flowscripter/pluggable-io-framework",
-  "version": "0.1.0",
-  "description": "Pluggable source/sink IO framework - provider discovery, copy/move orchestration, decorators, telemetry",
+  "version": "1.0.0",
+  "description": "A pluggable source/sink IO framework using https://github.com/flowscripter/dynamic-plugin-framework",
   "keywords": [
     "bun",
     "io",
@@ -40,8 +40,8 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@flowscripter/dynamic-plugin-framework": "file:../dynamic-plugin-framework",
-    "@flowscripter/pluggable-io-framework-api": "file:../pluggable-io-framework-api"
+    "@flowscripter/dynamic-plugin-framework": "^2.2.0",
+    "@flowscripter/pluggable-io-framework-api": "^1.0.1"
   },
   "devDependencies": {
     "@types/bun": "^1.3.14",

package/src/chunkLength.ts

@@ -1,5 +1,5 @@
-import type { ChunkRef } from "@flowscripter/pluggable-io-framework-api";
+import { ChunkKind, type ChunkRef } from "@flowscripter/pluggable-io-framework-api";
 
 export function chunkLength(chunk: ChunkRef): number {
-  return chunk.kind === "js" ? chunk.data.byteLength : chunk.length;
+  return chunk.kind === ChunkKind.Js ? chunk.data.byteLength : chunk.length;
 }

package/src/copyMove.ts

@@ -1,8 +1,11 @@
-import type {
-  ChunkRef,
-  IOProvider,
-  Part,
-  TelemetryHooks,
+import {
+  adaptReadableStream,
+  identityChunkConverter,
+  type ChunkConverter,
+  type ChunkRef,
+  type IOProvider,
+  type Part,
+  type TelemetryHooks,
 } from "@flowscripter/pluggable-io-framework-api";
 import { chunkLength } from "./chunkLength.ts";
 
@@ -10,6 +13,13 @@ export interface TransferOptions {
   readonly telemetry?: TelemetryHooks;
   /** Minimum file size (bytes) before multipart transfer is attempted over plain streaming. */
   readonly multipartThreshold?: number;
+  /**
+   * Required only when `source.kind !== sink.kind` - pure-TS code can only
+   * convert chunks already of the target kind (see
+   * `identityChunkConverter`); a real js<->native conversion needs an
+   * FFI-capable converter supplied by a runtime-specific package.
+   */
+  readonly chunkConverter?: ChunkConverter;
 }
 
 const DEFAULT_MULTIPART_THRESHOLD = 64 * 1024 * 1024;
@@ -20,12 +30,19 @@ async function streamingTransfer(
   sink: IOProvider,
   destPath: string,
   operationId: string,
-  telemetry: TelemetryHooks | undefined,
+  options: TransferOptions,
   totalBytes: number | undefined,
 ): Promise<void> {
   const readable = await source.getReadableStream(sourcePath);
   const writable = await sink.getWritableStream(destPath);
-  const reader = (readable.stream as ReadableStream<ChunkRef>).getReader();
+  // The source/sink kind mismatch is decided ONCE here, not per chunk.
+  const adaptedStream = adaptReadableStream(
+    readable.stream as ReadableStream<ChunkRef>,
+    readable.kind,
+    writable.kind,
+    options.chunkConverter ?? identityChunkConverter,
+  );
+  const reader = adaptedStream.getReader();
   const writer = (writable.stream as WritableStream<ChunkRef>).getWriter();
   let bytesProcessed = 0;
   try {
@@ -34,7 +51,7 @@ async function streamingTransfer(
       if (done) break;
       await writer.write(value);
       bytesProcessed += chunkLength(value);
-      telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+      options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
     }
     await writer.close();
   } catch (error) {
@@ -49,25 +66,33 @@ async function multipartTransfer(
   sink: IOProvider,
   destPath: string,
   operationId: string,
-  telemetry: TelemetryHooks | undefined,
+  options: TransferOptions,
   totalBytes: number | undefined,
 ): Promise<void> {
   let bytesProcessed = 0;
   const writer = sink.getMultipartWriter(destPath);
   async function* transferParts(): AsyncIterable<Part> {
     for await (const part of source.getMultipartReader(sourcePath)) {
-      const reader = (part.stream as ReadableStream<ChunkRef>).getReader();
+      // Kind mismatch decided once per part, not once per chunk within it.
+      const adaptedStream = adaptReadableStream(
+        part.stream as ReadableStream<ChunkRef>,
+        part.kind,
+        sink.kind,
+        options.chunkConverter ?? identityChunkConverter,
+      );
+      const reader = adaptedStream.getReader();
       const chunks: ChunkRef[] = [];
       for (;;) {
         const { done, value } = await reader.read();
         if (done) break;
         chunks.push(value);
         bytesProcessed += chunkLength(value);
-        telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
+        options.telemetry?.onProgress?.({ operationId, type: "copy", bytesProcessed, totalBytes });
       }
       yield {
         index: part.index,
         offset: part.offset,
+        kind: sink.kind,
         stream: new ReadableStream<ChunkRef>({
           start(controller) {
             for (const chunk of chunks) controller.enqueue(chunk);
@@ -124,7 +149,7 @@ export async function copy(
       sink,
       destPath,
       operationId,
-      options.telemetry,
+      options,
       properties.size,
     );
     return;
@@ -135,7 +160,7 @@ export async function copy(
     sink,
     destPath,
     operationId,
-    options.telemetry,
+    options,
     properties.size,
   );
 }