@flowscripter/pluggable-io-framework 1.0.1 → 1.1.0

package/README.md

@@ -24,6 +24,13 @@
     size crosses a configurable threshold) or plain streaming.
   - Reports progress via a global `TelemetryHooks` callback, tagged with a
     per-operation correlation id.
+- Stream decorators:
+  - `seekable` wraps a handle that supports `RangeReadable` with a single
+    logical stream whose read position can be jumped via `seek(offset)`,
+    instead of requiring a fresh stream per range.
+  - `locallyCached` wraps a `StreamHandle` factory so the underlying source
+    is read at most once - later calls replay cached chunks in memory
+    without touching the source again.
 - See
   [io-plugin-filesystem](https://github.com/flowscripter/io-plugin-filesystem)
   for a reference local filesystem source/sink plugin.

package/dist/index.d.ts

@@ -1,3 +1,5 @@
 export * from "./src/ProviderRegistry.ts";
 export * from "./src/copyMove.ts";
+export * from "./src/decorators/locallyCached.ts";
+export * from "./src/decorators/seekable.ts";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mCAAmC,CAAC;AAClD,cAAc,8BAA8B,CAAC"}

package/dist/index.js

@@ -1,2 +1,4 @@
 export * from "./src/ProviderRegistry.js";
 export * from "./src/copyMove.js";
+export * from "./src/decorators/locallyCached.js";
+export * from "./src/decorators/seekable.js";

package/dist/src/decorators/locallyCached.d.ts

@@ -0,0 +1,15 @@
+import type { ChunkKind, StreamHandle } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Wraps a `StreamHandle` factory (e.g. `() => provider.getReadableStream(path)`)
+ * so the underlying source is read at most once - the first call drains the
+ * source while caching every chunk in memory; every subsequent call replays
+ * the cached chunks without touching the source again.
+ *
+ * A plain `StreamHandle` only exposes a single one-shot `ReadableStream`, so
+ * caching can't be a `StreamDecorator<K, C>` operating on an already-open
+ * handle (there would be nothing left to re-read on a second call) - it has
+ * to intercept the *open* operation itself, provider-agnostic regardless of
+ * what kind of source is behind it.
+ */
+export declare function locallyCached<K extends ChunkKind>(open: () => Promise<StreamHandle<K>>): () => Promise<StreamHandle<K>>;
+//# sourceMappingURL=locallyCached.d.ts.map

package/dist/src/decorators/locallyCached.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"locallyCached.d.ts","sourceRoot":"","sources":["../../../src/decorators/locallyCached.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,YAAY,EACb,MAAM,0CAA0C,CAAC;AAElD;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAAC,CAAC,SAAS,SAAS,EAC/C,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GACnC,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAuChC"}

package/dist/src/decorators/locallyCached.js

@@ -0,0 +1,48 @@
+/**
+ * Wraps a `StreamHandle` factory (e.g. `() => provider.getReadableStream(path)`)
+ * so the underlying source is read at most once - the first call drains the
+ * source while caching every chunk in memory; every subsequent call replays
+ * the cached chunks without touching the source again.
+ *
+ * A plain `StreamHandle` only exposes a single one-shot `ReadableStream`, so
+ * caching can't be a `StreamDecorator<K, C>` operating on an already-open
+ * handle (there would be nothing left to re-read on a second call) - it has
+ * to intercept the *open* operation itself, provider-agnostic regardless of
+ * what kind of source is behind it.
+ */
+export function locallyCached(open) {
+    let cache;
+    function replay(chunks) {
+        return new ReadableStream({
+            start(controller) {
+                for (const chunk of chunks)
+                    controller.enqueue(chunk);
+                controller.close();
+            },
+        });
+    }
+    return async () => {
+        if (cache) {
+            return { kind: cache.kind, stream: replay(cache.chunks) };
+        }
+        const handle = await open();
+        const reader = handle.stream.getReader();
+        const chunks = [];
+        const stream = new ReadableStream({
+            async pull(controller) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    cache = { kind: handle.kind, chunks };
+                    controller.close();
+                    return;
+                }
+                chunks.push(value);
+                controller.enqueue(value);
+            },
+            cancel(reason) {
+                return reader.cancel(reason);
+            },
+        });
+        return { kind: handle.kind, stream };
+    };
+}

package/dist/src/decorators/seekable.d.ts

@@ -0,0 +1,13 @@
+import type { ChunkKind, RangeReadable, Seekable, StreamHandle } from "@flowscripter/pluggable-io-framework-api";
+/**
+ * Wraps a handle that already supports {@link RangeReadable} (arbitrary
+ * byte-range reads) with a {@link Seekable} capability: a single logical
+ * stream whose read position can be jumped via `seek(offset)`, rather than
+ * requiring the caller to open a fresh stream per range.
+ *
+ * `seek` must not be called while a read from the current position is still
+ * in flight - like any seekable stream, seeking and reading are sequential,
+ * not concurrent, operations on the same handle.
+ */
+export declare function seekable<K extends ChunkKind>(handle: StreamHandle<K> & RangeReadable<K>): StreamHandle<K> & Seekable;
+//# sourceMappingURL=seekable.d.ts.map

package/dist/src/decorators/seekable.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"seekable.d.ts","sourceRoot":"","sources":["../../../src/decorators/seekable.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,aAAa,EACb,QAAQ,EACR,YAAY,EACb,MAAM,0CAA0C,CAAC;AAElD;;;;;;;;;GASG;AACH,wBAAgB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAC1C,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GACzC,YAAY,CAAC,CAAC,CAAC,GAAG,QAAQ,CAiC5B"}

package/dist/src/decorators/seekable.js

@@ -0,0 +1,40 @@
+/**
+ * Wraps a handle that already supports {@link RangeReadable} (arbitrary
+ * byte-range reads) with a {@link Seekable} capability: a single logical
+ * stream whose read position can be jumped via `seek(offset)`, rather than
+ * requiring the caller to open a fresh stream per range.
+ *
+ * `seek` must not be called while a read from the current position is still
+ * in flight - like any seekable stream, seeking and reading are sequential,
+ * not concurrent, operations on the same handle.
+ */
+export function seekable(handle) {
+    let reader = handle.stream.getReader();
+    const stream = new ReadableStream({
+        async pull(controller) {
+            const { done, value } = await reader.read();
+            if (done) {
+                controller.close();
+                return;
+            }
+            controller.enqueue(value);
+        },
+        cancel(reason) {
+            return reader.cancel(reason);
+        },
+    }, 
+    // Default queuing strategy pre-fetches one chunk ahead of what the
+    // consumer has actually asked for, which would call pull() again -
+    // hitting "done" on the pre-seek reader and closing the stream - before
+    // a seek() issued between two explicit reads ever runs. highWaterMark 0
+    // makes pull() fire only for an explicit read, never speculatively.
+    { highWaterMark: 0 });
+    return {
+        kind: handle.kind,
+        stream,
+        async seek(offset) {
+            const ranged = await handle.readRange(offset, Number.MAX_SAFE_INTEGER);
+            reader = ranged.getReader();
+        },
+    };
+}

package/index.ts

@@ -1,2 +1,4 @@
 export * from "./src/ProviderRegistry.ts";
 export * from "./src/copyMove.ts";
+export * from "./src/decorators/locallyCached.ts";
+export * from "./src/decorators/seekable.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowscripter/pluggable-io-framework",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "A pluggable source/sink IO framework using https://github.com/flowscripter/dynamic-plugin-framework",
   "keywords": [
     "bun",
@@ -41,7 +41,7 @@
   },
   "dependencies": {
     "@flowscripter/dynamic-plugin-framework": "^2.2.0",
-    "@flowscripter/pluggable-io-framework-api": "^1.0.1"
+    "@flowscripter/pluggable-io-framework-api": "^1.0.4"
   },
   "devDependencies": {
     "@types/bun": "^1.3.14",

package/src/decorators/locallyCached.ts

@@ -0,0 +1,60 @@
+import type {
+  ChunkKind,
+  ChunkOfKind,
+  StreamHandle,
+} from "@flowscripter/pluggable-io-framework-api";
+
+/**
+ * Wraps a `StreamHandle` factory (e.g. `() => provider.getReadableStream(path)`)
+ * so the underlying source is read at most once - the first call drains the
+ * source while caching every chunk in memory; every subsequent call replays
+ * the cached chunks without touching the source again.
+ *
+ * A plain `StreamHandle` only exposes a single one-shot `ReadableStream`, so
+ * caching can't be a `StreamDecorator<K, C>` operating on an already-open
+ * handle (there would be nothing left to re-read on a second call) - it has
+ * to intercept the *open* operation itself, provider-agnostic regardless of
+ * what kind of source is behind it.
+ */
+export function locallyCached<K extends ChunkKind>(
+  open: () => Promise<StreamHandle<K>>,
+): () => Promise<StreamHandle<K>> {
+  let cache: { kind: K; chunks: ChunkOfKind<K>[] } | undefined;
+
+  function replay(chunks: ChunkOfKind<K>[]): ReadableStream<ChunkOfKind<K>> {
+    return new ReadableStream<ChunkOfKind<K>>({
+      start(controller) {
+        for (const chunk of chunks) controller.enqueue(chunk);
+        controller.close();
+      },
+    });
+  }
+
+  return async () => {
+    if (cache) {
+      return { kind: cache.kind, stream: replay(cache.chunks) };
+    }
+
+    const handle = await open();
+    const reader = (handle.stream as ReadableStream<ChunkOfKind<K>>).getReader();
+    const chunks: ChunkOfKind<K>[] = [];
+
+    const stream = new ReadableStream<ChunkOfKind<K>>({
+      async pull(controller) {
+        const { done, value } = await reader.read();
+        if (done) {
+          cache = { kind: handle.kind, chunks };
+          controller.close();
+          return;
+        }
+        chunks.push(value);
+        controller.enqueue(value);
+      },
+      cancel(reason) {
+        return reader.cancel(reason);
+      },
+    });
+
+    return { kind: handle.kind, stream };
+  };
+}

package/src/decorators/seekable.ts

@@ -0,0 +1,54 @@
+import type {
+  ChunkKind,
+  ChunkOfKind,
+  RangeReadable,
+  Seekable,
+  StreamHandle,
+} from "@flowscripter/pluggable-io-framework-api";
+
+/**
+ * Wraps a handle that already supports {@link RangeReadable} (arbitrary
+ * byte-range reads) with a {@link Seekable} capability: a single logical
+ * stream whose read position can be jumped via `seek(offset)`, rather than
+ * requiring the caller to open a fresh stream per range.
+ *
+ * `seek` must not be called while a read from the current position is still
+ * in flight - like any seekable stream, seeking and reading are sequential,
+ * not concurrent, operations on the same handle.
+ */
+export function seekable<K extends ChunkKind>(
+  handle: StreamHandle<K> & RangeReadable<K>,
+): StreamHandle<K> & Seekable {
+  let reader = (handle.stream as ReadableStream<ChunkOfKind<K>>).getReader();
+
+  const stream = new ReadableStream<ChunkOfKind<K>>(
+    {
+      async pull(controller) {
+        const { done, value } = await reader.read();
+        if (done) {
+          controller.close();
+          return;
+        }
+        controller.enqueue(value);
+      },
+      cancel(reason) {
+        return reader.cancel(reason);
+      },
+    },
+    // Default queuing strategy pre-fetches one chunk ahead of what the
+    // consumer has actually asked for, which would call pull() again -
+    // hitting "done" on the pre-seek reader and closing the stream - before
+    // a seek() issued between two explicit reads ever runs. highWaterMark 0
+    // makes pull() fire only for an explicit read, never speculatively.
+    { highWaterMark: 0 },
+  );
+
+  return {
+    kind: handle.kind,
+    stream,
+    async seek(offset: number) {
+      const ranged = await handle.readRange(offset, Number.MAX_SAFE_INTEGER);
+      reader = ranged.getReader();
+    },
+  };
+}