@milaboratories/pl-drivers 1.10.9 → 1.10.10

package/src/drivers/download_blob/download_blob.ts

@@ -12,6 +12,8 @@ import { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-
 import type {
   AnyLogHandle,
   BlobDriver,
+  ContentHandler,
+  GetContentOptions,
   LocalBlobHandle,
   LocalBlobHandleAndSize,
   ReadyLogHandle,
@@ -39,11 +41,11 @@ import * as fsp from 'node:fs/promises';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import * as readline from 'node:readline/promises';
-import { Writable } from 'node:stream';
 import { buffer } from 'node:stream/consumers';
+import { Readable } from 'node:stream';
 import type { ClientDownload } from '../../clients/download';
 import type { ClientLogs } from '../../clients/logs';
-import { readFileContent } from '../helpers/read_file';
+import { withFileContent } from '../helpers/read_file';
 import {
   isLocalBlobHandle,
   newLocalHandle,
@@ -293,13 +295,55 @@ export class DownloadDriver implements BlobDriver {
   }
 
   /** Gets a content of a blob by a handle. */
-  public async getContent(handle: LocalBlobHandle | RemoteBlobHandle, range?: RangeBytes): Promise<Uint8Array> {
-    if (range) {
-      validateRangeBytes(range, `getContent`);
+  public async getContent(handle: LocalBlobHandle | RemoteBlobHandle): Promise<Uint8Array>;
+  public async getContent(
+    handle: LocalBlobHandle | RemoteBlobHandle,
+    options?: GetContentOptions,
+  ): Promise<Uint8Array>;
+  /** @deprecated Use {@link getContent} with {@link GetContentOptions} instead */
+  public async getContent(
+    handle: LocalBlobHandle | RemoteBlobHandle,
+    range?: RangeBytes,
+  ): Promise<Uint8Array>;
+  public async getContent(
+    handle: LocalBlobHandle | RemoteBlobHandle,
+    optionsOrRange?: GetContentOptions | RangeBytes,
+  ): Promise<Uint8Array> {
+    let options: GetContentOptions = {};
+    if (typeof optionsOrRange === 'object' && optionsOrRange !== null) {
+      if ('range' in optionsOrRange) {
+        options = optionsOrRange;
+      } else {
+        const range = optionsOrRange as RangeBytes;
+        validateRangeBytes(range, `getContent`);
+        options = { range };
+      }
     }
 
+    return await this.withContent(handle, {
+      ...options,
+      handler: async (content) => {
+        const chunks: Uint8Array[] = [];
+        for await (const chunk of content) {
+          options.signal?.throwIfAborted();
+          chunks.push(chunk);
+        }
+        return Buffer.concat(chunks);
+      }
+    });
+  }
+
+  /** Gets a content stream of a blob by a handle and calls handler with it. */
+  public async withContent<T>(
+    handle: LocalBlobHandle | RemoteBlobHandle,
+    options: GetContentOptions & {
+      handler: ContentHandler<T>;
+    },
+  ): Promise<T> {
+    const { range, signal, handler } = options;
+
     if (isLocalBlobHandle(handle)) {
-      return await readFileContent(this.getLocalPath(handle), range);
+      return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });
     }
 
     if (isRemoteBlobHandle(handle)) {
@@ -307,19 +351,24 @@ export class DownloadDriver implements BlobDriver {
 
       const key = blobKey(result.info.id);
       const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });
-      if (filePath) {
-        return await readFileContent(filePath, range);
-      }
-
-      const data = await this.clientDownload.withBlobContent(
-        { id: result.info.id, type: result.info.type },
-        undefined,
-        { range },
-        async (content) => await buffer(content)
+      signal?.throwIfAborted();
+
+      if (filePath) return await withFileContent({ path: filePath, range, signal, handler });
+
+      return await this.clientDownload.withBlobContent(
+        result.info,
+        { signal },
+        options,
+        async (content, size) => {
+          const [handlerStream, cacheStream] = content.tee();
+          
+          const handlerPromise = handler(handlerStream, size);
+          const _cachePromise = buffer(cacheStream)
+            .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data));
+
+          return await handlerPromise;
+        }
       );
-      await this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data);
-
-      return data;
     }
 
     throw new Error('Malformed remote handle');
@@ -339,7 +388,7 @@ export class DownloadDriver implements BlobDriver {
 
     return Computable.make((ctx) =>
       this.getDownloadedBlob(res, ctx), {
-      postprocessValue: (v) => v ? this.getContent(v.handle, range) : undefined
+      postprocessValue: (v) => v ? this.getContent(v.handle, { range }) : undefined
     }
     ).withStableType()
   }

package/src/drivers/helpers/helpers.ts

@@ -1,12 +1,3 @@
-import {
-  BasicResourceData,
-  getField,
-  isNullResourceId,
-  PlClient,
-  ResourceId,
-  valErr,
-} from '@milaboratories/pl-client';
-
 /** Throws when a driver gets a resource with a wrong resource type. */
 export class WrongResourceTypeError extends Error {
   name = 'WrongResourceTypeError';

package/src/drivers/helpers/read_file.ts

@@ -1,7 +1,8 @@
 import { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
 import type { RangeBytes } from '@milaboratories/pl-model-common';
 import * as fs from 'node:fs';
-import { buffer } from 'node:stream/consumers';
+import * as fsp from 'node:fs/promises';
+import { Readable } from 'node:stream';
 
 // Global concurrency limiter for file reads - limit to 32 parallel reads
 const fileReadLimiter = new ConcurrencyLimitingExecutor(32);
@@ -10,20 +11,37 @@ const fileReadLimiter = new ConcurrencyLimitingExecutor(32);
  * Reads file content with concurrency limiting and proper error handling.
  * Ensures file descriptors are properly cleaned up even in error cases.
  */
-export async function readFileContent(path: string, range?: RangeBytes): Promise<Uint8Array> {
+export async function withFileContent<T>({
+  path,
+  range,
+  signal,
+  handler,
+}: {
+  path: string;
+  range?: RangeBytes;
+  signal?: AbortSignal;
+  handler: (content: ReadableStream, size: number) => Promise<T>;
+}): Promise<T> {
   return await fileReadLimiter.run(async () => {
-    const ops: { start?: number; end?: number } = {};
-    if (range) {
-      ops.start = range.from;
-      ops.end = range.to - 1;
-    }
-
+    const readOps = {
+      start: range?.from,
+      end: range?.to !== undefined ? range.to - 1 : undefined,
+      signal: signal,
+    };
     let stream: fs.ReadStream | undefined;
+    let handlerSuccess = false;
+
     try {
-      stream = fs.createReadStream(path, ops);
-      return await buffer(stream);
+      const stat = await fsp.stat(path);
+      stream = fs.createReadStream(path, readOps);
+      const webStream = Readable.toWeb(stream);
+
+      const result = await handler(webStream, stat.size);
+      handlerSuccess = true;
+      return result;
     } catch (error) {
-      if (stream && !stream.destroyed) {
+      // Cleanup on error (including handler errors)
+      if (!handlerSuccess && stream && !stream.destroyed) {
         stream.destroy();
       }
       throw error;

package/src/helpers/download.ts

@@ -5,12 +5,7 @@ import { request } from 'undici';
 import { Readable } from 'node:stream';
 import type { ReadableStream } from 'node:stream/web';
 import { text } from 'node:stream/consumers';
-import type { RangeBytes } from '@milaboratories/pl-model-common';
-
-export interface DownloadOps {
-  signal?: AbortSignal;
-  range?: RangeBytes;
-}
+import type { GetContentOptions } from '@milaboratories/pl-model-common';
 
 export type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;
 
@@ -25,7 +20,7 @@ export class RemoteFileDownloader {
   async withContent<T>(
     url: string,
     reqHeaders: Record<string, string>,
-    ops: DownloadOps,
+    ops: GetContentOptions,
     handler: ContentHandler<T>,
   ): Promise<T> {
     const headers = { ...reqHeaders };
@@ -40,14 +35,19 @@ export class RemoteFileDownloader {
       headers,
       signal: ops.signal,
     });
+    ops.signal?.throwIfAborted();
 
     const webBody = Readable.toWeb(body);
     let handlerSuccess = false;
 
     try {
       await checkStatusCodeOk(statusCode, webBody, url);
+      ops.signal?.throwIfAborted();
+
       const size = Number(responseHeaders['content-length']);
       const result = await handler(webBody, size);
+      ops.signal?.throwIfAborted();
+
       handlerSuccess = true;
       return result;
     } catch (error) {