@milaboratories/pl-drivers 1.7.0 → 1.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "engines": {
     "node": ">=20"
   },
@@ -31,12 +31,12 @@
     "undici": "~7.10.0",
     "zod": "~3.23.8",
     "upath": "^2.0.1",
+    "@milaboratories/ts-helpers": "^1.4.3",
     "@milaboratories/helpers": "^1.6.19",
-    "@milaboratories/computable": "^2.6.2",
-    "@milaboratories/ts-helpers": "^1.4.2",
-    "@milaboratories/pl-tree": "^1.7.4",
-    "@milaboratories/pl-client": "^2.11.5",
-    "@milaboratories/pl-model-common": "^1.19.0"
+    "@milaboratories/pl-client": "^2.11.6",
+    "@milaboratories/computable": "^2.6.3",
+    "@milaboratories/pl-model-common": "^1.19.1",
+    "@milaboratories/pl-tree": "^1.7.5"
   },
   "devDependencies": {
     "eslint": "^9.25.1",
@@ -47,8 +47,8 @@
     "vitest": "^2.1.9",
     "@vitest/coverage-v8": "^2.1.9",
     "@types/tar-fs": "^2.0.4",
-    "@milaboratories/build-configs": "1.0.5",
-    "@milaboratories/eslint-config": "^1.0.4"
+    "@milaboratories/eslint-config": "^1.0.4",
+    "@milaboratories/build-configs": "1.0.5"
   },
   "scripts": {
     "type-check": "tsc --noEmit --composite false",

package/src/clients/download.test.ts

@@ -45,10 +45,19 @@ test('client download from a local file', async () => {
         ]),
     });
 
-    const localFile = await clientDownload.readLocalFile(`storage://tmp/${fName}`);
+    const result = await clientDownload.withLocalFileContent(
+      `storage://tmp/${fName}`,
+      {},
+      async (content, size) => {
+        expect(size).toBe(2);
+        const textContent = await text(content);
+        expect(textContent).toBe('42');
+        return { size, textContent };
+      }
+    );
 
-    expect(localFile.size).toBe(2);
-    expect(await text(localFile.content)).toBe('42');
+    expect(result.size).toBe(2);
+    expect(result.textContent).toBe('42');
 
     await fs.rm(fPath);
   });

package/src/clients/download.ts

@@ -3,6 +3,7 @@ import type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaborator
 import { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';
 import type { ResourceInfo } from '@milaboratories/pl-tree';
 import type { MiLogger } from '@milaboratories/ts-helpers';
+import { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import * as fs from 'node:fs';
 import * as fsp from 'node:fs/promises';
@@ -10,12 +11,11 @@ import * as path from 'node:path';
 import { Readable } from 'node:stream';
 import type { Dispatcher } from 'undici';
 import type { LocalStorageProjection } from '../drivers/types';
-import type { DownloadResponse } from '../helpers/download';
+import type { DownloadOps, ContentHandler } from '../helpers/download';
 import { RemoteFileDownloader } from '../helpers/download';
 import { validateAbsolute } from '../helpers/validate';
 import type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';
 import { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';
-import { toHeadersMap } from './helpers';
 
 /** Gets URLs for downloading from pl-core, parses them and reads or downloads
  * files locally and from the web. */
@@ -26,6 +26,9 @@ export class ClientDownload {
   /** Helps to find a storage root directory by a storage id from URL scheme. */
   private readonly localStorageIdsToRoot: Map<string, string>;
 
+  /** Concurrency limiter for local file reads - limit to 32 parallel reads */
+  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);
+
   constructor(
     grpcClientProviderFactory: GrpcClientProviderFactory,
     public readonly httpClient: Dispatcher,
@@ -40,40 +43,60 @@ export class ClientDownload {
 
   close() {}
 
-  /** Gets a presign URL and downloads the file.
+  /**
+   * Gets a presign URL and downloads the file.
    * An optional range with 2 numbers from what byte and to what byte to download can be provided.
    * @param fromBytes - from byte including this byte
    * @param toBytes - to byte excluding this byte
    */
-  async downloadBlob(
+  async withBlobContent<T>(
     info: ResourceInfo,
-    options?: RpcOptions,
-    signal?: AbortSignal,
-    fromBytes?: number,
-    toBytes?: number,
-  ): Promise<DownloadResponse> {
-    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, signal);
+    options: RpcOptions | undefined,
+    ops: DownloadOps,
+    handler: ContentHandler<T>,
+  ): Promise<T> {
+    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
 
-    const remoteHeaders = toHeadersMap(headers, fromBytes, toBytes);
-    this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, headers: ${JSON.stringify(remoteHeaders)}`);
+    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
+    this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);
 
     return isLocal(downloadUrl)
-      ? await this.readLocalFile(downloadUrl, fromBytes, toBytes)
-      : await this.remoteFileDownloader.download(downloadUrl, remoteHeaders, signal);
+      ? await this.withLocalFileContent(downloadUrl, ops, handler)
+      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
   }
 
-  async readLocalFile(
+  async withLocalFileContent<T>(
     url: string,
-    fromBytes?: number, // including this byte
-    toBytes?: number, // excluding this byte
-  ): Promise<DownloadResponse> {
+    ops: DownloadOps,
+    handler: ContentHandler<T>,
+  ): Promise<T> {
     const { storageId, relativePath } = parseLocalUrl(url);
     const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);
 
-    return {
-      content: Readable.toWeb(fs.createReadStream(fullPath, { start: fromBytes, end: toBytes !== undefined ? toBytes - 1 : undefined })),
-      size: (await fsp.stat(fullPath)).size,
-    };
+    return await this.localFileReadLimiter.run(async () => {
+      const readOps = {
+        start: ops.range?.from,
+        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,
+      };
+      let stream: fs.ReadStream | undefined;
+      let handlerSuccess = false;
+
+      try {
+        const stat = await fsp.stat(fullPath);
+        stream = fs.createReadStream(fullPath, readOps);
+        const webStream = Readable.toWeb(stream);
+
+        const result = await handler(webStream, stat.size);
+        handlerSuccess = true;
+        return result;
+      } catch (error) {
+        // Cleanup on error (including handler errors)
+        if (!handlerSuccess && stream && !stream.destroyed) {
+          stream.destroy();
+        }
+        throw error;
+      }
+    });
   }
 
   private async grpcGetDownloadUrl(

package/src/clients/upload.ts

@@ -8,7 +8,7 @@ import type { Dispatcher } from 'undici';
 import { request } from 'undici';
 import type { uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
 import { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';
-import { toHeadersMap } from './helpers';
+
 import type { IncomingHttpHeaders } from 'undici/types/header';
 
 export class MTimeError extends Error {
@@ -90,7 +90,7 @@ export class ClientUpload {
         // that's why we got big timeout here.
         headersTimeout: 60000,
         bodyTimeout: 60000,
-        headers: toHeadersMap(info.headers),
+        headers: Object.fromEntries(info.headers.map(({ name, value }) => [name, value])),
         method: info.method.toUpperCase() as Dispatcher.HttpMethod,
       });
 

package/src/drivers/download_blob/download_blob.ts

@@ -39,10 +39,11 @@ import * as fsp from 'node:fs/promises';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import * as readline from 'node:readline/promises';
-import { Readable, Writable } from 'node:stream';
+import { Writable } from 'node:stream';
 import { buffer } from 'node:stream/consumers';
 import type { ClientDownload } from '../../clients/download';
 import type { ClientLogs } from '../../clients/logs';
+import { readFileContent } from '../helpers/read_file';
 import {
   isLocalBlobHandle,
   newLocalHandle,
@@ -298,7 +299,7 @@ export class DownloadDriver implements BlobDriver {
     }
 
     if (isLocalBlobHandle(handle)) {
-      return await read(this.getLocalPath(handle), range);
+      return await readFileContent(this.getLocalPath(handle), range);
     }
 
     if (isRemoteBlobHandle(handle)) {
@@ -307,18 +308,15 @@ export class DownloadDriver implements BlobDriver {
       const key = blobKey(result.info.id);
       const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });
       if (filePath) {
-        return await read(filePath, range);
+        return await readFileContent(filePath, range);
       }
 
-      const { content } = await this.clientDownload.downloadBlob(
+      const data = await this.clientDownload.withBlobContent(
         { id: result.info.id, type: result.info.type },
         undefined,
-        undefined,
-        range?.from,
-        range?.to,
+        { range },
+        async (content) => await buffer(content)
       );
-
-      const data = await buffer(content);
       await this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data);
 
       return data;
@@ -657,42 +655,45 @@ class LastLinesGetter {
 
 /** Gets last lines from a file by reading the file from the top and keeping
  * last N lines in a window queue. */
-function getLastLines(fPath: string, nLines: number, patternToSearch?: string): Promise<string> {
-  const inStream = fs.createReadStream(fPath);
-  const outStream = new Writable();
+async function getLastLines(fPath: string, nLines: number, patternToSearch?: string): Promise<string> {
+  let inStream: fs.ReadStream | undefined;
+  let rl: readline.Interface | undefined;
 
-  return new Promise((resolve, reject) => {
-    const rl = readline.createInterface(inStream, outStream);
+  try {
+    inStream = fs.createReadStream(fPath);
+    rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });
 
     const lines = new Denque();
-    rl.on('line', function (line) {
-      if (patternToSearch != undefined && !line.includes(patternToSearch)) return;
+
+    for await (const line of rl) {
+      if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;
 
       lines.push(line);
       if (lines.length > nLines) {
         lines.shift();
       }
-    });
-
-    rl.on('error', reject);
+    }
 
-    rl.on('close', function () {
-      // last EOL is for keeping backward compat with platforma implementation.
-      resolve(lines.toArray().join(os.EOL) + os.EOL);
-    });
-  });
-}
+    // last EOL is for keeping backward compat with platforma implementation.
+    return lines.toArray().join(os.EOL) + os.EOL;
+  } finally {
+    // Cleanup resources in finally block to ensure they're always cleaned up
+    try {
+      if (rl) {
+        rl.close();
+      }
+    } catch (cleanupError) {
+      console.error('Error closing readline interface:', cleanupError);
+    }
 
-async function read(path: string, range?: RangeBytes): Promise<Uint8Array> {
-  const ops: { start?: number; end?: number } = {};
-  if (range) {
-    ops.start = range.from;
-    ops.end = range.to - 1;
+    try {
+      if (inStream && !inStream.destroyed) {
+        inStream.destroy();
+      }
+    } catch (cleanupError) {
+      console.error('Error destroying read stream:', cleanupError);
+    }
   }
-
-  const stream = fs.createReadStream(path, ops);
-
-  return await buffer(Readable.toWeb(stream));
 }
 
 function validateDownloadableResourceType(methodName: string, rType: ResourceType) {

package/src/drivers/download_blob/download_blob_task.ts

@@ -94,23 +94,26 @@ export class DownloadBlobTask {
       return this.state.fileSize;
     }
 
-    const { content, size } = await this.clientDownload.downloadBlob(
+    const fileSize = await this.clientDownload.withBlobContent(
       this.rInfo,
       {},
-      this.signalCtl.signal,
+      { signal: this.signalCtl.signal },
+      async (content, size) => {
+        this.state.fileSize = size;
+        this.state.downloaded = true;
+
+        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {
+          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));
+          await content.pipeTo(f, { signal: this.signalCtl.signal });
+          this.state.tempWritten = true;
+        });
+
+        this.state.done = true;
+        return size;
+      }
     );
-    this.state.fileSize = size;
-    this.state.downloaded = true;
-
-    await createPathAtomically(this.logger, this.state.filePath, async (fPath: string) => {
-      const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));
-      await content.pipeTo(f, { signal: this.signalCtl.signal });
-      this.state.tempWritten = true;
-    });
-
-    this.state.done = true;
 
-    return size;
+    return fileSize;
   }
 
   public abort(reason: string) {

package/src/drivers/download_blob_url/task.ts

@@ -95,59 +95,64 @@ export class DownloadAndUnarchiveTask {
       return await dirSize(this.path);
     }
 
-    const { content, size } = await this.clientDownload.downloadBlob(
-      this.rInfo, {}, signal,
-    );
-    this.state.downloaded = true;
-
-    await createPathAtomically(this.logger, this.path, async (fPath: string) => {
-      this.state!.tempPath = fPath;
-      this.state!.archiveFormat = this.format;
-
-      switch (this.format) {
-        case 'tar':
-          await fsp.mkdir(fPath); // throws if a directory already exists.
-          const simpleUntar = Writable.toWeb(tar.extract(fPath));
-          await content.pipeTo(simpleUntar, { signal });
-          return;
-
-        case 'tgz':
-          await fsp.mkdir(fPath); // throws if a directory already exists.
-          const gunzip = Transform.toWeb(zlib.createGunzip());
-          const untar = Writable.toWeb(tar.extract(fPath));
-
-          await content
-            .pipeThrough(gunzip, { signal })
-            .pipeTo(untar, { signal });
-          return;
-
-        case 'zip':
-          this.state!.zipPath = this.path + '.zip';
-
-          const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));
-          await content.pipeTo(f, { signal });
-          this.state!.zipPathCreated = true;
-
-          // Without this filter it fails with
-          // "EISDIR: illegal operation on a directory".
-          // The workaround is from
-          // https://github.com/kevva/decompress/issues/46#issuecomment-525048104
-          await decompress(this.state!.zipPath, fPath, {
-            filter: file => !file.path.endsWith('/'),
-          });
-          this.state!.zipDecompressed = true;
-
-          await fs.promises.rm(this.state!.zipPath);
-          this.state!.zipPathDeleted = true;
-
-          return;
-
-        default:
-          assertNever(this.format);
+    const size = await this.clientDownload.withBlobContent(
+      this.rInfo, 
+      {}, 
+      { signal },
+      async (content, size) => {
+        this.state!.downloaded = true;
+
+        await createPathAtomically(this.logger, this.path, async (fPath: string) => {
+          this.state!.tempPath = fPath;
+          this.state!.archiveFormat = this.format;
+
+          switch (this.format) {
+            case 'tar':
+              await fsp.mkdir(fPath); // throws if a directory already exists.
+              const simpleUntar = Writable.toWeb(tar.extract(fPath));
+              await content.pipeTo(simpleUntar, { signal });
+              return;
+
+            case 'tgz':
+              await fsp.mkdir(fPath); // throws if a directory already exists.
+              const gunzip = Transform.toWeb(zlib.createGunzip());
+              const untar = Writable.toWeb(tar.extract(fPath));
+
+              await content
+                .pipeThrough(gunzip, { signal })
+                .pipeTo(untar, { signal });
+              return;
+
+            case 'zip':
+              this.state!.zipPath = this.path + '.zip';
+
+              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));
+              await content.pipeTo(f, { signal });
+              this.state!.zipPathCreated = true;
+
+              // Without this filter it fails with
+              // "EISDIR: illegal operation on a directory".
+              // The workaround is from
+              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104
+              await decompress(this.state!.zipPath, fPath, {
+                filter: file => !file.path.endsWith('/'),
+              });
+              this.state!.zipDecompressed = true;
+
+              await fs.promises.rm(this.state!.zipPath);
+              this.state!.zipPathDeleted = true;
+
+              return;
+
+            default:
+              assertNever(this.format);
+          }
+        });
+
+        this.state!.pathCreated = true;
+        return size;
       }
-    });
-
-    this.state.pathCreated = true;
+    );
 
     return size;
   }

package/src/drivers/download_url/task.ts

@@ -82,21 +82,28 @@ export class DownloadByUrlTask {
       return await dirSize(this.path);
     }
 
-    const resp = await clientDownload.download(this.url.toString(), {}, signal);
-
-    let content = resp.content;
-    if (withGunzip) {
-      const gunzip = Transform.toWeb(zlib.createGunzip());
-      content = content.pipeThrough(gunzip, { signal });
-    }
-
-    await createPathAtomically(this.logger, this.path, async (fPath: string) => {
-      await fsp.mkdir(fPath); // throws if a directory already exists.
-      const untar = Writable.toWeb(tar.extract(fPath));
-      await content.pipeTo(untar, { signal });
-    });
+    const size = await clientDownload.withContent(
+      this.url.toString(), 
+      {}, 
+      { signal },
+      async (content, size) => {
+        let processedContent = content;
+        if (withGunzip) {
+          const gunzip = Transform.toWeb(zlib.createGunzip());
+          processedContent = content.pipeThrough(gunzip, { signal });
+        }
+
+        await createPathAtomically(this.logger, this.path, async (fPath: string) => {
+          await fsp.mkdir(fPath); // throws if a directory already exists.
+          const untar = Writable.toWeb(tar.extract(fPath));
+          await processedContent.pipeTo(untar, { signal });
+        });
+
+        return size;
+      }
+    );
 
-    return resp.size;
+    return size;
   }
 
   getUrl(): UrlResult | undefined {

package/src/drivers/helpers/read_file.ts

@@ -0,0 +1,32 @@
+import { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
+import type { RangeBytes } from '@milaboratories/pl-model-common';
+import * as fs from 'node:fs';
+import { buffer } from 'node:stream/consumers';
+
+// Global concurrency limiter for file reads - limit to 32 parallel reads
+const fileReadLimiter = new ConcurrencyLimitingExecutor(32);
+
+/**
+ * Reads file content with concurrency limiting and proper error handling.
+ * Ensures file descriptors are properly cleaned up even in error cases.
+ */
+export async function readFileContent(path: string, range?: RangeBytes): Promise<Uint8Array> {
+  return await fileReadLimiter.run(async () => {
+    const ops: { start?: number; end?: number } = {};
+    if (range) {
+      ops.start = range.from;
+      ops.end = range.to - 1;
+    }
+
+    let stream: fs.ReadStream | undefined;
+    try {
+      stream = fs.createReadStream(path, ops);
+      return await buffer(stream);
+    } catch (error) {
+      if (stream && !stream.destroyed) {
+        stream.destroy();
+      }
+      throw error;
+    }
+  });
+}

package/src/helpers/download.ts

@@ -5,12 +5,15 @@ import { request } from 'undici';
 import { Readable } from 'node:stream';
 import type { ReadableStream } from 'node:stream/web';
 import { text } from 'node:stream/consumers';
+import type { RangeBytes } from '@milaboratories/pl-model-common';
 
-export interface DownloadResponse {
-  content: ReadableStream;
-  size: number;
+export interface DownloadOps {
+  signal?: AbortSignal;
+  range?: RangeBytes;
 }
 
+export type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;
+
 /** Throws when a status code of the downloading URL was in range [400, 500). */
 export class NetworkError400 extends Error {
   name = 'NetworkError400';
@@ -19,24 +22,45 @@ export class NetworkError400 extends Error {
 export class RemoteFileDownloader {
   constructor(public readonly httpClient: Dispatcher) {}
 
-  async download(
+  async withContent<T>(
     url: string,
     reqHeaders: Record<string, string>,
-    signal?: AbortSignal,
-  ): Promise<DownloadResponse> {
-    const { statusCode, body, headers } = await request(url, {
+    ops: DownloadOps,
+    handler: ContentHandler<T>,
+  ): Promise<T> {
+    const headers = { ...reqHeaders };
+
+    // Add range header if specified
+    if (ops.range) {
+      headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - 1}`;
+    }
+
+    const { statusCode, body, headers: responseHeaders } = await request(url, {
       dispatcher: this.httpClient,
-      headers: reqHeaders,
-      signal,
+      headers,
+      signal: ops.signal,
     });
 
     const webBody = Readable.toWeb(body);
-    await checkStatusCodeOk(statusCode, webBody, url);
+    let handlerSuccess = false;
 
-    return {
-      content: webBody,
-      size: Number(headers['content-length']),
-    };
+    try {
+      await checkStatusCodeOk(statusCode, webBody, url);
+      const size = Number(responseHeaders['content-length']);
+      const result = await handler(webBody, size);
+      handlerSuccess = true;
+      return result;
+    } catch (error) {
+      // Cleanup on error (including handler errors)
+      if (!handlerSuccess && !webBody.locked) {
+        try {
+          await webBody.cancel();
+        } catch {
+          // Ignore cleanup errors
+        }
+      }
+      throw error;
+    }
   }
 }
 

package/dist/clients/helpers.d.ts

@@ -1,6 +0,0 @@
-export declare function toHeadersMap(headers: {
-    name: string;
-    value: string;
-}[], fromBytes?: number, // including this byte
-toBytes?: number): Record<string, string>;
-//# sourceMappingURL=helpers.d.ts.map

package/dist/clients/helpers.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/clients/helpers.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAC1B,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,EAC1C,SAAS,CAAC,EAAE,MAAM,EAAE,sBAAsB;AAC1C,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOxB"}

package/src/clients/helpers.ts

@@ -1,12 +0,0 @@
-export function toHeadersMap(
-  headers: { name: string; value: string }[],
-  fromBytes?: number, // including this byte
-  toBytes?: number, // excluding this byte
-): Record<string, string> {
-  const result = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
-  if (fromBytes !== undefined && toBytes !== undefined) {
-    result['Range'] = `bytes=${fromBytes}-${toBytes - 1}`;
-  }
-
-  return result;
-}