@milaboratories/pl-drivers 1.3.6 → 1.3.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.3.6",
+  "version": "1.3.8",
   "description": "Drivers and a low-level clients for log streaming, downloading and uploading files from and to pl",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -27,16 +27,16 @@
     "undici": "^6.20.1",
     "zod": "^3.23.8",
     "@milaboratories/ts-helpers": "^1.1.0",
-    "@milaboratories/pl-tree": "^1.4.6",
-    "@milaboratories/computable": "^2.2.0",
     "@milaboratories/pl-client": "^2.5.5",
-    "@milaboratories/pl-model-common": "^1.5.1"
+    "@milaboratories/computable": "^2.2.0",
+    "@milaboratories/pl-tree": "^1.4.6",
+    "@milaboratories/pl-model-common": "^1.5.2"
   },
   "devDependencies": {
     "typescript": "~5.5.4",
     "vite": "^5.4.10",
-    "@types/node": "~20.16.10",
-    "@types/jest": "^29.5.13",
+    "@types/node": "~20.16.15",
+    "@types/jest": "^29.5.14",
     "@types/tar-fs": "^2.0.4",
     "jest": "^29.7.0",
     "@jest/globals": "^29.7.0",

package/src/drivers/logs.test.ts

@@ -22,6 +22,7 @@ import { DownloadDriver } from './download_and_logs_blob';
 import { createDownloadClient, createLogsClient } from '../clients/helpers';
 import { LogsStreamDriver } from './logs_stream';
 import { LogsDriver } from './logs';
+import { test, expect } from '@jest/globals';
 
 test('should get all logs', async () => {
   await TestHelpers.withTempRoot(async (client) => {
@@ -31,7 +32,7 @@ test('should get all logs', async () => {
       stopPollingDelay: 10,
       pollingInterval: 10
     });
-    const logsStream = new LogsStreamDriver(createLogsClient(client, logger));
+    const logsStream = new LogsStreamDriver(logger, createLogsClient(client, logger));
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-1-'));
     const download = new DownloadDriver(
       logger,
@@ -41,7 +42,7 @@ test('should get all logs', async () => {
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
       { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
     );
-    const logs = new LogsDriver(logsStream, download);
+    const logs = new LogsDriver(logger, logsStream, download);
 
     await createRunCommandWithStdoutStream(client, 'bash', ['-c', 'echo 1; sleep 1; echo 2']);
 
@@ -76,7 +77,7 @@ test('should get last line with a prefix', async () => {
       stopPollingDelay: 10,
       pollingInterval: 10
     });
-    const logsStream = new LogsStreamDriver(createLogsClient(client, logger));
+    const logsStream = new LogsStreamDriver(logger, createLogsClient(client, logger));
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-2-'));
     const download = new DownloadDriver(
       logger,
@@ -86,7 +87,7 @@ test('should get last line with a prefix', async () => {
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
       { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
     );
-    const logs = new LogsDriver(logsStream, download);
+    const logs = new LogsDriver(logger, logsStream, download);
 
     const c = Computable.make((ctx) => {
       const streamManager = ctx.accessor(tree.entry()).node().traverse('result')?.persist();
@@ -126,7 +127,7 @@ test('should get log smart object and get log lines from that', async () => {
       stopPollingDelay: 10,
       pollingInterval: 10
     });
-    const logsStream = new LogsStreamDriver(createLogsClient(client, logger));
+    const logsStream = new LogsStreamDriver(logger, createLogsClient(client, logger));
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-3-'));
     const download = new DownloadDriver(
       logger,
@@ -136,7 +137,7 @@ test('should get log smart object and get log lines from that', async () => {
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
       { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
     );
-    const logs = new LogsDriver(logsStream, download);
+    const logs = new LogsDriver(logger, logsStream, download);
 
     const c = Computable.make((ctx) => {
       const streamManager = ctx.accessor(tree.entry()).node().traverse('result')?.persist();

package/src/drivers/logs.ts

@@ -1,12 +1,14 @@
 import { Computable, ComputableCtx } from '@milaboratories/computable';
 import { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';
-import { bigintToResourceId } from '@milaboratories/pl-client';
+import { bigintToResourceId, stringifyWithResourceId } from '@milaboratories/pl-client';
 import { LogsStreamDriver } from './logs_stream';
 import { DownloadDriver } from './download_and_logs_blob';
 import * as sdk from '@milaboratories/pl-model-common';
+import { ConsoleLoggerAdapter, MiLogger } from '@milaboratories/ts-helpers';
 
 export class LogsDriver implements sdk.LogsDriver {
   constructor(
+    private readonly logger: MiLogger,
     private readonly logsStreamDriver: LogsStreamDriver,
     private readonly downloadDriver: DownloadDriver
   ) {}

package/src/drivers/logs_stream.ts

@@ -5,8 +5,14 @@ import {
   PollingComputableHooks,
   Watcher
 } from '@milaboratories/computable';
-import { ResourceId } from '@milaboratories/pl-client';
-import { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';
+import { ResourceId, stringifyWithResourceId } from '@milaboratories/pl-client';
+import {
+  asyncPool,
+  CallersCounter,
+  ConsoleLoggerAdapter,
+  MiLogger,
+  ValueOrError
+} from '@milaboratories/ts-helpers';
 import { ClientLogs } from '../clients/logs';
 import { randomUUID } from 'node:crypto';
 import { PlTreeEntry, ResourceInfo, treeEntryToResourceInfo } from '@milaboratories/pl-tree';
@@ -15,6 +21,7 @@ import { scheduler } from 'node:timers/promises';
 import { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';
 import * as sdk from '@milaboratories/pl-model-common';
 import { PollingOps } from './helpers/polling_ops';
+import { RpcError } from '@protobuf-ts/runtime-rpc';
 
 export type LogsStreamDriverOps = PollingOps & {
   /** Max number of concurrent requests to log streaming backend while calculating computable states */
@@ -32,6 +39,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
   private readonly hooks: PollingComputableHooks;
 
   constructor(
+    private readonly logger: MiLogger,
     private readonly clientLogs: ClientLogs,
     private readonly opts: LogsStreamDriverOps = {
       nConcurrentGetLogs: 10,
@@ -82,7 +90,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     let logGetter = this.idToLastLines.get(rInfo.id);
 
     if (logGetter == undefined) {
-      const newLogGetter = new LogGetter(this.clientLogs, rInfo, lines);
+      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);
       this.idToLastLines.set(rInfo.id, newLogGetter);
 
       logGetter = newLogGetter;
@@ -120,7 +128,6 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));
 
     const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);
-
     ctx.markUnstable(
       'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.'
     );
@@ -137,7 +144,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     let logGetter = this.idToProgressLog.get(rInfo.id);
 
     if (logGetter == undefined) {
-      const newLogGetter = new LogGetter(this.clientLogs, rInfo, 1, patternToSearch);
+      const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);
       this.idToProgressLog.set(rInfo.id, newLogGetter);
 
       logGetter = newLogGetter;
@@ -264,9 +271,10 @@ export class LogsStreamDriver implements sdk.LogsDriver {
       this.scheduledOnNextState = [];
 
       try {
+        const logs = this.getAllLogs();
         await asyncPool(
           this.opts.nConcurrentGetLogs,
-          this.getAllNotDoneLogs().map((getter) => async () => await getter.update())
+          logs.map((getter) => async () => await getter.update())
         );
 
         toNotify.forEach((n) => n.resolve());
@@ -282,10 +290,9 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     this.currentLoop = undefined;
   }
 
-  private getAllNotDoneLogs(): Array<LogGetter> {
+  private getAllLogs(): Array<LogGetter> {
     return Array.from(this.idToLastLines.entries())
       .concat(Array.from(this.idToProgressLog.entries()))
-      .filter(([_, getter]) => !getter.getLog().done)
       .map(([_, getter]) => getter);
   }
 }
@@ -294,12 +301,12 @@ export class LogsStreamDriver implements sdk.LogsDriver {
 class LogGetter {
   private logs: string | undefined;
   private error: any | undefined = undefined;
-  private done = false;
 
   private readonly change: ChangeSource = new ChangeSource();
   private readonly counter: CallersCounter = new CallersCounter();
 
   constructor(
+    private readonly logger: MiLogger,
     private readonly clientLogs: ClientLogs,
     private readonly rInfo: ResourceInfo,
     private readonly lines: number,
@@ -309,12 +316,10 @@ class LogGetter {
   getLog(): {
     log: string | undefined;
     error?: any | undefined;
-    done: boolean;
   } {
     return {
       log: this.logs,
-      error: this.error,
-      done: this.done
+      error: this.error
     };
   }
 
@@ -340,18 +345,22 @@ class LogGetter {
 
       if (this.logs != newLogs) this.change.markChanged();
       this.logs = newLogs;
+      this.error = undefined;
 
       return;
     } catch (e: any) {
+      e as RpcError;
       if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
         // No resource
         this.logs = '';
         this.error = e;
-        this.done = true;
         this.change.markChanged();
         return;
       }
 
+      this.logger.error(
+        `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`
+      );
       throw e;
     }
   }

package/src/drivers/ls.test.ts

@@ -15,15 +15,9 @@ test('should ok when get all storages from ls driver', async () => {
   const logger = new ConsoleLoggerAdapter();
   await TestHelpers.withTempRoot(async (client) => {
     const lsClient = createLsFilesClient(client, logger);
-    const driver = await LsDriver.init(
-      logger,
-      client,
-      signer,
-      [],
-      () => {
-        throw Error();
-      }
-    );
+    const driver = await LsDriver.init(logger, client, signer, [], () => {
+      throw Error();
+    });
 
     const got = await driver.getStorageList();
 
@@ -41,15 +35,9 @@ test('should ok when list files from remote storage in ls driver', async () => {
   const signer = new HmacSha256Signer('abc');
   const logger = new ConsoleLoggerAdapter();
   await TestHelpers.withTempRoot(async (client) => {
-    const driver = await LsDriver.init(
-      logger,
-      client,
-      signer,
-      [],
-      () => {
-        throw Error();
-      }
-    );
+    const driver = await LsDriver.init(logger, client, signer, [], () => {
+      throw Error();
+    });
 
     const storages = await driver.getStorageList();
     const library = storages.find((se) => se.name == 'library')!.handle;
@@ -82,15 +70,9 @@ test('should ok when list files from local storage in ls driver', async () => {
   const signer = new HmacSha256Signer('abc');
   const logger = new ConsoleLoggerAdapter();
   await TestHelpers.withTempRoot(async (client) => {
-    const driver = await LsDriver.init(
-      logger,
-      client,
-      signer,
-      [],
-      () => {
-        throw Error();
-      }
-    );
+    const driver = await LsDriver.init(logger, client, signer, [], () => {
+      throw Error();
+    });
 
     const storages = await driver.getStorageList();
     const local = storages.find((se) => se.name == 'local')!;
@@ -146,13 +128,9 @@ test('should ok when get file using local dialog, and read its content', async (
   const signer = new HmacSha256Signer('abc');
   const logger = new ConsoleLoggerAdapter();
   await TestHelpers.withTempRoot(async (client) => {
-    const driver = await LsDriver.init(
-      logger,
-      client,
-      signer,
-      [],
-      async () => [path.join(assetsPath, 'answer_to_the_ultimate_question.txt')]
-    );
+    const driver = await LsDriver.init(logger, client, signer, [], async () => [
+      path.join(assetsPath, 'answer_to_the_ultimate_question.txt')
+    ]);
 
     const result = await driver.showOpenSingleFileDialog();
     expect(result.file).toBeDefined();

package/src/drivers/virtual_storages.ts

@@ -36,14 +36,16 @@ export async function DefaultVirtualLocalStorages(): Promise<VirtualLocalStorage
           name: `local_disk_${drive}`,
           root: `${drive}:\\`,
           initialPath: isHomeDrive ? home : `${drive}:\\`
-        }
+        };
       });
     } catch (e: any) {
-      return [{
-        name: `local_disk_${homeDrive}`,
-        root: `${homeDrive}:\\`,
-        initialPath: home
-      }]
+      return [
+        {
+          name: `local_disk_${homeDrive}`,
+          root: `${homeDrive}:\\`,
+          initialPath: home
+        }
+      ];
     }
   }
 }