@milaboratories/pl-drivers 1.5.4 → 1.5.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.5.4",
+  "version": "1.5.6",
   "description": "Drivers and a low-level clients for log streaming, downloading and uploading files from and to pl",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -28,9 +28,9 @@
     "zod": "~3.23.8",
     "@milaboratories/ts-helpers": "^1.1.3",
     "@milaboratories/computable": "^2.3.4",
+    "@milaboratories/pl-client": "^2.7.1",
     "@milaboratories/pl-tree": "^1.4.20",
-    "@milaboratories/pl-model-common": "^1.10.1",
-    "@milaboratories/pl-client": "^2.7.1"
+    "@milaboratories/pl-model-common": "^1.10.2"
   },
   "devDependencies": {
     "typescript": "~5.5.4",

package/src/drivers/download_blob.test.ts

@@ -20,7 +20,7 @@ import { OnDemandBlobResourceSnapshot } from './types';
 
 const fileName = 'answer_to_the_ultimate_question.txt';
 
-test.skip('should download a blob and read its content', async () => {
+test('should download a blob and read its content', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-1-'));
@@ -49,7 +49,55 @@ test.skip('should download a blob and read its content', async () => {
   });
 });
 
-test.skip('should get on demand blob without downloading a blob', async () => {
+test('should not redownload a blob a file already exists', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-1-'));
+
+    const signer = new HmacSha256Signer(HmacSha256Signer.generateSecret());
+
+    const driver = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client, []),
+      createLogsClient(client, logger),
+      dir,
+      signer,
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+
+    console.log('Download the first time');
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+    const c = driver.getDownloadedBlob(downloadable);
+    await c.getValue();
+    await c.awaitChange();
+    const blob = await c.getValue();
+    expect(blob).toBeDefined();
+    expect(blob!.size).toBe(3);
+    expect((await driver.getContent(blob!.handle))?.toString()).toBe('42\n');
+
+    await driver.releaseAll();
+
+    const driver2 = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client, []),
+      createLogsClient(client, logger),
+      dir,
+      signer,
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+
+    console.log('Download the second time');
+    const c2 = driver2.getDownloadedBlob(downloadable);
+    await c2.getValue();
+    await c2.awaitChange();
+    const blob2 = await c2.getValue();
+    expect(blob2).toBeDefined();
+    expect(blob2!.size).toBe(3);
+    expect((await driver.getContent(blob2!.handle))?.toString()).toBe('42\n');
+  });
+});
+
+test('should get on demand blob without downloading a blob', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-2-'));
@@ -74,7 +122,7 @@ test.skip('should get on demand blob without downloading a blob', async () => {
   });
 });
 
-test.skip('should get undefined when releasing a blob from a small cache and the blob was deleted.', async () => {
+test('should get undefined when releasing a blob from a small cache and the blob was deleted.', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-3-'));
@@ -111,7 +159,7 @@ test.skip('should get undefined when releasing a blob from a small cache and the
   });
 });
 
-test.skip('should get the blob when releasing a blob, but a cache is big enough and it keeps a file on the local drive.', async () => {
+test('should get the blob when releasing a blob, but a cache is big enough and it keeps a file on the local drive.', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-4-'));

package/src/drivers/download_blob.ts

@@ -5,7 +5,7 @@ import {
   ComputableStableDefined,
   Watcher
 } from '@milaboratories/computable';
-import { ResourceId, ResourceType } from '@milaboratories/pl-client';
+import { ResourceId, ResourceType, stringifyWithResourceId } from '@milaboratories/pl-client';
 import {
   AnyLogHandle,
   BlobDriver,
@@ -435,8 +435,8 @@ export class DownloadDriver implements BlobDriver {
 
           this.removeTask(
             task,
-            `the task ${task.path} was removed` +
-              `from cache along with ${toDelete.map((d) => d.path)}`
+            `the task ${stringifyWithResourceId(task.info())} was removed` +
+              `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.info()))}`
           );
         })
       );
@@ -444,7 +444,10 @@ export class DownloadDriver implements BlobDriver {
       // The task is still in a downloading queue.
       const deleted = task.counter.dec(callerId);
       if (deleted) {
-        this.removeTask(task, `the task ${task.path} was removed from cache`);
+        this.removeTask(
+          task,
+          `the task ${stringifyWithResourceId(task.info())} was removed from cache`
+        );
       }
     }
   }

package/src/drivers/download_blob_task.ts

@@ -34,6 +34,17 @@ export class DownloadBlobTask {
     private readonly handle: LocalBlobHandle
   ) {}
 
+  /** Returns a simple object that describes this task. */
+  public info() {
+    return {
+      rInfo: this.rInfo,
+      path: this.path,
+      done: this.done,
+      size: this.size,
+      error: this.error
+    };
+  }
+
   public attach(w: Watcher, callerId: string) {
     this.counter.inc(callerId);
     if (!this.done) this.change.attachWatcher(w);
@@ -41,18 +52,7 @@ export class DownloadBlobTask {
 
   public async download() {
     try {
-      await ensureDirExists(path.dirname(this.path));
-      const { content, size } = await this.clientDownload.downloadBlob(this.rInfo);
-
-      if (await fileExists(this.path)) {
-        await content.cancel(`the file already exists.`); // finalize body
-      } else {
-        await createPathAtomically(this.logger, this.path, async (fPath: string) => {
-          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));
-          await content.pipeTo(f);
-        });
-      }
-
+      const size = await this.ensureDownloaded();
       this.setDone(size);
       this.change.markChanged();
     } catch (e: any) {
@@ -67,6 +67,25 @@ export class DownloadBlobTask {
     }
   }
 
+  private async ensureDownloaded() {
+    await ensureDirExists(path.dirname(this.path));
+
+    if (await fileExists(this.path)) {
+      this.logger.info(`a blob was already downloaded: ${this.path}`);
+      const stat = await fsp.stat(this.path);
+      return stat.size;
+    }
+
+    const { content, size } = await this.clientDownload.downloadBlob(this.rInfo);
+
+    await createPathAtomically(this.logger, this.path, async (fPath: string) => {
+      const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));
+      await content.pipeTo(f);
+    });
+
+    return size;
+  }
+
   public abort(reason: string) {
     this.signalCtl.abort(new DownloadAborted(reason));
   }

package/src/drivers/download_url.ts

@@ -17,6 +17,8 @@ import * as tar from 'tar-fs';
 import { Dispatcher } from 'undici';
 import { NetworkError400, RemoteFileDownloader } from '../helpers/download';
 import { FilesCache } from './helpers/files_cache';
+import { stringifyWithResourceId } from '@milaboratories/pl-client';
+import { DownloadBlobTask } from './download_blob_task';
 
 export interface DownloadUrlSyncReader {
   /** Returns a Computable that (when the time will come)
@@ -128,21 +130,25 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
       const toDelete = this.cache.removeFile(task.path, callerId);
 
       await Promise.all(
-        toDelete.map(async (task) => {
+        toDelete.map(async (task: DownloadByUrlTask) => {
           await rmRFDir(task.path);
           this.cache.removeCache(task);
 
           this.removeTask(
             task,
-            `the task ${JSON.stringify(task)} was removed` +
-              `from cache along with ${JSON.stringify(toDelete)}`
+            `the task ${stringifyWithResourceId(task.info())} was removed` +
+              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`
           );
         })
       );
     } else {
       // The task is still in a downloading queue.
       const deleted = task.counter.dec(callerId);
-      if (deleted) this.removeTask(task, `the task ${JSON.stringify(task)} was removed from cache`);
+      if (deleted)
+        this.removeTask(
+          task,
+          `the task ${stringifyWithResourceId(task.info())} was removed from cache`
+        );
     }
   }
 
@@ -155,7 +161,10 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
         await rmRFDir(task.path);
         this.cache.removeCache(task);
 
-        this.removeTask(task, `the task ${task} was released when the driver was closed`);
+        this.removeTask(
+          task,
+          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`
+        );
       })
     );
   }
@@ -195,6 +204,16 @@ class DownloadByUrlTask {
     readonly url: URL
   ) {}
 
+  public info() {
+    return {
+      url: this.url.toString(),
+      path: this.path,
+      done: this.done,
+      size: this.size,
+      error: this.error
+    };
+  }
+
   attach(w: Watcher, callerId: string) {
     this.counter.inc(callerId);
     if (!this.done) this.change.attachWatcher(w);