@milaboratories/pl-drivers 1.5.64 → 1.5.65

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.5.64",
+  "version": "1.5.65",
   "engines": {
     "node": ">=20"
   },
@@ -31,17 +31,18 @@
     "undici": "~7.5.0",
     "zod": "~3.23.8",
     "upath": "^2.0.1",
-    "@milaboratories/ts-helpers": "^1.3.1",
-    "@milaboratories/computable": "^2.4.10",
-    "@milaboratories/pl-client": "^2.10.0",
-    "@milaboratories/pl-tree": "^1.6.6",
-    "@milaboratories/pl-model-common": "^1.15.2"
+    "@milaboratories/helpers": "^1.6.12",
+    "@milaboratories/ts-helpers": "^1.3.2",
+    "@milaboratories/pl-client": "^2.10.1",
+    "@milaboratories/computable": "^2.4.11",
+    "@milaboratories/pl-tree": "^1.6.7",
+    "@milaboratories/pl-model-common": "^1.15.3"
   },
   "devDependencies": {
     "eslint": "^9.25.1",
     "@types/decompress": "^4.2.7",
     "typescript": "~5.5.4",
-    "vite": "^5.4.11",
+    "vite": "^6.3.5",
     "@types/node": "~20.16.15",
     "vitest": "^2.1.9",
     "@vitest/coverage-v8": "^2.1.9",

package/src/drivers/download_blob/download_blob.test.ts

@@ -4,7 +4,8 @@ import type {
   FieldRef,
   PlClient,
   PlTransaction,
-  PollTxAccessor } from '@milaboratories/pl-client';
+  PollTxAccessor
+} from '@milaboratories/pl-client';
 import {
   jsonToData,
   poll,
@@ -22,10 +23,12 @@ import * as env from '../../test_env';
 
 const fileName = 'answer_to_the_ultimate_question.txt';
 
-test('should download a blob and read its content', { timeout: 10000 },  async () => {
+test('should download a blob and read its content', { timeout: 10000 }, async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
-    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-ranges'));
+
+    const driver = await genDriver(client, dir, rangesCacheDir, genSigner());
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
     const c = driver.getDownloadedBlob(downloadable);
@@ -50,7 +53,9 @@ test('should download a blob and read its content', { timeout: 10000 },  async (
 test('should download a blob range and read its content with a range', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
-    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-ranges'));
+
+    const driver = await genDriver(client, dir, rangesCacheDir, genSigner());
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
     const c = driver.getDownloadedBlob(downloadable);
@@ -75,8 +80,10 @@ test('should download a blob range and read its content with a range', async ()
 test('should not redownload a blob when a file already exists', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-ranges'));
     const signer = genSigner();
-    const driver = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+
+    const driver = await genDriver(client, dir, rangesCacheDir, signer);
 
     console.log('Download the first time');
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
@@ -88,7 +95,7 @@ test('should not redownload a blob when a file already exists', async () => {
 
     await driver.releaseAll();
 
-    const driver2 = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+    const driver2 = await genDriver(client, dir, rangesCacheDir, signer);
 
     console.log('Download the second time');
     const c2 = driver2.getDownloadedBlob(downloadable);
@@ -104,7 +111,9 @@ test('should not redownload a blob when a file already exists', async () => {
 test('should get on demand blob without downloading a blob', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-2-'));
-    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-ranges'));
+
+    const driver = await genDriver(client, dir, rangesCacheDir, genSigner());
 
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
     const c = driver.getOnDemandBlob(downloadable);
@@ -121,7 +130,9 @@ test('should get on demand blob without downloading a blob', async () => {
 test('should get on demand blob without downloading a blob range', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-2-'));
-    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-ranges'));
+
+    const driver = await genDriver(client, dir, rangesCacheDir, genSigner());
 
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
@@ -151,7 +162,13 @@ test('should get on demand blob without downloading a blob range', async () => {
 test('should get undefined when releasing a blob from a small cache and the blob was deleted.', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-3-'));
-    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 });
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-ranges'));
+
+    const driver = await genDriver(client, dir, rangesCacheDir, genSigner(), {
+      cacheSoftSizeBytes: 1,
+      nConcurrentDownloads: 10,
+      rangesCacheMaxSizeBytes: 1024,
+    });
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
     const c = driver.getDownloadedBlob(downloadable);
@@ -180,7 +197,13 @@ test('should get undefined when releasing a blob from a small cache and the blob
 test('should get undefined when releasing a blob from a small cache and the blob was deleted range.', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
-    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 });
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-ranges'));
+
+    const driver = await genDriver(client, dir, rangesCacheDir, genSigner(), {
+      cacheSoftSizeBytes: 1,
+      nConcurrentDownloads: 10,
+      rangesCacheMaxSizeBytes: 1024,
+    });
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
     const c = driver.getDownloadedBlob(downloadable);
@@ -206,49 +229,33 @@ test('should get undefined when releasing a blob from a small cache and the blob
   });
 });
 
-// test('should get the blob range after init if it already existed.', async () => {
-//   await TestHelpers.withTempRoot(async (client) => {
-//     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
-//     const signer = genSigner();
-//     const driver = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 720, nConcurrentDownloads: 10 });
-//     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
-
-//     const c = driver.getDownloadedBlob(downloadable);
-//     const blob = await c.getValue();
-//     expect(blob).toBeUndefined();
-//     await c.awaitChange();
-
-//     const blob2 = await c.getValue();
-//     expect(blob2).toBeDefined();
-//     expect(blob2!.size).toBe(3);
-//     expect((await driver.getContent(blob2!.handle, { from: 1, to: 3 }))?.toString()).toBe('2\n');
-
-//     // Make the second driver, the cache was already initialized.
-//     // We should get the blob instantly.
-//     const driver2 = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 720, nConcurrentDownloads: 10 });
-//     const c2 = driver2.getDownloadedBlob(downloadable);
-//     const blob3 = await c2.getValue();
-//     expect(blob3).toBeDefined();
-//     expect(blob3!.size).toBe(2);
-//     expect((await driver.getContent(blob3!.handle, { from: 1, to: 3 }))?.toString()).toBe('2\n');
-//   });
-// })
-;
-
 function genSigner() {
   return new HmacSha256Signer(HmacSha256Signer.generateSecret())
 }
 
-async function genDriver(client: PlClient, dir: string, signer: Signer, ops: DownloadDriverOps) {
+async function genDriver(
+  client: PlClient,
+  dir: string,
+  rangesCacheDir: string,
+  signer: Signer,
+  ops?: Partial<DownloadDriverOps>,
+) {
   const logger = new ConsoleLoggerAdapter();
 
+  const defaultOps = {
+    cacheSoftSizeBytes: 700 * 1024,
+    nConcurrentDownloads: 10,
+    rangesCacheMaxSizeBytes: 1024,
+  };
+
   const driver = await DownloadDriver.init(
     logger,
     createDownloadClient(logger, client, []),
     createLogsClient(client, logger),
     dir,
+    rangesCacheDir,
     signer,
-    ops,
+    { ...defaultOps, ...ops },
   );
 
   return driver;

package/src/drivers/download_blob/download_blob.ts

@@ -19,7 +19,7 @@ import type {
   RemoteBlobHandleAndSize,
   StreamingApiResponse,
 } from '@milaboratories/pl-model-common';
-import { newRangeBytesOpt, type RangeBytes, validateRangeBytes } from '@milaboratories/pl-model-common';
+import { type RangeBytes, validateRangeBytes } from '@milaboratories/pl-model-common';
 import type {
   PlTreeEntry,
   ResourceInfo,
@@ -31,7 +31,7 @@ import {
   treeEntryToResourceInfo,
 } from '@milaboratories/pl-tree';
 import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
-import { CallersCounter, mapGet, notEmpty, TaskProcessor } from '@milaboratories/ts-helpers';
+import { CallersCounter, mapGet, TaskProcessor } from '@milaboratories/ts-helpers';
 import Denque from 'denque';
 import * as fs from 'fs';
 import { randomUUID } from 'node:crypto';
@@ -59,6 +59,7 @@ import { getSize, OnDemandBlobResourceSnapshot } from '../types';
 import { blobKey, pathToKey } from './blob_key';
 import { DownloadBlobTask, nonRecoverableError } from './download_blob_task';
 import { FilesCache } from '../helpers/files_cache';
+import { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from './sparse_cache/cache';
 
 export type DownloadDriverOps = {
   /**
@@ -67,6 +68,15 @@ export type DownloadDriverOps = {
    * when they become unneeded.
    * */
   cacheSoftSizeBytes: number;
+
+  /**
+   * A hard limit of the amount of sparse cache, in bytes.
+   * Once exceeded, the download driver will start deleting blobs one by one.
+   *
+   * The sparse cache is used to store ranges of blobs.
+   * */
+  rangesCacheMaxSizeBytes: number;
+
   /**
    * Max number of concurrent downloads while calculating computable states
    * derived from this driver
@@ -83,6 +93,7 @@ export class DownloadDriver implements BlobDriver {
   /** Writes and removes files to a hard drive and holds a counter for every
    * file that should be kept. */
   private cache: FilesCache<DownloadBlobTask>;
+  private rangesCache: SparseCache;
 
   /** Downloads files and writes them to the local dir. */
   private downloadQueue: TaskProcessor;
@@ -99,10 +110,16 @@ export class DownloadDriver implements BlobDriver {
     private readonly clientDownload: ClientDownload,
     private readonly clientLogs: ClientLogs,
     saveDir: string,
+    private readonly rangesCacheDir: string,
     private readonly signer: Signer,
-    ops: DownloadDriverOps,
+    private readonly ops: DownloadDriverOps,
   ) {
-    this.cache = new FilesCache(ops.cacheSoftSizeBytes);
+    this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);
+
+    const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);
+    const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);
+    this.rangesCache = new SparseCache(this.logger, this.ops.rangesCacheMaxSizeBytes, fsRanges, fsStorage);
+
     this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);
 
     this.saveDir = path.resolve(saveDir);
@@ -113,45 +130,16 @@ export class DownloadDriver implements BlobDriver {
     clientDownload: ClientDownload,
     clientLogs: ClientLogs,
     saveDir: string,
+    rangesCacheDir: string,
     signer: Signer,
     ops: DownloadDriverOps,
   ): Promise<DownloadDriver> {
-    const driver = new DownloadDriver(logger, clientDownload, clientLogs, saveDir, signer, ops);
-    await driver.initCache();
+    const driver = new DownloadDriver(logger, clientDownload, clientLogs, saveDir, rangesCacheDir, signer, ops);
+    await driver.rangesCache.reset();
 
     return driver;
   }
 
-  private async initCache() {
-    let files: string[];
-    try {
-      files = await fsp.readdir(this.saveDir);
-    } catch (e: unknown) {
-      if (typeof e === 'object' && e !== null && (e as { code?: string }).code === 'ENOENT') {
-        return;
-      }
-
-      throw e;
-    }
-
-    // for (const file of files) {
-    //   const { size } = await fsp.stat(path.resolve(this.saveDir, file));
-
-    //   const blobInfo = pathToBlobInfo(file);
-    //   if (blobInfo == undefined) {
-    //     continue;
-    //   }
-
-    //   this.cache.addCache({
-    //     path: path.resolve(this.saveDir, file),
-    //     baseKey: blobKey(blobInfo.resourceId),
-    //     key: blobKey(blobInfo.resourceId, blobInfo.range),
-    //     counter: new CallersCounter(),
-    //     range,
-    //   });
-    // }
-  }
-
   /** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */
   public getDownloadedBlob(
     res: ResourceInfo | PlTreeEntry,
@@ -312,17 +300,28 @@ export class DownloadDriver implements BlobDriver {
     if (isLocalBlobHandle(handle)) {
       return await read(this.getLocalPath(handle), range);
     }
+
     if (isRemoteBlobHandle(handle)) {
       const result = parseRemoteHandle(handle, this.signer);
+
+      const key = blobKey(result.info.id);
+      const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });
+      if (filePath) {
+        return await read(filePath, range);
+      }
+
       const { content } = await this.clientDownload.downloadBlob(
-        { id: result.id, type: result.type },
+        { id: result.info.id, type: result.info.type },
         undefined,
         undefined,
         range?.from,
         range?.to,
       );
 
-      return await buffer(content);
+      const data = await buffer(content);
+      await this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data);
+
+      return data;
     }
 
     throw new Error('Malformed remote handle');
@@ -342,7 +341,7 @@ export class DownloadDriver implements BlobDriver {
 
     return Computable.make((ctx) =>
       this.getDownloadedBlob(res, ctx), {
-        postprocessValue: (v) => v ? this.getContent(v.handle, range) : undefined
+      postprocessValue: (v) => v ? this.getContent(v.handle, range) : undefined
     }
     ).withStableType()
   }