@milaboratories/pl-drivers 1.5.57 → 1.5.59

package/dist/test_env.d.ts

@@ -0,0 +1,6 @@
+/** A storage where we keep all the assets.
+ * If you run tests locally, the storage should be named `library` and point
+ * to the `assets` directory in this repository,
+ * but in CI the storage is defined in env and it points to S3 bucket. */
+export declare const libraryStorage: string;
+//# sourceMappingURL=test_env.d.ts.map

package/dist/test_env.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test_env.d.ts","sourceRoot":"","sources":["../src/test_env.ts"],"names":[],"mappings":"AAAA;;;yEAGyE;AACzE,eAAO,MAAM,cAAc,QAEd,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.5.57",
+  "version": "1.5.59",
   "engines": {
     "node": ">=20"
   },
@@ -30,11 +30,12 @@
     "tar-fs": "^3.0.8",
     "undici": "~7.5.0",
     "zod": "~3.23.8",
-    "@milaboratories/ts-helpers": "^1.2.0",
-    "@milaboratories/computable": "^2.4.7",
-    "@milaboratories/pl-tree": "^1.6.1",
-    "@milaboratories/pl-model-common": "^1.15.0",
-    "@milaboratories/pl-client": "^2.9.0"
+    "upath": "^2.0.1",
+    "@milaboratories/ts-helpers": "^1.3.0",
+    "@milaboratories/computable": "^2.4.8",
+    "@milaboratories/pl-client": "^2.9.1",
+    "@milaboratories/pl-model-common": "^1.15.1",
+    "@milaboratories/pl-tree": "^1.6.2"
   },
   "devDependencies": {
     "eslint": "^9.25.1",
@@ -42,7 +43,8 @@
     "typescript": "~5.5.4",
     "vite": "^5.4.11",
     "@types/node": "~20.16.15",
-    "vitest": "^2.1.8",
+    "vitest": "^2.1.9",
+    "@vitest/coverage-v8": "^2.1.9",
     "@types/tar-fs": "^2.0.4",
     "@milaboratories/eslint-config": "^1.0.4",
     "@milaboratories/platforma-build-configs": "1.0.3"
@@ -50,7 +52,7 @@
   "scripts": {
     "type-check": "tsc --noEmit --composite false",
     "build": "vite build",
-    "test": "vitest",
+    "test": "vitest run --coverage",
     "lint": "eslint .",
     "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz"
   }

package/src/clients/download.ts

@@ -41,7 +41,10 @@ export class ClientDownload {
   close() {}
 
   /** Gets a presign URL and downloads the file.
-   * An optional range with 2 numbers from what byte and to what byte to download can be provided. */
+   * An optional range with 2 numbers from what byte and to what byte to download can be provided.
+   * @param fromBytes - from byte including this byte
+   * @param toBytes - to byte excluding this byte
+   */
   async downloadBlob(
     info: ResourceInfo,
     options?: RpcOptions,
@@ -61,14 +64,14 @@ export class ClientDownload {
 
   async readLocalFile(
     url: string,
-    fromBytes?: number,
-    toBytes?: number,
+    fromBytes?: number, // including this byte
+    toBytes?: number, // excluding this byte
   ): Promise<DownloadResponse> {
     const { storageId, relativePath } = parseLocalUrl(url);
     const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);
 
     return {
-      content: Readable.toWeb(fs.createReadStream(fullPath, { start: fromBytes, end: toBytes })),
+      content: Readable.toWeb(fs.createReadStream(fullPath, { start: fromBytes, end: toBytes !== undefined ? toBytes - 1 : undefined })),
       size: (await fsp.stat(fullPath)).size,
     };
   }

package/src/clients/helpers.ts

@@ -1,11 +1,11 @@
 export function toHeadersMap(
   headers: { name: string; value: string }[],
-  fromBytes?: number,
-  toBytes?: number,
+  fromBytes?: number, // including this byte
+  toBytes?: number, // excluding this byte
 ): Record<string, string> {
   const result = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
   if (fromBytes !== undefined && toBytes !== undefined) {
-    result['Range'] = `bytes=${fromBytes}-${toBytes}`;
+    result['Range'] = `bytes=${fromBytes}-${toBytes - 1}`;
   }
 
   return result;

package/src/drivers/download_blob/blob_key.ts

@@ -0,0 +1,15 @@
+import { bigintToResourceId, ResourceId } from "@milaboratories/pl-client";
+import * as path from 'node:path';
+
+export function blobKey(rId: ResourceId): string {
+  return `${BigInt(rId)}`;
+}
+
+export function pathToKey(fPath: string): string {
+  return path.basename(fPath);
+}
+
+export function pathToBlobInfo(fPath: string): ResourceId | undefined {
+  const base = path.basename(fPath);
+  return bigintToResourceId(BigInt(base));
+}

package/src/drivers/download_blob/download_blob.test.ts

@@ -10,30 +10,22 @@ import {
   poll,
   TestHelpers,
 } from '@milaboratories/pl-client';
-import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
+import { ConsoleLoggerAdapter, HmacSha256Signer, Signer } from '@milaboratories/ts-helpers';
 import * as fsp from 'node:fs/promises';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { scheduler } from 'node:timers/promises';
 import { createDownloadClient, createLogsClient } from '../../clients/constructors';
-import { DownloadDriver } from './download_blob';
+import { DownloadDriver, DownloadDriverOps } from './download_blob';
 import type { OnDemandBlobResourceSnapshot } from '../types';
+import * as env from '../../test_env';
 
 const fileName = 'answer_to_the_ultimate_question.txt';
 
-test('should download a blob and read its content', async () => {
+test('should download a blob and read its content', { timeout: 10000 },  async () => {
   await TestHelpers.withTempRoot(async (client) => {
-    const logger = new ConsoleLoggerAdapter();
-    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-1-'));
-
-    const driver = new DownloadDriver(
-      logger,
-      createDownloadClient(logger, client, []),
-      createLogsClient(client, logger),
-      dir,
-      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
-    );
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
+    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
     const c = driver.getDownloadedBlob(downloadable);
@@ -53,23 +45,38 @@ test('should download a blob and read its content', async () => {
 
     console.log(`should download a blob: exiting`)
   });
-}, 10000);
+});
 
-test('should not redownload a blob a file already exists', async () => {
+test('should download a blob range and read its content with a range', async () => {
   await TestHelpers.withTempRoot(async (client) => {
-    const logger = new ConsoleLoggerAdapter();
-    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-1-'));
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
+    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+
+    const c = driver.getDownloadedBlob(downloadable);
+
+    console.log(`should download a blob range: getting computable first time`)
+    const blob = await c.getValue();
+    expect(blob).toBeUndefined();
+
+    console.log(`should download a blob range: awaiting change`)
+    await c.awaitChange();
+
+    console.log(`should download a blob range: getting the blob second time`)
+    const blob2 = await c.getValue();
+    expect(blob2).toBeDefined();
+    expect(blob2!.size).toBe(3);
+    expect((await driver.getContent(blob2!.handle, { from: 0, to: 2 }))?.toString()).toBe('42');
 
-    const signer = new HmacSha256Signer(HmacSha256Signer.generateSecret());
+    console.log(`should download a blob range: exiting`)
+  });
+});
 
-    const driver = new DownloadDriver(
-      logger,
-      createDownloadClient(logger, client, []),
-      createLogsClient(client, logger),
-      dir,
-      signer,
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
-    );
+test('should not redownload a blob when a file already exists', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
+    const signer = genSigner();
+    const driver = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
 
     console.log('Download the first time');
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
@@ -77,20 +84,11 @@ test('should not redownload a blob a file already exists', async () => {
     await c.getValue();
     await c.awaitChange();
     const blob = await c.getValue();
-    expect(blob).toBeDefined();
-    expect(blob!.size).toBe(3);
     expect((await driver.getContent(blob!.handle))?.toString()).toBe('42\n');
 
     await driver.releaseAll();
 
-    const driver2 = new DownloadDriver(
-      logger,
-      createDownloadClient(logger, client, []),
-      createLogsClient(client, logger),
-      dir,
-      signer,
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
-    );
+    const driver2 = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
 
     console.log('Download the second time');
     const c2 = driver2.getDownloadedBlob(downloadable);
@@ -105,16 +103,8 @@ test('should not redownload a blob a file already exists', async () => {
 
 test('should get on demand blob without downloading a blob', async () => {
   await TestHelpers.withTempRoot(async (client) => {
-    const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-2-'));
-    const driver = new DownloadDriver(
-      logger,
-      createDownloadClient(logger, client, []),
-      createLogsClient(client, logger),
-      dir,
-      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
-    );
+    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
 
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
     const c = driver.getOnDemandBlob(downloadable);
@@ -128,18 +118,40 @@ test('should get on demand blob without downloading a blob', async () => {
   });
 });
 
+test('should get on demand blob without downloading a blob range', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-2-'));
+    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 });
+
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+
+    const c = driver.getOnDemandBlob(downloadable, undefined);
+    const blob = await c.getValue();
+    expect(blob).toBeDefined();
+    expect(blob.size).toEqual(3);
+    const content = await driver.getContent(blob!.handle, { from: 1, to: 2 });
+    expect(content?.toString()).toStrictEqual('2');
+
+    const c2 = driver.getOnDemandBlob(downloadable, undefined);
+    const blob2 = await c2.getValue();
+    expect(blob2).toBeDefined();
+    expect(blob2.size).toEqual(3);
+    const content2 = await driver.getContent(blob2!.handle, { from: 0, to: 1 });
+    expect(content2?.toString()).toStrictEqual('4');
+
+    const c3 = driver.getOnDemandBlob(downloadable);
+    const blob3 = await c3.getValue();
+    expect(blob3).toBeDefined();
+    expect(blob3.size).toEqual(3);
+    const content3 = await driver.getContent(blob3!.handle, { from: 1, to: 3 });
+    expect(content3?.toString()).toStrictEqual('2\n');
+  });
+});
+
 test('should get undefined when releasing a blob from a small cache and the blob was deleted.', async () => {
   await TestHelpers.withTempRoot(async (client) => {
-    const logger = new ConsoleLoggerAdapter();
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-3-'));
-    const driver = new DownloadDriver(
-      logger,
-      createDownloadClient(logger, client, []),
-      createLogsClient(client, logger),
-      dir,
-      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 },
-    );
+    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 });
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
     const c = driver.getDownloadedBlob(downloadable);
@@ -165,18 +177,10 @@ test('should get undefined when releasing a blob from a small cache and the blob
   });
 });
 
-test('should get the blob when releasing a blob, but a cache is big enough and it keeps a file on the local drive.', async () => {
+test('should get undefined when releasing a blob from a small cache and the blob was deleted range.', async () => {
   await TestHelpers.withTempRoot(async (client) => {
-    const logger = new ConsoleLoggerAdapter();
-    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-4-'));
-    const driver = new DownloadDriver(
-      logger,
-      createDownloadClient(logger, client, []),
-      createLogsClient(client, logger),
-      dir,
-      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
-    );
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
+    const driver = await genDriver(client, dir, genSigner(), { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 });
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
     const c = driver.getDownloadedBlob(downloadable);
@@ -189,7 +193,7 @@ test('should get the blob when releasing a blob, but a cache is big enough and i
     const blob2 = await c.getValue();
     expect(blob2).toBeDefined();
     expect(blob2!.size).toBe(3);
-    expect((await driver.getContent(blob2!.handle))?.toString()).toBe('42\n');
+    expect((await driver.getContent(blob2!.handle, { from: 1, to: 3 }))?.toString()).toBe('2\n');
 
     // The blob is removed from a cache since the size is too big.
     c.resetState();
@@ -197,18 +201,64 @@ test('should get the blob when releasing a blob, but a cache is big enough and i
 
     const c2 = driver.getDownloadedBlob(downloadable);
 
-    const blob3 = await c2.getValue();
-    expect(blob3).toBeDefined();
-    expect(blob3!.size).toBe(3);
-    expect((await driver.getContent(blob3!.handle))?.toString()).toBe('42\n');
+    const noBlob = await c2.getValue();
+    expect(noBlob).toBeUndefined();
   });
 });
 
+// test('should get the blob range after init if it already existed.', async () => {
+//   await TestHelpers.withTempRoot(async (client) => {
+//     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-driver'));
+//     const signer = genSigner();
+//     const driver = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 720, nConcurrentDownloads: 10 });
+//     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+
+//     const c = driver.getDownloadedBlob(downloadable);
+//     const blob = await c.getValue();
+//     expect(blob).toBeUndefined();
+//     await c.awaitChange();
+
+//     const blob2 = await c.getValue();
+//     expect(blob2).toBeDefined();
+//     expect(blob2!.size).toBe(3);
+//     expect((await driver.getContent(blob2!.handle, { from: 1, to: 3 }))?.toString()).toBe('2\n');
+
+//     // Make the second driver, the cache was already initialized.
+//     // We should get the blob instantly.
+//     const driver2 = await genDriver(client, dir, signer, { cacheSoftSizeBytes: 720, nConcurrentDownloads: 10 });
+//     const c2 = driver2.getDownloadedBlob(downloadable);
+//     const blob3 = await c2.getValue();
+//     expect(blob3).toBeDefined();
+//     expect(blob3!.size).toBe(2);
+//     expect((await driver.getContent(blob3!.handle, { from: 1, to: 3 }))?.toString()).toBe('2\n');
+//   });
+// })
+;
+
+function genSigner() {
+  return new HmacSha256Signer(HmacSha256Signer.generateSecret())
+}
+
+async function genDriver(client: PlClient, dir: string, signer: Signer, ops: DownloadDriverOps) {
+  const logger = new ConsoleLoggerAdapter();
+
+  const driver = await DownloadDriver.init(
+    logger,
+    createDownloadClient(logger, client, []),
+    createLogsClient(client, logger),
+    dir,
+    signer,
+    ops,
+  );
+
+  return driver;
+}
+
 async function makeDownloadableBlobFromAssets(client: PlClient, fileName: string) {
   await client.withWriteTx('MakeAssetDownloadable', async (tx: PlTransaction) => {
     const importSettings = jsonToData({
       path: fileName,
-      storageId: 'library',
+      storageId: env.libraryStorage,
     });
     const importer = tx.createStruct({ name: 'BlobImportInternal', version: '1' }, importSettings);
     const importerBlob: FieldRef = {