@milaboratories/pl-drivers 1.2.16

package/src/drivers/download_blob.test.ts

@@ -0,0 +1,223 @@
+import * as fsp from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import {
+  ConsoleLoggerAdapter,
+  HmacSha256Signer
+} from '@milaboratories/ts-helpers';
+import {
+  PlClient,
+  PlTransaction,
+  TestHelpers,
+  jsonToData,
+  FieldRef,
+  poll,
+  PollTxAccessor,
+  BasicResourceData,
+  FieldId
+} from '@milaboratories/pl-client';
+import { scheduler } from 'node:timers/promises';
+import { createDownloadClient, createLogsClient } from '../clients/helpers';
+import {
+  DownloadDriver,
+  OnDemandBlobResourceSnapshot
+} from './download_and_logs_blob';
+
+const fileName = 'answer_to_the_ultimate_question.txt';
+
+test('should download a blob and read its content', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-1-'));
+
+    const driver = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client),
+      createLogsClient(client, logger),
+      dir,
+      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+
+    const c = driver.getDownloadedBlob(downloadable);
+
+    const blob = await c.getValue();
+    expect(blob).toBeUndefined();
+
+    await c.awaitChange();
+
+    const blob2 = await c.getValue();
+    expect(blob2).toBeDefined();
+    expect(blob2!.size).toBe(3);
+    expect((await driver.getContent(blob2!.handle))?.toString()).toBe('42\n');
+  });
+});
+
+test('should get on demand blob without downloading a blob', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-2-'));
+    const driver = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client),
+      createLogsClient(client, logger),
+      dir,
+      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+    const c = driver.getOnDemandBlob(downloadable);
+
+    const blob = await c.getValue();
+    expect(blob).toBeDefined();
+    expect(blob.size).toEqual(3);
+
+    const content = await driver.getContent(blob!.handle);
+    expect(content?.toString()).toStrictEqual('42\n');
+  });
+});
+
+test('should get undefined when releasing a blob from a small cache and the blob was deleted.', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-3-'));
+    const driver = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client),
+      createLogsClient(client, logger),
+      dir,
+      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
+      { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 }
+    );
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+
+    const c = driver.getDownloadedBlob(downloadable);
+
+    const blob = await c.getValue();
+    expect(blob).toBeUndefined();
+
+    await c.awaitChange();
+
+    const blob2 = await c.getValue();
+    expect(blob2).toBeDefined();
+    expect(blob2!.size).toBe(3);
+    expect((await driver.getContent(blob2!.handle))?.toString()).toBe('42\n');
+
+    // The blob is removed from a cache since the size is too big.
+    c.resetState();
+    await scheduler.wait(100);
+
+    const c2 = driver.getDownloadedBlob(downloadable);
+
+    const noBlob = await c2.getValue();
+    expect(noBlob).toBeUndefined();
+  });
+});
+
+test('should get the blob when releasing a blob, but a cache is big enough and it keeps a file on the local drive.', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-4-'));
+    const driver = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client),
+      createLogsClient(client, logger),
+      dir,
+      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+
+    const c = driver.getDownloadedBlob(downloadable);
+
+    const blob = await c.getValue();
+    expect(blob).toBeUndefined();
+
+    await c.awaitChange();
+
+    const blob2 = await c.getValue();
+    expect(blob2).toBeDefined();
+    expect(blob2!.size).toBe(3);
+    expect((await driver.getContent(blob2!.handle))?.toString()).toBe('42\n');
+
+    // The blob is removed from a cache since the size is too big.
+    c.resetState();
+    await scheduler.wait(100);
+
+    const c2 = driver.getDownloadedBlob(downloadable);
+
+    const blob3 = await c2.getValue();
+    expect(blob3).toBeDefined();
+    expect(blob3!.size).toBe(3);
+    expect((await driver.getContent(blob3!.handle))?.toString()).toBe('42\n');
+  });
+});
+
+async function makeDownloadableBlobFromAssets(
+  client: PlClient,
+  fileName: string
+) {
+  await client.withWriteTx(
+    'MakeAssetDownloadable',
+    async (tx: PlTransaction) => {
+      const importSettings = jsonToData({
+        path: fileName,
+        storageId: 'library'
+      });
+      const importer = tx.createStruct(
+        { name: 'BlobImportInternal', version: '1' },
+        importSettings
+      );
+      const importerBlob: FieldRef = {
+        resourceId: importer,
+        fieldName: 'blob'
+      };
+
+      const download = tx.createStruct({
+        name: 'BlobDownload',
+        version: '2'
+      });
+      const downloadBlob: FieldRef = {
+        resourceId: download,
+        fieldName: 'blob'
+      };
+      const downloadDownloadable: FieldRef = {
+        resourceId: download,
+        fieldName: 'downloadable'
+      };
+
+      const dynamicId: FieldId = {
+        resourceId: client.clientRoot,
+        fieldName: 'result'
+      };
+
+      tx.setField(downloadBlob, importerBlob);
+      tx.createField(dynamicId, 'Dynamic', downloadDownloadable);
+      await tx.commit();
+    }
+  );
+
+  const [download, kv] = await poll(client, async (tx: PollTxAccessor) => {
+    const root = await tx.get(client.clientRoot);
+    const download = await root.get('result');
+
+    return [
+      download.data,
+      await download.getKValueObj<{ sizeBytes: string }>('ctl/file/blobInfo')
+    ];
+  });
+
+  return {
+    id: download.id,
+    type: download.type,
+    data: undefined,
+    fields: undefined,
+    kv: {
+      'ctl/file/blobInfo': {
+        sizeBytes: Number(kv.sizeBytes)
+      }
+    }
+  } as OnDemandBlobResourceSnapshot;
+}

package/src/drivers/download_url.test.ts

@@ -0,0 +1,90 @@
+import { TestHelpers } from '@milaboratories/pl-client';
+import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
+import * as os from 'node:os';
+import { text } from 'node:stream/consumers';
+import { Readable } from 'node:stream';
+import * as fs from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+import { DownloadUrlDriver } from './download_url';
+
+test('should download a tar archive and extracts its content and then deleted', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
+    const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir);
+
+    const url = new URL(
+      'https://block.registry.platforma.bio/releases/v1/milaboratory/enter-numbers/0.4.1/frontend.tgz'
+    );
+
+    const c = driver.getPath(url);
+
+    const path1 = await c.getValue();
+    expect(path1).toBeUndefined();
+
+    await c.awaitChange();
+
+    const path2 = await c.getValue();
+    expect(path2).not.toBeUndefined();
+    expect(path2?.error).toBeUndefined();
+    expect(path2?.path).not.toBeUndefined();
+
+    console.log('frontend saved to dir: ', path2);
+    const indexJs = fs.createReadStream(path.join(path2!.path!, 'index.js'));
+    const indexJsCode = await text(Readable.toWeb(indexJs));
+    expect(indexJsCode).toContain('use strict');
+
+    c.resetState();
+  });
+});
+
+test('should show a error when 403 status code', async () => {
+  try {
+    await TestHelpers.withTempRoot(async (client) => {
+      const logger = new ConsoleLoggerAdapter();
+      const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
+      const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir);
+
+      const url = new URL(
+        'https://block.registry.platforma.bio/releases/v1/milaboratory/NOT_FOUND'
+      );
+
+      const c = driver.getPath(url);
+
+      const path1 = await c.getValue();
+      expect(path1).toBeUndefined();
+
+      await c.awaitChange();
+
+      const path2 = await c.getValue();
+      expect(path2).not.toBeUndefined();
+      expect(path2?.error).not.toBeUndefined();
+    });
+  } catch (e) {
+    console.log('HERE: ', e);
+  }
+});
+
+test('should abort a downloading process when we reset a state of a computable', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test2-'));
+    const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir);
+
+    const url = new URL(
+      'https://block.registry.platforma.bio/releases/v1/milaboratory/enter-numbers/0.4.1/frontend.tgz'
+    );
+
+    const c = driver.getPath(url);
+
+    const path1 = await c.getValue();
+    expect(path1).toBeUndefined();
+
+    c.resetState();
+    await c.awaitChange();
+
+    const path2 = await c.getValue();
+    expect(path2).toBeUndefined();
+  });
+});

package/src/drivers/download_url.ts

@@ -0,0 +1,314 @@
+import {
+  CallersCounter,
+  MiLogger,
+  TaskProcessor,
+  notEmpty
+} from '@milaboratories/ts-helpers';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+import { Writable, Transform } from 'node:stream';
+import {
+  ChangeSource,
+  Computable,
+  ComputableCtx,
+  Watcher
+} from '@milaboratories/computable';
+import { randomUUID, createHash } from 'node:crypto';
+import * as zlib from 'node:zlib';
+import * as tar from 'tar-fs';
+import { FilesCache } from './helpers/files_cache';
+import { Dispatcher } from 'undici';
+import { DownloadHelper, NetworkError400 } from '../helpers/download';
+
+export interface DownloadUrlSyncReader {
+  /** Returns a Computable that (when the time will come)
+   * downloads an archive from an URL,
+   * extracts it to the local dir and returns a path to that dir. */
+  getPath(url: URL): Computable<PathResult | undefined>;
+}
+
+export interface PathResult {
+  /** Path to the downloadable blob, might be undefined when the error happened. */
+  path?: string;
+  /** Error that happened when the archive were downloaded. */
+  error?: string;
+}
+
+export type DownloadUrlDriverOps = {
+  cacheSoftSizeBytes: number;
+  withGunzip: boolean;
+  nConcurrentDownloads: number;
+};
+
+/** Downloads .tar or .tar.gz archives by given URLs
+ * and extracts them into saveDir. */
+export class DownloadUrlDriver implements DownloadUrlSyncReader {
+  private readonly downloadHelper: DownloadHelper;
+
+  private urlToDownload: Map<string, Download> = new Map();
+  private downloadQueue: TaskProcessor;
+
+  /** Writes and removes files to a hard drive and holds a counter for every
+   * file that should be kept. */
+  private cache: FilesCache<Download>;
+
+  constructor(
+    private readonly logger: MiLogger,
+    httpClient: Dispatcher,
+    private readonly saveDir: string,
+    private readonly opts: DownloadUrlDriverOps = {
+      cacheSoftSizeBytes: 50 * 1024 * 1024,
+      withGunzip: true,
+      nConcurrentDownloads: 50
+    }
+  ) {
+    this.downloadQueue = new TaskProcessor(
+      this.logger,
+      this.opts.nConcurrentDownloads
+    );
+    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);
+    this.downloadHelper = new DownloadHelper(httpClient);
+  }
+
+  /** Use to get a path result inside a computable context */
+  getPath(url: URL, ctx: ComputableCtx): PathResult | undefined;
+
+  /** Returns a Computable that do the work */
+  getPath(url: URL): Computable<PathResult | undefined>;
+
+  getPath(
+    url: URL,
+    ctx?: ComputableCtx
+  ): Computable<PathResult | undefined> | PathResult | undefined {
+    // wrap result as computable, if we were not given an existing computable context
+    if (ctx === undefined) return Computable.make((c) => this.getPath(url, c));
+
+    const callerId = randomUUID();
+
+    // read as ~ golang's defer
+    ctx.addOnDestroy(() => this.releasePath(url, callerId));
+
+    const result = this.getPathNoCtx(url, ctx.watcher, callerId);
+    if (result?.path === undefined)
+      ctx.markUnstable(
+        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`
+      );
+
+    return result;
+  }
+
+  getPathNoCtx(url: URL, w: Watcher, callerId: string) {
+    const key = url.toString();
+    const task = this.urlToDownload.get(key);
+
+    if (task != undefined) {
+      task.attach(w, callerId);
+      return task.getPath();
+    }
+
+    const newTask = this.setNewTask(w, url, callerId);
+    this.downloadQueue.push({
+      fn: async () => this.downloadUrl(newTask, callerId),
+      recoverableErrorPredicate: (e) => true
+    });
+
+    return newTask.getPath();
+  }
+
+  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
+  async downloadUrl(task: Download, callerId: string) {
+    await task.download(this.downloadHelper, this.opts.withGunzip);
+    // Might be undefined if a error happened
+    if (task.getPath()?.path != undefined) this.cache.addCache(task, callerId);
+  }
+
+  /** Removes a directory and aborts a downloading task when all callers
+   * are not interested in it. */
+  async releasePath(url: URL, callerId: string): Promise<void> {
+    const key = url.toString();
+    const task = this.urlToDownload.get(key);
+    if (task == undefined) return;
+
+    if (this.cache.existsFile(task.path)) {
+      const toDelete = this.cache.removeFile(task.path, callerId);
+
+      await Promise.all(
+        toDelete.map(async (task) => {
+          await rmRFDir(task.path);
+          this.cache.removeCache(task);
+
+          this.removeTask(
+            task,
+            `the task ${JSON.stringify(task)} was removed` +
+              `from cache along with ${JSON.stringify(toDelete)}`
+          );
+        })
+      );
+    } else {
+      // The task is still in a downloading queue.
+      const deleted = task.counter.dec(callerId);
+      if (deleted)
+        this.removeTask(
+          task,
+          `the task ${JSON.stringify(task)} was removed from cache`
+        );
+    }
+  }
+
+  /** Removes all files from a hard drive. */
+  async releaseAll() {
+    this.downloadQueue.stop();
+
+    await Promise.all(
+      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {
+        await rmRFDir(task.path);
+        this.cache.removeCache(task);
+
+        this.removeTask(
+          task,
+          `the task ${task} was released when the driver was closed`
+        );
+      })
+    );
+  }
+
+  private setNewTask(w: Watcher, url: URL, callerId: string) {
+    const result = new Download(this.getFilePath(url), url);
+    result.attach(w, callerId);
+    this.urlToDownload.set(url.toString(), result);
+
+    return result;
+  }
+
+  private removeTask(task: Download, reason: string) {
+    task.abort(reason);
+    task.change.markChanged();
+    this.urlToDownload.delete(task.url.toString());
+  }
+
+  private getFilePath(url: URL): string {
+    const sha256 = createHash('sha256').update(url.toString()).digest('hex');
+    return path.join(this.saveDir, sha256);
+  }
+}
+
+class Download {
+  readonly counter = new CallersCounter();
+  readonly change = new ChangeSource();
+  readonly signalCtl = new AbortController();
+  error: string | undefined;
+  done = false;
+  sizeBytes = 0;
+
+  constructor(
+    readonly path: string,
+    readonly url: URL
+  ) {}
+
+  attach(w: Watcher, callerId: string) {
+    this.counter.inc(callerId);
+    if (!this.done) this.change.attachWatcher(w);
+  }
+
+  async download(clientDownload: DownloadHelper, withGunzip: boolean) {
+    try {
+      const sizeBytes = await this.downloadAndUntar(
+        clientDownload,
+        withGunzip,
+        this.signalCtl.signal
+      );
+      this.setDone(sizeBytes);
+    } catch (e: any) {
+      if (e instanceof URLAborted || e instanceof NetworkError400) {
+        this.setError(e);
+        // Just in case we were half-way extracting an archive.
+        await rmRFDir(this.path);
+        return;
+      }
+
+      throw e;
+    }
+  }
+
+  private async downloadAndUntar(
+    clientDownload: DownloadHelper,
+    withGunzip: boolean,
+    signal: AbortSignal
+  ): Promise<number> {
+    if (await fileExists(this.path)) {
+      return await dirSize(this.path);
+    }
+
+    const resp = await clientDownload.downloadRemoteFile(
+      this.url.toString(),
+      {},
+      signal
+    );
+    let content = resp.content;
+
+    if (withGunzip) {
+      const gunzip = Transform.toWeb(zlib.createGunzip());
+      content = content.pipeThrough(gunzip, { signal });
+    }
+    const untar = Writable.toWeb(tar.extract(this.path));
+    await content.pipeTo(untar, { signal });
+
+    return resp.size;
+  }
+
+  getPath(): PathResult | undefined {
+    if (this.done) return { path: notEmpty(this.path) };
+
+    if (this.error) return { error: this.error };
+
+    return undefined;
+  }
+
+  private setDone(sizeBytes: number) {
+    this.done = true;
+    this.sizeBytes = sizeBytes;
+    this.change.markChanged();
+  }
+
+  abort(reason: string) {
+    this.signalCtl.abort(new URLAborted(reason));
+  }
+
+  private setError(e: any) {
+    this.error = String(e);
+    this.change.markChanged();
+  }
+}
+
+class URLAborted extends Error {}
+
+async function fileExists(path: string): Promise<boolean> {
+  try {
+    await fsp.access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Gets a directory size by calculating sizes recursively. */
+async function dirSize(dir: string): Promise<number> {
+  const files = await fsp.readdir(dir, { withFileTypes: true });
+  const sizes = await Promise.all(
+    files.map(async (file) => {
+      const fPath = path.join(dir, file.name);
+
+      if (file.isDirectory()) return await dirSize(fPath);
+
+      const stat = await fsp.stat(fPath);
+      return stat.size;
+    })
+  );
+
+  return sizes.reduce((sum, size) => sum + size, 0);
+}
+
+/** Do rm -rf on dir. */
+async function rmRFDir(path: string) {
+  await fsp.rm(path, { recursive: true, force: true });
+}

package/src/drivers/helpers/files_cache.test.ts

@@ -0,0 +1,79 @@
+import { CachedFile, FilesCache } from './files_cache';
+import { CallersCounter } from '@milaboratories/ts-helpers';
+
+test('should delete blob3 when add 3 blobs, exceed a soft limit and nothing holds blob3', () => {
+  const cache = new FilesCache(20);
+  const callerId1 = 'callerId1';
+  const blob1: CachedFile = {
+    path: 'path1',
+    sizeBytes: 5,
+    counter: new CallersCounter()
+  };
+  const blob2: CachedFile = {
+    path: 'path2',
+    sizeBytes: 10,
+    counter: new CallersCounter()
+  };
+  const blob3: CachedFile = {
+    path: 'path3',
+    sizeBytes: 10,
+    counter: new CallersCounter()
+  };
+
+  // add blobs and check that we don't exceed the soft limit.
+  cache.addCache(blob1, callerId1);
+  cache.addCache(blob2, callerId1);
+  expect(cache.toDelete()).toHaveLength(0);
+
+  // add already existing blob and, again, check that we don't exceed the limit.
+  cache.addCache(blob2, callerId1);
+  expect(cache.toDelete()).toHaveLength(0);
+
+  // add the third blob. We exceeds a soft limit,
+  // but every blob has a positive counter,
+  // so we can't delete anything.
+  cache.addCache(blob3, callerId1);
+  expect(cache.toDelete()).toHaveLength(0);
+
+  // blob3 have a zero counter, we can delete it.
+  const toDelete = cache.removeFile(blob3.path, callerId1);
+  expect(toDelete).toStrictEqual([blob3]);
+
+  // removes blob3 from a cache, checks that others are still there.
+  cache.removeCache(blob3);
+  expect(cache.getFile(blob1.path, callerId1)).toBe(blob1);
+  expect(cache.getFile(blob2.path, callerId1)).toBe(blob2);
+  expect(cache.getFile(blob3.path, callerId1)).toBeUndefined();
+});
+
+test('regression should allow to add empty files', () => {
+  const cache = new FilesCache(1);
+  const callerId1 = 'callerId1';
+  const blob1: CachedFile = {
+    path: 'path1',
+    sizeBytes: 0,
+    counter: new CallersCounter()
+  };
+  const blob2: CachedFile = {
+    path: 'path2',
+    sizeBytes: 2,
+    counter: new CallersCounter()
+  };
+
+  // add a blob with 0 size.
+  cache.addCache(blob1, callerId1);
+  expect(cache.toDelete()).toHaveLength(0);
+
+  // add a blob that exceed a soft limit
+  cache.addCache(blob2, callerId1);
+  expect(cache.toDelete()).toHaveLength(0);
+
+  // this blob should be deleted
+  const toDelete = cache.removeFile(blob2.path, callerId1);
+  expect(toDelete).toStrictEqual([blob2]);
+
+  cache.removeCache(blob2);
+
+  // and the the blob with zero size remained
+  expect(cache.getFile(blob1.path, callerId1)).toBe(blob1);
+});