@milaboratories/pl-drivers 1.8.3 → 1.9.1

package/dist/drivers/download_blob_url/task.js

@@ -0,0 +1,184 @@
+import { Writable, Transform } from 'node:stream';
+import * as zlib from 'node:zlib';
+import * as tar from 'tar-fs';
+import path__default from 'node:path';
+import fs__default from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import { NetworkError400 } from '../../helpers/download.js';
+import { ChangeSource } from '@milaboratories/computable';
+import { CallersCounter, ensureDirExists, fileExists, createPathAtomically, notEmpty } from '@milaboratories/ts-helpers';
+import { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download.js';
+import { newFolderURL } from '../urls/url.js';
+import decompress from 'decompress';
+import { assertNever } from '@protobuf-ts/runtime';
+import { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';
+
+/** Downloads and extracts an archive to a directory. */
+class DownloadAndUnarchiveTask {
+    logger;
+    signer;
+    saveDir;
+    path;
+    rInfo;
+    format;
+    clientDownload;
+    counter = new CallersCounter();
+    change = new ChangeSource();
+    signalCtl = new AbortController();
+    error;
+    done = false;
+    size = 0;
+    url;
+    state;
+    constructor(logger, signer, saveDir, path, rInfo, format, clientDownload) {
+        this.logger = logger;
+        this.signer = signer;
+        this.saveDir = saveDir;
+        this.path = path;
+        this.rInfo = rInfo;
+        this.format = format;
+        this.clientDownload = clientDownload;
+    }
+    /** A debug info of the task. */
+    info() {
+        return {
+            rInfo: this.rInfo,
+            format: this.format,
+            path: this.path,
+            done: this.done,
+            size: this.size,
+            error: this.error,
+            taskHistory: this.state,
+        };
+    }
+    attach(w, callerId) {
+        this.counter.inc(callerId);
+        if (!this.done)
+            this.change.attachWatcher(w);
+    }
+    async download() {
+        try {
+            const size = await this.downloadAndDecompress(this.signalCtl.signal);
+            this.setDone(size);
+            this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} finished`);
+            this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);
+        }
+        catch (e) {
+            this.logger.warn(`a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`);
+            if (nonRecoverableError(e)) {
+                this.setError(e);
+                this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} failed`);
+                // Just in case we were half-way extracting an archive.
+                await rmRFDir(this.path);
+                return;
+            }
+            throw e;
+        }
+    }
+    /** Does the download part and keeps a state of the process. */
+    async downloadAndDecompress(signal) {
+        this.state = {};
+        this.state.parentDir = path__default.dirname(this.path);
+        await ensureDirExists(this.state.parentDir);
+        this.state.fileExisted = await fileExists(this.path);
+        if (this.state.fileExisted) {
+            return await dirSize(this.path);
+        }
+        const size = await this.clientDownload.withBlobContent(this.rInfo, {}, { signal }, async (content, size) => {
+            this.state.downloaded = true;
+            await createPathAtomically(this.logger, this.path, async (fPath) => {
+                this.state.tempPath = fPath;
+                this.state.archiveFormat = this.format;
+                switch (this.format) {
+                    case 'tar':
+                        await fsp.mkdir(fPath); // throws if a directory already exists.
+                        const simpleUntar = Writable.toWeb(tar.extract(fPath));
+                        await content.pipeTo(simpleUntar, { signal });
+                        return;
+                    case 'tgz':
+                        await fsp.mkdir(fPath); // throws if a directory already exists.
+                        const gunzip = Transform.toWeb(zlib.createGunzip());
+                        const untar = Writable.toWeb(tar.extract(fPath));
+                        await content
+                            .pipeThrough(gunzip, { signal })
+                            .pipeTo(untar, { signal });
+                        return;
+                    case 'zip':
+                        this.state.zipPath = this.path + '.zip';
+                        const f = Writable.toWeb(fs__default.createWriteStream(this.state.zipPath));
+                        await content.pipeTo(f, { signal });
+                        this.state.zipPathCreated = true;
+                        // Without this filter it fails with
+                        // "EISDIR: illegal operation on a directory".
+                        // The workaround is from
+                        // https://github.com/kevva/decompress/issues/46#issuecomment-525048104
+                        await decompress(this.state.zipPath, fPath, {
+                            filter: file => !file.path.endsWith('/'),
+                        });
+                        this.state.zipDecompressed = true;
+                        await fs__default.promises.rm(this.state.zipPath);
+                        this.state.zipPathDeleted = true;
+                        return;
+                    default:
+                        assertNever(this.format);
+                }
+            });
+            this.state.pathCreated = true;
+            return size;
+        });
+        return size;
+    }
+    getURL() {
+        if (this.done)
+            return { url: notEmpty(this.url) };
+        if (this.error)
+            return { error: this.error };
+        return undefined;
+    }
+    setDone(size) {
+        this.done = true;
+        this.size = size;
+        this.url = newFolderURL(this.signer, this.saveDir, this.path);
+    }
+    setError(e) {
+        this.error = String(e);
+    }
+    abort(reason) {
+        this.signalCtl.abort(new URLAborted(reason));
+    }
+}
+/** Gets a directory size by calculating sizes recursively. */
+async function dirSize(dir) {
+    const files = await fsp.readdir(dir, { withFileTypes: true });
+    const sizes = await Promise.all(files.map(async (file) => {
+        const fPath = path__default.join(dir, file.name);
+        if (file.isDirectory())
+            return await dirSize(fPath);
+        const stat = await fsp.stat(fPath);
+        return stat.size;
+    }));
+    return sizes.reduce((sum, size) => sum + size, 0);
+}
+/** Do rm -rf on dir. */
+async function rmRFDir(path) {
+    await fsp.rm(path, { recursive: true, force: true });
+}
+/** Throws when a downloading aborts. */
+class URLAborted extends Error {
+    name = 'URLAborted';
+}
+function nonRecoverableError(e) {
+    return (e instanceof URLAborted
+        || e instanceof NetworkError400
+        || e instanceof UnknownStorageError
+        || e instanceof WrongLocalFileUrl
+        // file that we downloads from was moved or deleted.
+        || e?.code == 'ENOENT'
+        // A resource was deleted.
+        || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))
+        // wrong archive format
+        || (String(e).includes('incorrect header check')));
+}
+
+export { DownloadAndUnarchiveTask, nonRecoverableError, rmRFDir };
+//# sourceMappingURL=task.js.map

package/dist/drivers/download_blob_url/task.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"task.js","sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport path from 'path';\nimport fs from 'fs';\nimport * as fsp from 'fs/promises';\nimport { NetworkError400 } from '../../helpers/download';\nimport type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from '@milaboratories/ts-helpers';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { UnknownStorageError, WrongLocalFileUrl, type ClientDownload } from '../../clients/download';\nimport type { ArchiveFormat, FolderURL } from '@milaboratories/pl-model-common';\nimport { newFolderURL } from '../urls/url';\nimport decompress from 'decompress';\nimport { assertNever } from '@protobuf-ts/runtime';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\nexport type URLResult = {\n  url?: FolderURL;\n  error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadAndUnarchiveTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n  private url: FolderURL | undefined;\n  private state: DownloadCtx | undefined;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    readonly saveDir: string,\n    readonly path: string,\n    readonly rInfo: DownloadableBlobSnapshot,\n    readonly format: ArchiveFormat,\n    private readonly clientDownload: ClientDownload,\n  ) {}\n\n  /** A debug info of the task. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      format: this.format,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n      taskHistory: this.state,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download() {\n    try {\n      const size = await this.downloadAndDecompress(this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} finished`);\n\n      this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n    } catch (e: any) {\n      this.logger.warn(`a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`);\n\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  /** Does the download part and keeps a state of the process. */\n  private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n    this.state = {};\n\n    this.state.parentDir = path.dirname(this.path);\n    await ensureDirExists(this.state.parentDir);\n\n    this.state.fileExisted = await fileExists(this.path);\n    if (this.state.fileExisted) {\n      return await dirSize(this.path);\n    }\n\n    const size = await this.clientDownload.withBlobContent(\n      this.rInfo, \n      {}, \n      { signal },\n      async (content, size) => {\n        this.state!.downloaded = true;\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          this.state!.tempPath = fPath;\n          this.state!.archiveFormat = this.format;\n\n          switch (this.format) {\n            case 'tar':\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const simpleUntar = Writable.toWeb(tar.extract(fPath));\n              await content.pipeTo(simpleUntar, { signal });\n              return;\n\n            case 'tgz':\n              await fsp.mkdir(fPath); // throws if a directory already exists.\n              const gunzip = Transform.toWeb(zlib.createGunzip());\n              const untar = Writable.toWeb(tar.extract(fPath));\n\n              await content\n                .pipeThrough(gunzip, { signal })\n                .pipeTo(untar, { signal });\n              return;\n\n            case 'zip':\n              this.state!.zipPath = this.path + '.zip';\n\n              const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n              await content.pipeTo(f, { signal });\n              this.state!.zipPathCreated = true;\n\n              // Without this filter it fails with\n              // \"EISDIR: illegal operation on a directory\".\n              // The workaround is from\n              // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n              await decompress(this.state!.zipPath, fPath, {\n                filter: file => !file.path.endsWith('/'),\n              });\n              this.state!.zipDecompressed = true;\n\n              await fs.promises.rm(this.state!.zipPath);\n              this.state!.zipPathDeleted = true;\n\n              return;\n\n            default:\n              assertNever(this.format);\n          }\n        });\n\n        this.state!.pathCreated = true;\n        return size;\n      }\n    );\n\n    return size;\n  }\n\n  getURL(): URLResult | undefined {\n    if (this.done) return { url: notEmpty(this.url) };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n    this.url = newFolderURL(this.signer, this.saveDir, this.path);\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n\n/** Just a type that adds lots of context when the error happens. */\ntype DownloadCtx = {\n  parentDir?: string;\n  fileExisted?: boolean;\n  downloaded?: boolean;\n  archiveFormat?: ArchiveFormat;\n  tempPath?: string;\n  zipPath?: string;\n  zipPathCreated?: boolean;\n  zipDecompressed?: boolean;\n  zipPathDeleted?: boolean;\n  pathCreated?: boolean;\n};\n\n/** Throws when a downloading aborts. */\nclass URLAborted extends Error {\n  name = 'URLAborted';\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof URLAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n    // wrong archive format\n    || (String(e).includes('incorrect header check'))\n  );\n}\n"],"names":["path","fs"],"mappings":";;;;;;;;;;;;;;;AAwBA;MACa,wBAAwB,CAAA;AAWhB,IAAA,MAAA;AACA,IAAA,MAAA;AACR,IAAA,OAAA;AACA,IAAA,IAAA;AACA,IAAA,KAAA;AACA,IAAA,MAAA;AACQ,IAAA,cAAA;AAhBV,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;AACA,IAAA,GAAG;AACH,IAAA,KAAK;AAEb,IAAA,WAAA,CACmB,MAAgB,EAChB,MAAc,EACtB,OAAe,EACf,IAAY,EACZ,KAA+B,EAC/B,MAAqB,EACb,cAA8B,EAAA;QAN9B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,MAAM,GAAN,MAAM;QACE,IAAA,CAAA,cAAc,GAAd,cAAc;IAC9B;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACpE,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;AAEpG,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,0BAAA,EAA6B,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;QACvF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA,uBAAA,EAA0B,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;AAE5G,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAElG,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,qBAAqB,CAAC,MAAmB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;AAEf,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAGA,aAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;AAE3C,QAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;AACpD,QAAA,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;AAC1B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACpD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAM,CAAC,UAAU,GAAG,IAAI;AAE7B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;AACzE,gBAAA,IAAI,CAAC,KAAM,CAAC,QAAQ,GAAG,KAAK;gBAC5B,IAAI,CAAC,KAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM;AAEvC,gBAAA,QAAQ,IAAI,CAAC,MAAM;AACjB,oBAAA,KAAK,KAAK;wBACR,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,wBAAA,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACtD,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC7C;AAEF,oBAAA,KAAK,KAAK;wBACR,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;AACnD,wBAAA,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEhD,wBAAA,MAAM;AACH,6BAAA,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE;AAC9B,6BAAA,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC5B;AAEF,oBAAA,KAAK,KAAK;wBACR,IAAI,CAAC,KAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM;AAExC,wBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACC,WAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC;wBACnE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC;AACnC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;;;;;wBAMjC,MAAM,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,EAAE,KAAK,EAAE;AAC3C,4BAAA,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;AACzC,yBAAA,CAAC;AACF,wBAAA,IAAI,CAAC,KAAM,CAAC,eAAe,GAAG,IAAI;AAElC,wBAAA,MAAMA,WAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC;AACzC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;wBAEjC;AAEF,oBAAA;AACE,wBAAA,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC;;AAE9B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAM,CAAC,WAAW,GAAG,IAAI;AAC9B,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAEjD,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IAC/D;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAGD,aAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;AAgBA;AACA,MAAM,UAAW,SAAQ,KAAK,CAAA;IAC5B,IAAI,GAAG,YAAY;AACpB;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;AAEX,YAAC,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC;;YAEtE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;AAErD;;;;"}

package/dist/drivers/download_blob_url/url.d.ts

@@ -1 +1 @@
-//# sourceMappingURL=url.d.ts.map
+export {};

package/dist/drivers/download_url/driver.cjs

@@ -0,0 +1,149 @@
+'use strict';
+
+var computable = require('@milaboratories/computable');
+var tsHelpers = require('@milaboratories/ts-helpers');
+var node_crypto = require('node:crypto');
+var path = require('node:path');
+var download = require('../../helpers/download.cjs');
+var files_cache = require('../helpers/files_cache.cjs');
+var plClient = require('@milaboratories/pl-client');
+var plModelCommon = require('@milaboratories/pl-model-common');
+var url = require('../urls/url.cjs');
+var task = require('./task.cjs');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
+
+/** Downloads .tar or .tar.gz archives by given URLs
+ * and extracts them into saveDir. */
+class DownloadUrlDriver {
+    logger;
+    saveDir;
+    signer;
+    opts;
+    downloadHelper;
+    urlToDownload = new Map();
+    downloadQueue;
+    /** Writes and removes files to a hard drive and holds a counter for every
+     * file that should be kept. */
+    cache;
+    constructor(logger, httpClient, saveDir, signer, opts = {
+        cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB
+        withGunzip: true,
+        nConcurrentDownloads: 50,
+    }) {
+        this.logger = logger;
+        this.saveDir = saveDir;
+        this.signer = signer;
+        this.opts = opts;
+        this.downloadQueue = new tsHelpers.TaskProcessor(this.logger, this.opts.nConcurrentDownloads);
+        this.cache = new files_cache.FilesCache(this.opts.cacheSoftSizeBytes);
+        this.downloadHelper = new download.RemoteFileDownloader(httpClient);
+    }
+    /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */
+    getUrl(url, ctx) {
+        // wrap result as computable, if we were not given an existing computable context
+        if (ctx === undefined)
+            return computable.Computable.make((c) => this.getUrl(url, c));
+        const callerId = node_crypto.randomUUID();
+        // read as ~ golang's defer
+        ctx.addOnDestroy(() => this.releasePath(url, callerId));
+        const result = this.getUrlNoCtx(url, ctx.watcher, callerId);
+        if (result?.url === undefined)
+            ctx.markUnstable(`a path to the downloaded and untared archive might be undefined. The current result: ${result}`);
+        return result;
+    }
+    getUrlNoCtx(url, w, callerId) {
+        const key = url.toString();
+        const task = this.urlToDownload.get(key);
+        if (task !== undefined) {
+            task.attach(w, callerId);
+            return task.getUrl();
+        }
+        const newTask = this.setNewTask(w, url, callerId);
+        this.downloadQueue.push({
+            fn: async () => this.downloadUrl(newTask, callerId),
+            recoverableErrorPredicate: (e) => true,
+        });
+        return newTask.getUrl();
+    }
+    getPathForBlockUI(url$1) {
+        if (!plModelCommon.isBlockUIURL(url$1)) {
+            throw new Error(`getPathForBlockUI: ${url$1} is invalid`);
+        }
+        return url.getPathForBlockUIURL(this.signer, url$1, this.saveDir);
+    }
+    /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
+    async downloadUrl(task, callerId) {
+        await task.download(this.downloadHelper, this.opts.withGunzip);
+        // Might be undefined if a error happened
+        if (task.getUrl()?.url !== undefined)
+            this.cache.addCache(task, callerId);
+    }
+    /** Removes a directory and aborts a downloading task when all callers
+     * are not interested in it. */
+    async releasePath(url, callerId) {
+        const key = url.toString();
+        const task$1 = this.urlToDownload.get(key);
+        if (task$1 == undefined)
+            return;
+        if (this.cache.existsFile(task$1.path)) {
+            const toDelete = this.cache.removeFile(task$1.path, callerId);
+            await Promise.all(toDelete.map(async (task$1) => {
+                await task.rmRFDir(task$1.path);
+                this.cache.removeCache(task$1);
+                this.removeTask(task$1, `the task ${plClient.stringifyWithResourceId(task$1.info())} was removed`
+                    + `from cache along with ${plClient.stringifyWithResourceId(toDelete.map((t) => t.info()))}`);
+            }));
+        }
+        else {
+            // The task is still in a downloading queue.
+            const deleted = task$1.counter.dec(callerId);
+            if (deleted)
+                this.removeTask(task$1, `the task ${plClient.stringifyWithResourceId(task$1.info())} was removed from cache`);
+        }
+    }
+    /** Removes all files from a hard drive. */
+    async releaseAll() {
+        this.downloadQueue.stop();
+        await Promise.all(Array.from(this.urlToDownload.entries()).map(async ([id, task$1]) => {
+            await task.rmRFDir(task$1.path);
+            this.cache.removeCache(task$1);
+            this.removeTask(task$1, `the task ${plClient.stringifyWithResourceId(task$1.info())} was released when the driver was closed`);
+        }));
+    }
+    setNewTask(w, url, callerId) {
+        const result = new task.DownloadByUrlTask(this.logger, this.getFilePath(url), url, this.signer, this.saveDir);
+        result.attach(w, callerId);
+        this.urlToDownload.set(url.toString(), result);
+        return result;
+    }
+    removeTask(task, reason) {
+        task.abort(reason);
+        task.change.markChanged(`task for url ${task.url} removed: ${reason}`);
+        this.urlToDownload.delete(task.url.toString());
+    }
+    getFilePath(url) {
+        const sha256 = node_crypto.createHash('sha256').update(url.toString()).digest('hex');
+        return path__namespace.join(this.saveDir, sha256);
+    }
+}
+
+exports.DownloadUrlDriver = DownloadUrlDriver;
+//# sourceMappingURL=driver.cjs.map

package/dist/drivers/download_url/driver.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.cjs","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => true,\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","randomUUID","url","isBlockUIURL","getPathForBlockUIURL","task","rmRFDir","stringifyWithResourceId","DownloadByUrlTask","createHash","path"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAIA,uBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAIC,sBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,IAAI;AACvC,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAACC,KAAW,EAAA;AAC3B,QAAA,IAAI,CAACC,0BAAY,CAACD,KAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsBA,KAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAOE,wBAAoB,CAAC,IAAI,CAAC,MAAM,EAAEF,KAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAMG,MAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAIA,MAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAOA,MAAuB,KAAI;AAC7C,gBAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyBE,gCAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAGF,MAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAEA,MAAI,CAAC,KAAI;AAChE,YAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAIG,sBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAGC,sBAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}

package/dist/drivers/download_url/driver.d.ts

@@ -60,4 +60,3 @@ export declare class DownloadUrlDriver implements DownloadUrlSyncReader, Fronten
     private removeTask;
     private getFilePath;
 }
-//# sourceMappingURL=driver.d.ts.map

package/dist/drivers/download_url/driver.js

@@ -0,0 +1,128 @@
+import { Computable } from '@milaboratories/computable';
+import { TaskProcessor } from '@milaboratories/ts-helpers';
+import { randomUUID, createHash } from 'node:crypto';
+import * as path from 'node:path';
+import { RemoteFileDownloader } from '../../helpers/download.js';
+import { FilesCache } from '../helpers/files_cache.js';
+import { stringifyWithResourceId } from '@milaboratories/pl-client';
+import { isBlockUIURL } from '@milaboratories/pl-model-common';
+import { getPathForBlockUIURL } from '../urls/url.js';
+import { rmRFDir, DownloadByUrlTask } from './task.js';
+
+/** Downloads .tar or .tar.gz archives by given URLs
+ * and extracts them into saveDir. */
+class DownloadUrlDriver {
+    logger;
+    saveDir;
+    signer;
+    opts;
+    downloadHelper;
+    urlToDownload = new Map();
+    downloadQueue;
+    /** Writes and removes files to a hard drive and holds a counter for every
+     * file that should be kept. */
+    cache;
+    constructor(logger, httpClient, saveDir, signer, opts = {
+        cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB
+        withGunzip: true,
+        nConcurrentDownloads: 50,
+    }) {
+        this.logger = logger;
+        this.saveDir = saveDir;
+        this.signer = signer;
+        this.opts = opts;
+        this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);
+        this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);
+        this.downloadHelper = new RemoteFileDownloader(httpClient);
+    }
+    /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */
+    getUrl(url, ctx) {
+        // wrap result as computable, if we were not given an existing computable context
+        if (ctx === undefined)
+            return Computable.make((c) => this.getUrl(url, c));
+        const callerId = randomUUID();
+        // read as ~ golang's defer
+        ctx.addOnDestroy(() => this.releasePath(url, callerId));
+        const result = this.getUrlNoCtx(url, ctx.watcher, callerId);
+        if (result?.url === undefined)
+            ctx.markUnstable(`a path to the downloaded and untared archive might be undefined. The current result: ${result}`);
+        return result;
+    }
+    getUrlNoCtx(url, w, callerId) {
+        const key = url.toString();
+        const task = this.urlToDownload.get(key);
+        if (task !== undefined) {
+            task.attach(w, callerId);
+            return task.getUrl();
+        }
+        const newTask = this.setNewTask(w, url, callerId);
+        this.downloadQueue.push({
+            fn: async () => this.downloadUrl(newTask, callerId),
+            recoverableErrorPredicate: (e) => true,
+        });
+        return newTask.getUrl();
+    }
+    getPathForBlockUI(url) {
+        if (!isBlockUIURL(url)) {
+            throw new Error(`getPathForBlockUI: ${url} is invalid`);
+        }
+        return getPathForBlockUIURL(this.signer, url, this.saveDir);
+    }
+    /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
+    async downloadUrl(task, callerId) {
+        await task.download(this.downloadHelper, this.opts.withGunzip);
+        // Might be undefined if a error happened
+        if (task.getUrl()?.url !== undefined)
+            this.cache.addCache(task, callerId);
+    }
+    /** Removes a directory and aborts a downloading task when all callers
+     * are not interested in it. */
+    async releasePath(url, callerId) {
+        const key = url.toString();
+        const task = this.urlToDownload.get(key);
+        if (task == undefined)
+            return;
+        if (this.cache.existsFile(task.path)) {
+            const toDelete = this.cache.removeFile(task.path, callerId);
+            await Promise.all(toDelete.map(async (task) => {
+                await rmRFDir(task.path);
+                this.cache.removeCache(task);
+                this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was removed`
+                    + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`);
+            }));
+        }
+        else {
+            // The task is still in a downloading queue.
+            const deleted = task.counter.dec(callerId);
+            if (deleted)
+                this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was removed from cache`);
+        }
+    }
+    /** Removes all files from a hard drive. */
+    async releaseAll() {
+        this.downloadQueue.stop();
+        await Promise.all(Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {
+            await rmRFDir(task.path);
+            this.cache.removeCache(task);
+            this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`);
+        }));
+    }
+    setNewTask(w, url, callerId) {
+        const result = new DownloadByUrlTask(this.logger, this.getFilePath(url), url, this.signer, this.saveDir);
+        result.attach(w, callerId);
+        this.urlToDownload.set(url.toString(), result);
+        return result;
+    }
+    removeTask(task, reason) {
+        task.abort(reason);
+        task.change.markChanged(`task for url ${task.url} removed: ${reason}`);
+        this.urlToDownload.delete(task.url.toString());
+    }
+    getFilePath(url) {
+        const sha256 = createHash('sha256').update(url.toString()).digest('hex');
+        return path.join(this.saveDir, sha256);
+    }
+}
+
+export { DownloadUrlDriver };
+//# sourceMappingURL=driver.js.map

package/dist/drivers/download_url/driver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.js","sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { createHash, randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport type { Dispatcher } from 'undici';\nimport { RemoteFileDownloader } from '../../helpers/download';\nimport { FilesCache } from '../helpers/files_cache';\nimport { stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';\nimport { isBlockUIURL } from '@milaboratories/pl-model-common';\nimport { getPathForBlockUIURL } from '../urls/url';\nimport { DownloadByUrlTask, rmRFDir } from './task';\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(\n    url: URL,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => true,\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([id, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash('sha256').update(url.toString()).digest('hex');\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;;AAmDA;AACqC;MACxB,iBAAiB,CAAA;AAWT,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,IAAA;AAdF,IAAA,cAAc;AAEvB,IAAA,aAAa,GAAmC,IAAI,GAAG,EAAE;AACzD,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EACjC,UAAsB,EACL,OAAe,EACf,MAAc,EACd,IAAA,GAA6B;QAC5C,kBAAkB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAC1C,QAAA,UAAU,EAAE,IAAI;AAChB,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QARgB,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,IAAI,GAAJ,IAAI;AAMrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACnF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;IAC5D;;IASA,MAAM,CACJ,GAAQ,EACR,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAEzE,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;;AAG7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AAEvD,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,wFAAwF,MAAM,CAAA,CAAE,CACjG;AAEH,QAAA,OAAO,MAAM;IACf;AAEA,IAAA,WAAW,CAAC,GAAQ,EAAE,CAAU,EAAE,QAAgB,EAAA;AAChD,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAExC,QAAA,IAAI,IAAI,KAAK,SAAS,EAAE;AACtB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC;AACjD,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnD,YAAA,yBAAyB,EAAE,CAAC,CAAC,KAAK,IAAI;AACvC,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;AAEA,IAAA,iBAAiB,CAAC,GAAW,EAAA;AAC3B,QAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAA,WAAA,CAAa,CAAC;QACzD;AAEA,QAAA,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;IAC7D;;AAGA,IAAA,MAAM,WAAW,CAAC,IAAuB,EAAE,QAAgB,EAAA;AACzD,QAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;;AAE9D,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC3E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,GAAQ,EAAE,QAAgB,EAAA;AAC1C,QAAA,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QACxC,IAAI,IAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAuB,KAAI;AAC7C,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyB,uBAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,KAAI;AAChE,YAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACb,IAAI,EACJ,CAAA,SAAA,EAAY,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,GAAQ,EAAE,QAAgB,EAAA;QACvD,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EACrB,GAAG,EACH,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CACb;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC;AAE9C,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAAuB,EAAE,MAAc,EAAA;AACxD,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAE,CAAC;AACtE,QAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD;AAEQ,IAAA,WAAW,CAAC,GAAQ,EAAA;AAC1B,QAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACxE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;IACxC;AACD;;;;"}