npm - @milaboratories/pl-drivers - Versions diffs - 1.16.1 → 1.16.3 - Mend

@milaboratories/pl-drivers 1.16.1 → 1.16.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (239) hide show

package/dist/drivers/download_blob_url/task.cjs CHANGED Viewed

@@ -8,20 +8,27 @@ let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_stream = require("node:stream");
 let _milaboratories_computable = require("@milaboratories/computable");
 let fs = require("fs");
-fs = require_runtime.__toESM(fs);
+fs = require_runtime.__toESM(fs, 1);
 let node_zlib = require("node:zlib");
-node_zlib = require_runtime.__toESM(node_zlib);
+node_zlib = require_runtime.__toESM(node_zlib, 1);
 let tar_fs = require("tar-fs");
-tar_fs = require_runtime.__toESM(tar_fs);
+tar_fs = require_runtime.__toESM(tar_fs, 1);
 let path = require("path");
-path = require_runtime.__toESM(path);
+path = require_runtime.__toESM(path, 1);
 let fs_promises = require("fs/promises");
-fs_promises = require_runtime.__toESM(fs_promises);
+fs_promises = require_runtime.__toESM(fs_promises, 1);
 let decompress = require("decompress");
-decompress = require_runtime.__toESM(decompress);
+decompress = require_runtime.__toESM(decompress, 1);
 //#region src/drivers/download_blob_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadAndUnarchiveTask = class {
+	logger;
+	signer;
+	saveDir;
+	path;
+	rInfo;
+	format;
+	clientDownload;
 	counter = new _milaboratories_ts_helpers.CallersCounter();
 	change = new _milaboratories_computable.ChangeSource();
 	signalCtl = new AbortController();
@@ -154,6 +161,7 @@ function nonRecoverableError(e) {
 }
 //#endregion
 exports.DownloadAndUnarchiveTask = DownloadAndUnarchiveTask;
+exports.nonRecoverableError = nonRecoverableError;
 exports.rmRFDir = rmRFDir;
 //# sourceMappingURL=task.cjs.map

package/dist/drivers/download_blob_url/task.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,UAAmB,IAAIA,2BAAAA,gBAAgB;CACvC,SAAkB,IAAIC,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,QACA,OACA,QACA,gBACA;AAPiB,OAAA,SAAA;AACA,OAAA,SAAA;AACR,OAAA,UAAA;AACA,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,SAAA;AACQ,OAAA,iBAAA;;;CAInB,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,oBAAkD,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,8BAAA,GAAA,0BAAA,yBAAqD,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,0BAAA,GAAA,0BAAA,yBAAiD,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,oBAAkD,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAA,QAAK,QAAQ,KAAK,KAAK;AAC9C,SAAA,GAAA,2BAAA,iBAAsB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAME,YAAI,MAAM,MAAM;MACtB,MAAM,cAAcC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAMF,YAAI,MAAM,MAAM;MACtB,MAAM,SAASG,YAAAA,UAAU,MAAMC,UAAK,cAAc,CAAC;MACnD,MAAM,QAAQH,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,YAAAA,SAAS,MAAM,GAAA,QAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,aAAA,GAAA,WAAA,SAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAA,QAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,EAAA,GAAA,qBAAA,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,MAAA,GAAA,2BAAA,UAAc,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAMI,YAAAA,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAA,QAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMA,YAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,QAAc;AAC1C,OAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACbO,wBAAAA,0BAA0B,EAAE,IAC5B,aAAaC,iBAAAA,uBACb,aAAaC,iBAAAA,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
1	+ {"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CAWjB;CACA;CACR;CACA;CACA;CACA;CACQ;CAhBnB,UAAmB,IAAIA,2BAAAA,eAAe;CACtC,SAAkB,IAAIC,2BAAAA,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,QACA,OACA,QACA,gBACA;EAPiB,KAAA,SAAA;EACA,KAAA,SAAA;EACR,KAAA,UAAA;EACA,KAAA,OAAA;EACA,KAAA,QAAA;EACA,KAAA,SAAA;EACQ,KAAA,iBAAA;CAChB;;CAGH,OAAc;EACZ,OAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;EACpB;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,WAAW;EACf,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,MAAM;GACnE,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,mBAAA,CAAkD,KAAK,MAAM,EAAE,EAAE,UACnE;GAEA,KAAK,OAAO,KAAK,8BAAA,GAAA,0BAAA,wBAAA,CAAqD,KAAK,KAAK,CAAC,GAAG;EACtF,SAAS,GAAQ;GACf,KAAK,OAAO,KACV,yBAAyB,EAAE,0BAAA,GAAA,0BAAA,wBAAA,CAAiD,KAAK,KAAK,CAAC,GACzF;GAEA,IAAI,oBAAoB,CAAC,GAAG;IAC1B,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,mBAAA,CAAkD,KAAK,MAAM,EAAE,EAAE,QACnE;IAEA,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;;CAGA,MAAc,sBAAsB,QAAsC;EACxE,KAAK,QAAQ,CAAC;EAEd,KAAK,MAAM,YAAY,KAAA,QAAK,QAAQ,KAAK,IAAI;EAC7C,OAAA,GAAA,2BAAA,gBAAA,CAAsB,KAAK,MAAM,SAAS;EAE1C,KAAK,MAAM,cAAc,OAAA,GAAA,2BAAA,WAAA,CAAiB,KAAK,IAAI;EACnD,IAAI,KAAK,MAAM,aACb,OAAO,MAAM,QAAQ,KAAK,IAAI;EA4DhC,OAAO,MAzDY,KAAK,eAAe,gBACrC,KAAK,OACL,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,KAAK,MAAO,aAAa;GAEzB,OAAA,GAAA,2BAAA,qBAAA,CAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;IAC1E,KAAK,MAAO,WAAW;IACvB,KAAK,MAAO,gBAAgB,KAAK;IAEjC,QAAQ,KAAK,QAAb;KACE,KAAK;MACH,MAAME,YAAI,MAAM,KAAK;MACrB,MAAM,cAAcC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,KAAK,CAAC;MACrD,MAAM,QAAQ,OAAO,aAAa,EAAE,OAAO,CAAC;MAC5C;KAEF,KAAK;MACH,MAAMF,YAAI,MAAM,KAAK;MACrB,MAAM,SAASG,YAAAA,UAAU,MAAMC,UAAK,aAAa,CAAC;MAClD,MAAM,QAAQH,YAAAA,SAAS,MAAMC,OAAI,QAAQ,KAAK,CAAC;MAE/C,MAAM,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,EAAE,OAAO,CAAC;MACtE;KAEF,KAAK;MACH,KAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,YAAAA,SAAS,MAAM,GAAA,QAAG,kBAAkB,KAAK,MAAO,OAAO,CAAC;MAClE,MAAM,QAAQ,OAAO,GAAG,EAAE,OAAO,CAAC;MAClC,KAAK,MAAO,iBAAiB;MAM7B,OAAA,GAAA,WAAA,QAAA,CAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,GAAG,EAC3C,CAAC;MACD,KAAK,MAAO,kBAAkB;MAE9B,MAAM,GAAA,QAAG,SAAS,GAAG,KAAK,MAAO,OAAO;MACxC,KAAK,MAAO,iBAAiB;MAE7B;KAEF,SACE,CAAA,GAAA,qBAAA,YAAA,CAAY,KAAK,MAAM;IAC3B;GACF,CAAC;GAED,KAAK,MAAO,cAAc;GAC1B,OAAO;EACT,CACF;CAGF;CAEA,SAAgC;EAC9B,IAAI,KAAK,MAAM,OAAO,EAAE,MAAA,GAAA,2BAAA,SAAA,CAAc,KAAK,GAAG,EAAE;EAEhD,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,MAAMI,YAAAA,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,IAAI;CAC9D;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAA,QAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADYA,YAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,QAAc;CAC1C,MAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD;;AAiBA,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;AACT;AAEA,SAAgB,oBAAoB,GAAQ;CAC1C,OACE,aAAa,cACbO,wBAAAA,0BAA0B,CAAC,KAC3B,aAAaC,iBAAAA,uBACb,aAAaC,iBAAAA,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,CAAC,CAAC,CAAC,SAAS,wBAAwB;AAE/C"}

package/dist/drivers/download_blob_url/task.d.ts CHANGED Viewed

@@ -60,5 +60,5 @@ type DownloadCtx = {
   pathCreated?: boolean;
 };
 //#endregion
-export { DownloadAndUnarchiveTask };
+export { DownloadAndUnarchiveTask, URLResult };
 //# sourceMappingURL=task.d.ts.map

package/dist/drivers/download_blob_url/task.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"mappings":";;;;;;;KA6BY,SAAA;EACV,GAAA,GAAM,~~SAAA~~;~~EACN~~,KAAA;AAAA;;cAIW,wBAAA;EAAA,iBAWQ,MAAA;EAAA,iBACA,MAAA;EAAA,SACR,OAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA,EAAO,wBAAA;EAAA,SACP,MAAA,EAAQ,aAAA;EAAA,iBACA,cAAA;EAAA,SAhBV,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;EAAA,QACQ,GAAA;EAAA,QACA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EAChB,OAAA,UACA,IAAA,UACA,KAAA,EAAO,wBAAA,EACP,MAAA,EAAQ,aAAA,EACA,cAAA,EAAgB,cAAA;;EAI5B,IAAA~~,CAAA~~;;;;;;;;;EAYP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,~~CAAA~~,~~GAAQ,~~OAAA;EAvBI;EAAA,QAoDJ,qBAAA;EAuEd,MAAA,~~CAAA~~,~~GAAU,~~SAAA;EAAA,QAQF,OAAA;EAAA,QAMA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA;;KA4BH,WAAA;EACH,SAAA;EACA,WAAA;EACA,UAAA;EACA,aAAA,GAAgB,~~aAAA~~;~~EAChB~~,QAAA;EACA,OAAA;EACA,cAAA;EACA,eAAA;EACA,cAAA;EACA,WAAA;AAAA"}
1	+ {"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"mappings":";;;;;;;KA6BY,SAAA;EACV,GAAA,GAAM,SAAS;EACf,KAAA;AAAA;;cAIW,wBAAA;EAAA,iBAWQ,MAAA;EAAA,iBACA,MAAA;EAAA,SACR,OAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA,EAAO,wBAAA;EAAA,SACP,MAAA,EAAQ,aAAA;EAAA,iBACA,cAAA;EAAA,SAhBV,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;EAAA,QACQ,GAAA;EAAA,QACA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EAChB,OAAA,UACA,IAAA,UACA,KAAA,EAAO,wBAAA,EACP,MAAA,EAAQ,aAAA,EACA,cAAA,EAAgB,cAAA;;EAI5B,IAAA;;;;;;;;;EAYP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,IAAQ,OAAA;EAvBI;EAAA,QAoDJ,qBAAA;EAuEd,MAAA,IAAU,SAAA;EAAA,QAQF,OAAA;EAAA,QAMA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA;;KA4BH,WAAA;EACH,SAAA;EACA,WAAA;EACA,UAAA;EACA,aAAA,GAAgB,aAAa;EAC7B,QAAA;EACA,OAAA;EACA,cAAA;EACA,eAAA;EACA,cAAA;EACA,WAAA;AAAA"}

package/dist/drivers/download_blob_url/task.js CHANGED Viewed

@@ -15,6 +15,13 @@ import decompress from "decompress";
 //#region src/drivers/download_blob_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadAndUnarchiveTask = class {
+	logger;
+	signer;
+	saveDir;
+	path;
+	rInfo;
+	format;
+	clientDownload;
 	counter = new CallersCounter();
 	change = new ChangeSource();
 	signalCtl = new AbortController();
@@ -146,6 +153,6 @@ function nonRecoverableError(e) {
 	return e instanceof URLAborted || isDownloadNetworkError400(e) || e instanceof UnknownStorageError || e instanceof WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED") || String(e).includes("incorrect header check");
 }
 //#endregion
-export { DownloadAndUnarchiveTask, rmRFDir };
+export { DownloadAndUnarchiveTask, nonRecoverableError, rmRFDir };
 //# sourceMappingURL=task.js.map

package/dist/drivers/download_blob_url/task.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,MACA,OACA,QACA,gBACA;AAPiB,OAAA,SAAA;AACA,OAAA,SAAA;AACR,OAAA,UAAA;AACA,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,SAAA;AACQ,OAAA,iBAAA;;;CAInB,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
1	+ {"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CAWjB;CACA;CACR;CACA;CACA;CACA;CACQ;CAhBnB,UAAmB,IAAI,eAAe;CACtC,SAAkB,IAAI,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,MACA,OACA,QACA,gBACA;EAPiB,KAAA,SAAA;EACA,KAAA,SAAA;EACR,KAAA,UAAA;EACA,KAAA,OAAA;EACA,KAAA,QAAA;EACA,KAAA,SAAA;EACQ,KAAA,iBAAA;CAChB;;CAGH,OAAc;EACZ,OAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;EACpB;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,WAAW;EACf,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,MAAM;GACnE,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,EAAE,EAAE,UACnE;GAEA,KAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,KAAK,CAAC,GAAG;EACtF,SAAS,GAAQ;GACf,KAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,KAAK,CAAC,GACzF;GAEA,IAAI,oBAAoB,CAAC,GAAG;IAC1B,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,EAAE,EAAE,QACnE;IAEA,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;;CAGA,MAAc,sBAAsB,QAAsC;EACxE,KAAK,QAAQ,CAAC;EAEd,KAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,IAAI;EAC7C,MAAM,gBAAgB,KAAK,MAAM,SAAS;EAE1C,KAAK,MAAM,cAAc,MAAM,WAAW,KAAK,IAAI;EACnD,IAAI,KAAK,MAAM,aACb,OAAO,MAAM,QAAQ,KAAK,IAAI;EA4DhC,OAAO,MAzDY,KAAK,eAAe,gBACrC,KAAK,OACL,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,KAAK,MAAO,aAAa;GAEzB,MAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;IAC1E,KAAK,MAAO,WAAW;IACvB,KAAK,MAAO,gBAAgB,KAAK;IAEjC,QAAQ,KAAK,QAAb;KACE,KAAK;MACH,MAAM,IAAI,MAAM,KAAK;MACrB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,KAAK,CAAC;MACrD,MAAM,QAAQ,OAAO,aAAa,EAAE,OAAO,CAAC;MAC5C;KAEF,KAAK;MACH,MAAM,IAAI,MAAM,KAAK;MACrB,MAAM,SAAS,UAAU,MAAM,KAAK,aAAa,CAAC;MAClD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,KAAK,CAAC;MAE/C,MAAM,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,EAAE,OAAO,CAAC;MACtE;KAEF,KAAK;MACH,KAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,OAAO,CAAC;MAClE,MAAM,QAAQ,OAAO,GAAG,EAAE,OAAO,CAAC;MAClC,KAAK,MAAO,iBAAiB;MAM7B,MAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,GAAG,EAC3C,CAAC;MACD,KAAK,MAAO,kBAAkB;MAE9B,MAAM,GAAG,SAAS,GAAG,KAAK,MAAO,OAAO;MACxC,KAAK,MAAO,iBAAiB;MAE7B;KAEF,SACE,YAAY,KAAK,MAAM;IAC3B;GACF,CAAC;GAED,KAAK,MAAO,cAAc;GAC1B,OAAO;EACT,CACF;CAGF;CAEA,SAAgC;EAC9B,IAAI,KAAK,MAAM,OAAO,EAAE,KAAK,SAAS,KAAK,GAAG,EAAE;EAEhD,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,IAAI;CAC9D;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADY,IAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,MAAc;CAC1C,MAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD;;AAiBA,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;AACT;AAEA,SAAgB,oBAAoB,GAAQ;CAC1C,OACE,aAAa,cACb,0BAA0B,CAAC,KAC3B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,CAAC,CAAC,CAAC,SAAS,wBAAwB;AAE/C"}

package/dist/drivers/download_url/driver.cjs CHANGED Viewed

@@ -7,7 +7,7 @@ const require_task = require("./task.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_path = require("node:path");
-node_path = require_runtime.__toESM(node_path);
+node_path = require_runtime.__toESM(node_path, 1);
 let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let node_crypto = require("node:crypto");
@@ -15,8 +15,12 @@ let node_crypto = require("node:crypto");
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
 var DownloadUrlDriver = class {
+	logger;
+	saveDir;
+	signer;
+	opts;
 	downloadHelper;
-	urlToDownload = /* @__PURE__ */ new Map();
+	taskByKey = /* @__PURE__ */ new Map();
 	downloadQueue;
 	/** Writes and removes files to a hard drive and holds a counter for every
 	* file that should be kept. */
@@ -37,20 +41,37 @@ var DownloadUrlDriver = class {
 	/** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */
 	getUrl(url, ctx) {
 		if (ctx === void 0) return _milaboratories_computable.Computable.make((c) => this.getUrl(url, c));
+		return this.getResult({
+			type: "url",
+			url
+		}, url.toString(), ctx);
+	}
+	/** Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it
+	* into the recycled cache (so a live UI keeps the folder pinned) and returns
+	* a custom-protocol URL. Keyed on `path + ":" + mtime`, so the unpacked UI is
+	* re-derived when the source block changes. */
+	getLocalTgz(path, mtime, ctx) {
+		if (ctx === void 0) return _milaboratories_computable.Computable.make((c) => this.getLocalTgz(path, mtime, c));
+		return this.getResult({
+			type: "local-tgz",
+			path,
+			mtime
+		}, `${path}:${mtime}`, ctx);
+	}
+	getResult(source, key, ctx) {
 		const callerId = (0, node_crypto.randomUUID)();
-		ctx.addOnDestroy(() => this.releasePath(url, callerId));
-		const result = this.getUrlNoCtx(url, ctx.watcher, callerId);
+		ctx.addOnDestroy(() => this.releaseByKey(key, callerId));
+		const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);
 		if (result?.url === void 0) ctx.markUnstable(`a path to the downloaded and untared archive might be undefined. The current result: ${result}`);
 		return result;
 	}
-	getUrlNoCtx(url, w, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	getResultNoCtx(source, key, w, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task !== void 0) {
 			task.attach(w, callerId);
 			return task.getUrl();
 		}
-		const newTask = this.setNewTask(w, url, callerId);
+		const newTask = this.setNewTask(w, source, key, callerId);
 		this.downloadQueue.push({
 			fn: async () => this.downloadUrl(newTask, callerId),
 			recoverableErrorPredicate: (e) => !(e instanceof require_task.URLAborted) && !require_download_errors.isDownloadNetworkError400(e)
@@ -68,9 +89,8 @@ var DownloadUrlDriver = class {
 	}
 	/** Removes a directory and aborts a downloading task when all callers
 	* are not interested in it. */
-	async releasePath(url, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	async releaseByKey(key, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task == void 0) return;
 		if (this.cache.existsFile(task.path)) {
 			const toDelete = this.cache.removeFile(task.path, callerId);
@@ -84,25 +104,27 @@ var DownloadUrlDriver = class {
 	/** Removes all files from a hard drive. */
 	async releaseAll() {
 		this.downloadQueue.stop();
-		await Promise.all(Array.from(this.urlToDownload.entries()).map(async ([, task]) => {
+		await Promise.all(Array.from(this.taskByKey.entries()).map(async ([, task]) => {
 			await require_task.rmRFDir(task.path);
 			this.cache.removeCache(task);
 			this.removeTask(task, `the task ${(0, _milaboratories_pl_client.stringifyWithResourceId)(task.info())} was released when the driver was closed`);
 		}));
 	}
-	setNewTask(w, url, callerId) {
-		const result = new require_task.DownloadByUrlTask(this.logger, this.getFilePath(url), url, this.signer, this.saveDir);
+	setNewTask(w, source, key, callerId) {
+		const result = new require_task.DownloadByUrlTask(this.logger, this.getExtractionPath(key), source, key, this.signer, this.saveDir);
 		result.attach(w, callerId);
-		this.urlToDownload.set(url.toString(), result);
+		this.taskByKey.set(key, result);
 		return result;
 	}
 	removeTask(task, reason) {
 		task.abort(reason);
-		task.change.markChanged(`task for url ${task.url} removed: ${reason}`);
-		this.urlToDownload.delete(task.url.toString());
+		task.change.markChanged(`task for ${task.key} removed: ${reason}`);
+		this.taskByKey.delete(task.key);
 	}
-	getFilePath(url) {
-		const sha256 = (0, node_crypto.createHash)("sha256").update(url.toString()).digest("hex");
+	/** Directory an archive (url or local file) is extracted into; keyed by a
+	* hash of the task key so the same source maps to a stable on-disk path. */
+	getExtractionPath(key) {
+		const sha256 = (0, node_crypto.createHash)("sha256").update(key).digest("hex");
 		return node_path.join(this.saveDir, sha256);
 	}
 };

package/dist/drivers/download_url/driver.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n /** Returns a Computable that (when the time will come)\n * downloads an archive from an URL,\n * extracts it to the local dir and returns a path to that dir. /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n}\n\nexport interface UrlResult {\n /* Path to the downloadable blob along with the signature and a custom protocol,\n * might be undefined when the error happened. /\n url?: BlockUIURL;\n /* Error that happened when the archive were downloaded. /\n error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n /* A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /* Whether to gunzip the downloaded archive (it will be untared automatically). /\n withGunzip: boolean;\n\n /* Max number of concurrent downloads while calculating computable states\n * derived from this driver.\n * /\n nConcurrentDownloads: number;\n};\n\n/* Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. /\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n private readonly downloadHelper: RemoteFileDownloader;\n\n private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadByUrlTask>;\n\n constructor(\n private readonly logger: MiLogger,\n httpClient: Dispatcher,\n private readonly saveDir: string,\n private readonly signer: Signer,\n private readonly opts: DownloadUrlDriverOps = {\n cacheSoftSizeBytes: 1 1024 * 1024 * 1024, // 1 GB\n withGunzip: true,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n this.downloadHelper = new RemoteFileDownloader(httpClient);\n }\n\n /** Use to get a path result inside a computable context /\n getUrl(url: URL, ctx: ComputableCtx): UrlResult \| undefined;\n\n /* Returns a Computable that do the work /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n\n /* Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. /\n getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n const callerId = randomUUID();\n\n // read as ~ golang's defer\n ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n );\n\n return result;\n }\n\n getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n\n if (task !== undefined) {\n task.attach(w, callerId);\n return task.getUrl();\n }\n\n const newTask = this.setNewTask(w, url, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n });\n\n return newTask.getUrl();\n }\n\n getPathForBlockUI(url: string): string {\n if (!isBlockUIURL(url)) {\n throw new Error(`getPathForBlockUI: ${url} is invalid`);\n }\n\n return getPathForBlockUIURL(this.signer, url, this.saveDir);\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n await task.download(this.downloadHelper, this.opts.withGunzip);\n // Might be undefined if a error happened\n if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(url: URL, callerId: string): Promise<void> {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadByUrlTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(w: Watcher, url: URL, callerId: string) {\n const result = new DownloadByUrlTask(\n this.logger,\n this.getFilePath(url),\n url,\n this.signer,\n this.saveDir,\n );\n result.attach(w, callerId);\n this.urlToDownload.set(url.toString(), result);\n\n return result;\n }\n\n private removeTask(task: DownloadByUrlTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n this.urlToDownload.delete(task.url.toString());\n }\n\n private getFilePath(url: URL): string {\n const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n return path.join(this.saveDir, sha256);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E;CAEA,gCAAwD,IAAI,KAAK;CACjE;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;AATiB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,OAAA;AAMjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAIC,iBAAAA,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,KAAA,EAAW,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,YAAA,GAAA,YAAA,aAAuB;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,KAAA,GAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAaC,aAAAA,eAAe,CAACC,wBAAAA,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,EAAA,GAAA,gCAAA,cAAc,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAOC,YAAAA,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAMC,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WAAmB,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAIC,aAAAA,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,WAAmB,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,YAAoB,KAAkB;EACpC,MAAM,UAAA,GAAA,YAAA,YAAoB,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOC,UAAK,KAAK,KAAK,SAAS,OAAO"}
1	+ {"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport type { DownloadSource } from \"./task\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n /** Returns a Computable that (when the time will come)\n * downloads an archive from an URL,\n * extracts it to the local dir and returns a path to that dir. /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n}\n\nexport interface UrlResult {\n /* Path to the downloadable blob along with the signature and a custom protocol,\n * might be undefined when the error happened. /\n url?: BlockUIURL;\n /* Error that happened when the archive were downloaded. /\n error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n /* A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /* Whether to gunzip the downloaded archive (it will be untared automatically). /\n withGunzip: boolean;\n\n /* Max number of concurrent downloads while calculating computable states\n * derived from this driver.\n * /\n nConcurrentDownloads: number;\n};\n\n/* Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. /\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n private readonly downloadHelper: RemoteFileDownloader;\n\n private taskByKey: Map<string, DownloadByUrlTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadByUrlTask>;\n\n constructor(\n private readonly logger: MiLogger,\n httpClient: Dispatcher,\n private readonly saveDir: string,\n private readonly signer: Signer,\n private readonly opts: DownloadUrlDriverOps = {\n cacheSoftSizeBytes: 1 1024 * 1024 * 1024, // 1 GB\n withGunzip: true,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n this.downloadHelper = new RemoteFileDownloader(httpClient);\n }\n\n /** Use to get a path result inside a computable context /\n getUrl(url: URL, ctx: ComputableCtx): UrlResult \| undefined;\n\n /* Returns a Computable that do the work /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n\n /* Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. /\n getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n return this.getResult({ type: \"url\", url }, url.toString(), ctx);\n }\n\n getLocalTgz(path: string, mtime: string, ctx: ComputableCtx): UrlResult \| undefined;\n getLocalTgz(path: string, mtime: string): Computable<UrlResult \| undefined>;\n\n /* Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it\n * into the recycled cache (so a live UI keeps the folder pinned) and returns\n * a custom-protocol URL. Keyed on `path + \":\" + mtime`, so the unpacked UI is\n * re-derived when the source block changes. /\n getLocalTgz(\n path: string,\n mtime: string,\n ctx?: ComputableCtx,\n ): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n if (ctx === undefined) return Computable.make((c) => this.getLocalTgz(path, mtime, c));\n return this.getResult({ type: \"local-tgz\", path, mtime }, `${path}:${mtime}`, ctx);\n }\n\n private getResult(\n source: DownloadSource,\n key: string,\n ctx: ComputableCtx,\n ): UrlResult \| undefined {\n const callerId = randomUUID();\n\n // read as ~ golang's defer\n ctx.addOnDestroy(() => this.releaseByKey(key, callerId));\n\n const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n );\n\n return result;\n }\n\n private getResultNoCtx(source: DownloadSource, key: string, w: Watcher, callerId: string) {\n const task = this.taskByKey.get(key);\n\n if (task !== undefined) {\n task.attach(w, callerId);\n return task.getUrl();\n }\n\n const newTask = this.setNewTask(w, source, key, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n });\n\n return newTask.getUrl();\n }\n\n getPathForBlockUI(url: string): string {\n if (!isBlockUIURL(url)) {\n throw new Error(`getPathForBlockUI: ${url} is invalid`);\n }\n\n return getPathForBlockUIURL(this.signer, url, this.saveDir);\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n await task.download(this.downloadHelper, this.opts.withGunzip);\n // Might be undefined if a error happened\n if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releaseByKey(key: string, callerId: string): Promise<void> {\n const task = this.taskByKey.get(key);\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadByUrlTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.taskByKey.entries()).map(async ([, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(w: Watcher, source: DownloadSource, key: string, callerId: string) {\n const result = new DownloadByUrlTask(\n this.logger,\n this.getExtractionPath(key),\n source,\n key,\n this.signer,\n this.saveDir,\n );\n result.attach(w, callerId);\n this.taskByKey.set(key, result);\n\n return result;\n }\n\n private removeTask(task: DownloadByUrlTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for ${task.key} removed: ${reason}`);\n this.taskByKey.delete(task.key);\n }\n\n /* Directory an archive (url or local file) is extracted into; keyed by a\n * hash of the task key so the same source maps to a stable on-disk path. */\n private getExtractionPath(key: string): string {\n const sha256 = createHash(\"sha256\").update(key).digest(\"hex\");\n return path.join(this.saveDir, sha256);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkDA,IAAa,oBAAb,MAAgF;CAW3D;CAEA;CACA;CACA;CAdnB;CAEA,4BAAoD,IAAI,IAAI;CAC5D;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;CACxB,GACA;EATiB,KAAA,SAAA;EAEA,KAAA,UAAA;EACA,KAAA,SAAA;EACA,KAAA,OAAA;EAMjB,KAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,oBAAoB;EAClF,KAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,kBAAkB;EACxD,KAAK,iBAAiB,IAAIC,iBAAAA,qBAAqB,UAAU;CAC3D;;CASA,OAAO,KAAU,KAAgF;EAE/F,IAAI,QAAQ,KAAA,GAAW,OAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,CAAC,CAAC;EACxE,OAAO,KAAK,UAAU;GAAE,MAAM;GAAO;EAAI,GAAG,IAAI,SAAS,GAAG,GAAG;CACjE;;;;;CASA,YACE,MACA,OACA,KAC2D;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAOA,2BAAAA,WAAW,MAAM,MAAM,KAAK,YAAY,MAAM,OAAO,CAAC,CAAC;EACrF,OAAO,KAAK,UAAU;GAAE,MAAM;GAAa;GAAM;EAAM,GAAG,GAAG,KAAK,GAAG,SAAS,GAAG;CACnF;CAEA,UACE,QACA,KACA,KACuB;EACvB,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAG5B,IAAI,mBAAmB,KAAK,aAAa,KAAK,QAAQ,CAAC;EAEvD,MAAM,SAAS,KAAK,eAAe,QAAQ,KAAK,IAAI,SAAS,QAAQ;EACrE,IAAI,QAAQ,QAAQ,KAAA,GAClB,IAAI,aACF,wFAAwF,QAC1F;EAEF,OAAO;CACT;CAEA,eAAuB,QAAwB,KAAa,GAAY,UAAkB;EACxF,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EAEnC,IAAI,SAAS,KAAA,GAAW;GACtB,KAAK,OAAO,GAAG,QAAQ;GACvB,OAAO,KAAK,OAAO;EACrB;EAEA,MAAM,UAAU,KAAK,WAAW,GAAG,QAAQ,KAAK,QAAQ;EACxD,KAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,QAAQ;GAClD,4BAA4B,MAAM,EAAE,aAAaC,aAAAA,eAAe,CAACC,wBAAAA,0BAA0B,CAAC;EAC9F,CAAC;EAED,OAAO,QAAQ,OAAO;CACxB;CAEA,kBAAkB,KAAqB;EACrC,IAAI,EAAA,GAAA,gCAAA,aAAA,CAAc,GAAG,GACnB,MAAM,IAAI,MAAM,sBAAsB,IAAI,YAAY;EAGxD,OAAOC,YAAAA,qBAAqB,KAAK,QAAQ,KAAK,KAAK,OAAO;CAC5D;;CAGA,MAAM,YAAY,MAAyB,UAAkB;EAC3D,MAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,UAAU;EAE7D,IAAI,KAAK,OAAO,CAAC,EAAE,QAAQ,KAAA,GAAW,KAAK,MAAM,SAAS,MAAM,QAAQ;CAC1E;;;CAIA,MAAM,aAAa,KAAa,UAAiC;EAC/D,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EACnC,IAAI,QAAQ,KAAA,GAAW;EAEvB,IAAI,KAAK,MAAM,WAAW,KAAK,IAAI,GAAG;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ;GAE1D,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;IAC9C,MAAMC,aAAAA,QAAQ,KAAK,IAAI;IACvB,KAAK,MAAM,YAAY,IAAI;IAE3B,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,qCAAA,GAAA,0BAAA,wBAAA,CACE,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,GAClF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,wBACnD;CAEN;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,MAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,UAAU,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;GAC3D,MAAMA,aAAAA,QAAQ,KAAK,IAAI;GACvB,KAAK,MAAM,YAAY,IAAI;GAE3B,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,yCACnD;EACF,CAAC,CACH;CACF;CAEA,WAAmB,GAAY,QAAwB,KAAa,UAAkB;EACpF,MAAM,SAAS,IAAIC,aAAAA,kBACjB,KAAK,QACL,KAAK,kBAAkB,GAAG,GAC1B,QACA,KACA,KAAK,QACL,KAAK,OACP;EACA,OAAO,OAAO,GAAG,QAAQ;EACzB,KAAK,UAAU,IAAI,KAAK,MAAM;EAE9B,OAAO;CACT;CAEA,WAAmB,MAAyB,QAAgB;EAC1D,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,YAAY,KAAK,IAAI,YAAY,QAAQ;EACjE,KAAK,UAAU,OAAO,KAAK,GAAG;CAChC;;;CAIA,kBAA0B,KAAqB;EAC7C,MAAM,UAAA,GAAA,YAAA,WAAA,CAAoB,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,KAAK;EAC5D,OAAOC,UAAK,KAAK,KAAK,SAAS,MAAM;CACvC;AACF"}

package/dist/drivers/download_url/driver.d.ts CHANGED Viewed

@@ -1,7 +1,7 @@
 import { DownloadByUrlTask } from "./task.js";
 import { Dispatcher } from "undici";
 import { MiLogger, Signer } from "@milaboratories/ts-helpers";
-import { Computable, ComputableCtx, Watcher } from "@milaboratories/computable";
+import { Computable, ComputableCtx } from "@milaboratories/computable";
 import { BlockUIURL, FrontendDriver } from "@milaboratories/pl-model-common";
 //#region src/drivers/download_url/driver.d.ts
@@ -38,7 +38,7 @@ declare class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver
   private readonly signer;
   private readonly opts;
   private readonly downloadHelper;
-  private urlToDownload;
+  private taskByKey;
   private downloadQueue;
   /** Writes and removes files to a hard drive and holds a counter for every
    * file that should be kept. */
@@ -48,18 +48,23 @@ declare class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver
   getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;
   /** Returns a Computable that do the work */
   getUrl(url: URL): Computable<UrlResult | undefined>;
-  getUrlNoCtx(url: URL, w: Watcher, callerId: string): UrlResult | undefined;
+  getLocalTgz(path: string, mtime: string, ctx: ComputableCtx): UrlResult | undefined;
+  getLocalTgz(path: string, mtime: string): Computable<UrlResult | undefined>;
+  private getResult;
+  private getResultNoCtx;
   getPathForBlockUI(url: string): string;
   /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
   downloadUrl(task: DownloadByUrlTask, callerId: string): Promise<void>;
   /** Removes a directory and aborts a downloading task when all callers
    * are not interested in it. */
-  releasePath(url: URL, callerId: string): Promise<void>;
+  releaseByKey(key: string, callerId: string): Promise<void>;
   /** Removes all files from a hard drive. */
   releaseAll(): Promise<void>;
   private setNewTask;
   private removeTask;
-  private getFilePath;
+  /** Directory an archive (url or local file) is extracted into; keyed by a
+   * hash of the task key so the same source maps to a stable on-disk path. */
+  private getExtractionPath;
 }
 //#endregion
 export { DownloadUrlDriver, DownloadUrlDriverOps, DownloadUrlSyncReader, UrlResult };

package/dist/drivers/download_url/driver.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_url/driver.ts"],"mappings":";;;;;;;~~UAgBiB~~,qBAAA;EAAA;;;EAIf,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;AAAA;AAAA,UAGd,SAAA;EAHa;;EAM5B,GAAA,GAAM,~~UAAA~~;~~EANM~~;EAQZ,KAAA;AAAA;AAAA,KAGU,oBAAA;EAX4B;;AAGxC;;EAaE,kBAAA,UAVgB;EAahB,UAAA;EAbM;;;EAkBN,oBAAA;AAAA;;;cAKW,iBAAA,YAA6B,qBAAA,EAAuB,cAAA;EAAA,iBAW5C,MAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,MAAA;EAAA,iBACA,IAAA;EAAA,iBAdF,cAAA;EAAA,QAET,~~aAAA~~;EAAA,QACA,aAAA;;;UAIA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACzB,UAAA,EAAY,UAAA,EACK,OAAA,UACA,MAAA,EAAQ,MAAA,EACR,IAAA,GAAM,oBAAA;EAAA;EAYzB,MAAA,CAAO,GAAA,EAAK,GAAA,EAAK,GAAA,EAAK,aAAA,GAAgB,SAAA;EAAhB;EAGtB,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;~~EAqB7B~~,WAAA,CAAY,~~GAAA~~,~~EAAK~~,GAAA,EAAK,~~CAAA~~,~~EAAG~~,~~OAAA~~,~~EAAS~~,~~QAAA~~,WAAgB,SAAA;~~EAkBlD~~,iBAAA,CAAkB,GAAA;~~EAvCA~~;~~EAgDZ~~,WAAA,CAAY,IAAA,EAAM,iBAAA,EAAmB,QAAA,WAAgB,OAAA;~~EA3BlC~~;;~~EAmCnB~~,~~WAAA~~,~~CAAY~~,GAAA,~~EAAK~~,~~GAAA,EAAK,~~QAAA,WAAmB,OAAA;~~EARY~~;~~EAwCrD~~,UAAA,~~CAAA~~,~~GAAU,~~OAAA;EAAA,QAgBR,UAAA;EAAA,~~QAcA~~,UAAA;EAAA,~~QAMA~~,~~WAAA~~;AAAA"}
1	+ {"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_url/driver.ts"],"mappings":";;;;;;;UAiBiB,qBAAA;EAAA;;;EAIf,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;AAAA;AAAA,UAGd,SAAA;EAHa;;EAM5B,GAAA,GAAM,UAAU;EANJ;EAQZ,KAAA;AAAA;AAAA,KAGU,oBAAA;EAX4B;AAAA;AAGxC;;EAaE,kBAAA,UAVgB;EAahB,UAAA;EAbM;;;EAkBN,oBAAA;AAAA;;;cAKW,iBAAA,YAA6B,qBAAA,EAAuB,cAAA;EAAA,iBAW5C,MAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,MAAA;EAAA,iBACA,IAAA;EAAA,iBAdF,cAAA;EAAA,QAET,SAAA;EAAA,QACA,aAAA;;;UAIA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACzB,UAAA,EAAY,UAAA,EACK,OAAA,UACA,MAAA,EAAQ,MAAA,EACR,IAAA,GAAM,oBAAA;EAAA;EAYzB,MAAA,CAAO,GAAA,EAAK,GAAA,EAAK,GAAA,EAAK,aAAA,GAAgB,SAAA;EAAhB;EAGtB,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;EAS7B,WAAA,CAAY,IAAA,UAAc,KAAA,UAAe,GAAA,EAAK,aAAA,GAAgB,SAAA;EAC9D,WAAA,CAAY,IAAA,UAAc,KAAA,WAAgB,UAAA,CAAW,SAAA;EAAA,QAe7C,SAAA;EAAA,QAmBA,cAAA;EAiBR,iBAAA,CAAkB,GAAA;EAnDmC;EA4D/C,WAAA,CAAY,IAAA,EAAM,iBAAA,EAAmB,QAAA,WAAgB,OAAA;EAAnC;;EAQlB,YAAA,CAAa,GAAA,UAAa,QAAA,WAAmB,OAAA;EA+BnC;EAAV,UAAA,IAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAeA,UAAA;EA1KqE;;EAAA,QAkLrE,iBAAA;AAAA"}

package/dist/drivers/download_url/driver.js CHANGED Viewed

@@ -13,8 +13,12 @@ import { createHash, randomUUID } from "node:crypto";
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
 var DownloadUrlDriver = class {
+	logger;
+	saveDir;
+	signer;
+	opts;
 	downloadHelper;
-	urlToDownload = /* @__PURE__ */ new Map();
+	taskByKey = /* @__PURE__ */ new Map();
 	downloadQueue;
 	/** Writes and removes files to a hard drive and holds a counter for every
 	* file that should be kept. */
@@ -35,20 +39,37 @@ var DownloadUrlDriver = class {
 	/** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */
 	getUrl(url, ctx) {
 		if (ctx === void 0) return Computable.make((c) => this.getUrl(url, c));
+		return this.getResult({
+			type: "url",
+			url
+		}, url.toString(), ctx);
+	}
+	/** Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it
+	* into the recycled cache (so a live UI keeps the folder pinned) and returns
+	* a custom-protocol URL. Keyed on `path + ":" + mtime`, so the unpacked UI is
+	* re-derived when the source block changes. */
+	getLocalTgz(path, mtime, ctx) {
+		if (ctx === void 0) return Computable.make((c) => this.getLocalTgz(path, mtime, c));
+		return this.getResult({
+			type: "local-tgz",
+			path,
+			mtime
+		}, `${path}:${mtime}`, ctx);
+	}
+	getResult(source, key, ctx) {
 		const callerId = randomUUID();
-		ctx.addOnDestroy(() => this.releasePath(url, callerId));
-		const result = this.getUrlNoCtx(url, ctx.watcher, callerId);
+		ctx.addOnDestroy(() => this.releaseByKey(key, callerId));
+		const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);
 		if (result?.url === void 0) ctx.markUnstable(`a path to the downloaded and untared archive might be undefined. The current result: ${result}`);
 		return result;
 	}
-	getUrlNoCtx(url, w, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	getResultNoCtx(source, key, w, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task !== void 0) {
 			task.attach(w, callerId);
 			return task.getUrl();
 		}
-		const newTask = this.setNewTask(w, url, callerId);
+		const newTask = this.setNewTask(w, source, key, callerId);
 		this.downloadQueue.push({
 			fn: async () => this.downloadUrl(newTask, callerId),
 			recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e)
@@ -66,9 +87,8 @@ var DownloadUrlDriver = class {
 	}
 	/** Removes a directory and aborts a downloading task when all callers
 	* are not interested in it. */
-	async releasePath(url, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	async releaseByKey(key, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task == void 0) return;
 		if (this.cache.existsFile(task.path)) {
 			const toDelete = this.cache.removeFile(task.path, callerId);
@@ -82,25 +102,27 @@ var DownloadUrlDriver = class {
 	/** Removes all files from a hard drive. */
 	async releaseAll() {
 		this.downloadQueue.stop();
-		await Promise.all(Array.from(this.urlToDownload.entries()).map(async ([, task]) => {
+		await Promise.all(Array.from(this.taskByKey.entries()).map(async ([, task]) => {
 			await rmRFDir(task.path);
 			this.cache.removeCache(task);
 			this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`);
 		}));
 	}
-	setNewTask(w, url, callerId) {
-		const result = new DownloadByUrlTask(this.logger, this.getFilePath(url), url, this.signer, this.saveDir);
+	setNewTask(w, source, key, callerId) {
+		const result = new DownloadByUrlTask(this.logger, this.getExtractionPath(key), source, key, this.signer, this.saveDir);
 		result.attach(w, callerId);
-		this.urlToDownload.set(url.toString(), result);
+		this.taskByKey.set(key, result);
 		return result;
 	}
 	removeTask(task, reason) {
 		task.abort(reason);
-		task.change.markChanged(`task for url ${task.url} removed: ${reason}`);
-		this.urlToDownload.delete(task.url.toString());
+		task.change.markChanged(`task for ${task.key} removed: ${reason}`);
+		this.taskByKey.delete(task.key);
 	}
-	getFilePath(url) {
-		const sha256 = createHash("sha256").update(url.toString()).digest("hex");
+	/** Directory an archive (url or local file) is extracted into; keyed by a
+	* hash of the task key so the same source maps to a stable on-disk path. */
+	getExtractionPath(key) {
+		const sha256 = createHash("sha256").update(key).digest("hex");
 		return path$1.join(this.saveDir, sha256);
 	}
 };

package/dist/drivers/download_url/driver.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n /** Returns a Computable that (when the time will come)\n * downloads an archive from an URL,\n * extracts it to the local dir and returns a path to that dir. /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n}\n\nexport interface UrlResult {\n /* Path to the downloadable blob along with the signature and a custom protocol,\n * might be undefined when the error happened. /\n url?: BlockUIURL;\n /* Error that happened when the archive were downloaded. /\n error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n /* A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /* Whether to gunzip the downloaded archive (it will be untared automatically). /\n withGunzip: boolean;\n\n /* Max number of concurrent downloads while calculating computable states\n * derived from this driver.\n * /\n nConcurrentDownloads: number;\n};\n\n/* Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. /\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n private readonly downloadHelper: RemoteFileDownloader;\n\n private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadByUrlTask>;\n\n constructor(\n private readonly logger: MiLogger,\n httpClient: Dispatcher,\n private readonly saveDir: string,\n private readonly signer: Signer,\n private readonly opts: DownloadUrlDriverOps = {\n cacheSoftSizeBytes: 1 1024 * 1024 * 1024, // 1 GB\n withGunzip: true,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n this.downloadHelper = new RemoteFileDownloader(httpClient);\n }\n\n /** Use to get a path result inside a computable context /\n getUrl(url: URL, ctx: ComputableCtx): UrlResult \| undefined;\n\n /* Returns a Computable that do the work /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n\n /* Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. /\n getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n const callerId = randomUUID();\n\n // read as ~ golang's defer\n ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n );\n\n return result;\n }\n\n getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n\n if (task !== undefined) {\n task.attach(w, callerId);\n return task.getUrl();\n }\n\n const newTask = this.setNewTask(w, url, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n });\n\n return newTask.getUrl();\n }\n\n getPathForBlockUI(url: string): string {\n if (!isBlockUIURL(url)) {\n throw new Error(`getPathForBlockUI: ${url} is invalid`);\n }\n\n return getPathForBlockUIURL(this.signer, url, this.saveDir);\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n await task.download(this.downloadHelper, this.opts.withGunzip);\n // Might be undefined if a error happened\n if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(url: URL, callerId: string): Promise<void> {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadByUrlTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(w: Watcher, url: URL, callerId: string) {\n const result = new DownloadByUrlTask(\n this.logger,\n this.getFilePath(url),\n url,\n this.signer,\n this.saveDir,\n );\n result.attach(w, callerId);\n this.urlToDownload.set(url.toString(), result);\n\n return result;\n }\n\n private removeTask(task: DownloadByUrlTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n this.urlToDownload.delete(task.url.toString());\n }\n\n private getFilePath(url: URL): string {\n const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n return path.join(this.saveDir, sha256);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E;CAEA,gCAAwD,IAAI,KAAK;CACjE;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;AATiB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,OAAA;AAMjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAI,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,KAAA,EAAW,QAAO,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,WAAW,YAAY;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,KAAA,GAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAa,eAAe,CAAC,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,CAAC,aAAa,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAO,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WAAmB,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAI,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,WAAmB,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,YAAoB,KAAkB;EACpC,MAAM,SAAS,WAAW,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOA,OAAK,KAAK,KAAK,SAAS,OAAO"}
1	+ {"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport type { DownloadSource } from \"./task\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n /** Returns a Computable that (when the time will come)\n * downloads an archive from an URL,\n * extracts it to the local dir and returns a path to that dir. /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n}\n\nexport interface UrlResult {\n /* Path to the downloadable blob along with the signature and a custom protocol,\n * might be undefined when the error happened. /\n url?: BlockUIURL;\n /* Error that happened when the archive were downloaded. /\n error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n /* A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /* Whether to gunzip the downloaded archive (it will be untared automatically). /\n withGunzip: boolean;\n\n /* Max number of concurrent downloads while calculating computable states\n * derived from this driver.\n * /\n nConcurrentDownloads: number;\n};\n\n/* Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. /\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n private readonly downloadHelper: RemoteFileDownloader;\n\n private taskByKey: Map<string, DownloadByUrlTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadByUrlTask>;\n\n constructor(\n private readonly logger: MiLogger,\n httpClient: Dispatcher,\n private readonly saveDir: string,\n private readonly signer: Signer,\n private readonly opts: DownloadUrlDriverOps = {\n cacheSoftSizeBytes: 1 1024 * 1024 * 1024, // 1 GB\n withGunzip: true,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n this.downloadHelper = new RemoteFileDownloader(httpClient);\n }\n\n /** Use to get a path result inside a computable context /\n getUrl(url: URL, ctx: ComputableCtx): UrlResult \| undefined;\n\n /* Returns a Computable that do the work /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n\n /* Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. /\n getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n return this.getResult({ type: \"url\", url }, url.toString(), ctx);\n }\n\n getLocalTgz(path: string, mtime: string, ctx: ComputableCtx): UrlResult \| undefined;\n getLocalTgz(path: string, mtime: string): Computable<UrlResult \| undefined>;\n\n /* Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it\n * into the recycled cache (so a live UI keeps the folder pinned) and returns\n * a custom-protocol URL. Keyed on `path + \":\" + mtime`, so the unpacked UI is\n * re-derived when the source block changes. /\n getLocalTgz(\n path: string,\n mtime: string,\n ctx?: ComputableCtx,\n ): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n if (ctx === undefined) return Computable.make((c) => this.getLocalTgz(path, mtime, c));\n return this.getResult({ type: \"local-tgz\", path, mtime }, `${path}:${mtime}`, ctx);\n }\n\n private getResult(\n source: DownloadSource,\n key: string,\n ctx: ComputableCtx,\n ): UrlResult \| undefined {\n const callerId = randomUUID();\n\n // read as ~ golang's defer\n ctx.addOnDestroy(() => this.releaseByKey(key, callerId));\n\n const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n );\n\n return result;\n }\n\n private getResultNoCtx(source: DownloadSource, key: string, w: Watcher, callerId: string) {\n const task = this.taskByKey.get(key);\n\n if (task !== undefined) {\n task.attach(w, callerId);\n return task.getUrl();\n }\n\n const newTask = this.setNewTask(w, source, key, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n });\n\n return newTask.getUrl();\n }\n\n getPathForBlockUI(url: string): string {\n if (!isBlockUIURL(url)) {\n throw new Error(`getPathForBlockUI: ${url} is invalid`);\n }\n\n return getPathForBlockUIURL(this.signer, url, this.saveDir);\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n await task.download(this.downloadHelper, this.opts.withGunzip);\n // Might be undefined if a error happened\n if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releaseByKey(key: string, callerId: string): Promise<void> {\n const task = this.taskByKey.get(key);\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadByUrlTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.taskByKey.entries()).map(async ([, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(w: Watcher, source: DownloadSource, key: string, callerId: string) {\n const result = new DownloadByUrlTask(\n this.logger,\n this.getExtractionPath(key),\n source,\n key,\n this.signer,\n this.saveDir,\n );\n result.attach(w, callerId);\n this.taskByKey.set(key, result);\n\n return result;\n }\n\n private removeTask(task: DownloadByUrlTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for ${task.key} removed: ${reason}`);\n this.taskByKey.delete(task.key);\n }\n\n /* Directory an archive (url or local file) is extracted into; keyed by a\n * hash of the task key so the same source maps to a stable on-disk path. */\n private getExtractionPath(key: string): string {\n const sha256 = createHash(\"sha256\").update(key).digest(\"hex\");\n return path.join(this.saveDir, sha256);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAkDA,IAAa,oBAAb,MAAgF;CAW3D;CAEA;CACA;CACA;CAdnB;CAEA,4BAAoD,IAAI,IAAI;CAC5D;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;CACxB,GACA;EATiB,KAAA,SAAA;EAEA,KAAA,UAAA;EACA,KAAA,SAAA;EACA,KAAA,OAAA;EAMjB,KAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,oBAAoB;EAClF,KAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,kBAAkB;EACxD,KAAK,iBAAiB,IAAI,qBAAqB,UAAU;CAC3D;;CASA,OAAO,KAAU,KAAgF;EAE/F,IAAI,QAAQ,KAAA,GAAW,OAAO,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,CAAC,CAAC;EACxE,OAAO,KAAK,UAAU;GAAE,MAAM;GAAO;EAAI,GAAG,IAAI,SAAS,GAAG,GAAG;CACjE;;;;;CASA,YACE,MACA,OACA,KAC2D;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAO,WAAW,MAAM,MAAM,KAAK,YAAY,MAAM,OAAO,CAAC,CAAC;EACrF,OAAO,KAAK,UAAU;GAAE,MAAM;GAAa;GAAM;EAAM,GAAG,GAAG,KAAK,GAAG,SAAS,GAAG;CACnF;CAEA,UACE,QACA,KACA,KACuB;EACvB,MAAM,WAAW,WAAW;EAG5B,IAAI,mBAAmB,KAAK,aAAa,KAAK,QAAQ,CAAC;EAEvD,MAAM,SAAS,KAAK,eAAe,QAAQ,KAAK,IAAI,SAAS,QAAQ;EACrE,IAAI,QAAQ,QAAQ,KAAA,GAClB,IAAI,aACF,wFAAwF,QAC1F;EAEF,OAAO;CACT;CAEA,eAAuB,QAAwB,KAAa,GAAY,UAAkB;EACxF,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EAEnC,IAAI,SAAS,KAAA,GAAW;GACtB,KAAK,OAAO,GAAG,QAAQ;GACvB,OAAO,KAAK,OAAO;EACrB;EAEA,MAAM,UAAU,KAAK,WAAW,GAAG,QAAQ,KAAK,QAAQ;EACxD,KAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,QAAQ;GAClD,4BAA4B,MAAM,EAAE,aAAa,eAAe,CAAC,0BAA0B,CAAC;EAC9F,CAAC;EAED,OAAO,QAAQ,OAAO;CACxB;CAEA,kBAAkB,KAAqB;EACrC,IAAI,CAAC,aAAa,GAAG,GACnB,MAAM,IAAI,MAAM,sBAAsB,IAAI,YAAY;EAGxD,OAAO,qBAAqB,KAAK,QAAQ,KAAK,KAAK,OAAO;CAC5D;;CAGA,MAAM,YAAY,MAAyB,UAAkB;EAC3D,MAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,UAAU;EAE7D,IAAI,KAAK,OAAO,CAAC,EAAE,QAAQ,KAAA,GAAW,KAAK,MAAM,SAAS,MAAM,QAAQ;CAC1E;;;CAIA,MAAM,aAAa,KAAa,UAAiC;EAC/D,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EACnC,IAAI,QAAQ,KAAA,GAAW;EAEvB,IAAI,KAAK,MAAM,WAAW,KAAK,IAAI,GAAG;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ;GAE1D,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;IAC9C,MAAM,QAAQ,KAAK,IAAI;IACvB,KAAK,MAAM,YAAY,IAAI;IAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,GAClF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,wBACnD;CAEN;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,MAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,UAAU,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;GAC3D,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK,MAAM,YAAY,IAAI;GAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,yCACnD;EACF,CAAC,CACH;CACF;CAEA,WAAmB,GAAY,QAAwB,KAAa,UAAkB;EACpF,MAAM,SAAS,IAAI,kBACjB,KAAK,QACL,KAAK,kBAAkB,GAAG,GAC1B,QACA,KACA,KAAK,QACL,KAAK,OACP;EACA,OAAO,OAAO,GAAG,QAAQ;EACzB,KAAK,UAAU,IAAI,KAAK,MAAM;EAE9B,OAAO;CACT;CAEA,WAAmB,MAAyB,QAAgB;EAC1D,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,YAAY,KAAK,IAAI,YAAY,QAAQ;EACjE,KAAK,UAAU,OAAO,KAAK,GAAG;CAChC;;;CAIA,kBAA0B,KAAqB;EAC7C,MAAM,SAAS,WAAW,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,KAAK;EAC5D,OAAOA,OAAK,KAAK,KAAK,SAAS,MAAM;CACvC;AACF"}