npm - @milaboratories/pl-drivers - Versions diffs - 1.12.8 → 1.12.9 - Mend

@milaboratories/pl-drivers 1.12.8 → 1.12.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (243) hide show

package/dist/drivers/download_blob/sparse_cache/ranges.js CHANGED Viewed

@@ -3,7 +3,6 @@ import * as fs from "node:fs/promises";
 import { createPathAtomically } from "@milaboratories/ts-helpers";
 import { RangeBytes } from "@milaboratories/pl-model-common";
 import { z } from "zod";
 //#region src/drivers/download_blob/sparse_cache/ranges.ts
 /** The content of the ranges file: ranges of bytes.
 * The ranges should be normalized: sorted and no overlaps.
@@ -63,7 +62,7 @@ function addRange(s, range) {
 	normalizeRanges(s);
 	return s;
 }
 //#endregion
 export { addRange, doesRangeExist, rangesFileName, rangesSize, readRangesFile, writeRangesFile };
 //# sourceMappingURL=ranges.js.map

package/dist/drivers/download_blob/sparse_cache/ranges.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ranges.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. /\nconst Ranges = z.object({\n ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n let ranges: Ranges = { ranges: [] };\n try {\n const file = await fs.readFile(path, \"utf8\");\n ranges = Ranges.parse(JSON.parse(file));\n } catch (e: unknown) {\n if (e instanceof SyntaxError \|\| e instanceof z.ZodError) {\n const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n logger.error(msg);\n throw new CorruptedRangesError(msg);\n }\n\n if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n throw e;\n }\n\n // If the file does not exist, assume the ranges are empty.\n }\n\n normalizeRanges(ranges);\n\n return ranges;\n}\n\n/* Writes to a temporal file and then renames it atomically. /\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n await createPathAtomically(logger, path, async (tempPath: string) => {\n await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n });\n}\n\n/* Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n s.ranges.sort((a, b) => a.from - b.from);\n\n for (let i = 0; i < s.ranges.length - 1; i++) {\n if (s.ranges[i].to >= s.ranges[i + 1].from) {\n mergeRanges(s, i);\n i--;\n }\n }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n for (const r of allRanges.ranges) {\n if (r.from <= range.from && range.to <= r.to) {\n return true;\n }\n }\n\n return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n s.ranges.push(range);\n normalizeRanges(s);\n\n return s;\n}\n"],"mappings":"~~;;;;;;;;;;~~AASA,MAAM,SAAS,EAAE,OAAO,EACtB,QAAQ,EAAE,MAAM,WAAW,EAC5B,CAAC;AAIF,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;AACpD,QAAO,QAAQ;;AAGjB,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,EAAE,EAAE;AACnC,KAAI;EACF,MAAM,OAAO,MAAM,GAAG,SAAS,MAAM,OAAO;AAC5C,WAAS,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC;UAChC,GAAY;AACnB,MAAI,aAAa,eAAe,aAAa,EAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;AAC/D,UAAO,MAAM,IAAI;AACjB,SAAM,IAAI,qBAAqB,IAAI;;AAGrC,MAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,UACpD,OAAM;;AAMV,iBAAgB,OAAO;AAEvB,QAAO;;;AAIT,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;AACpF,OAAM,qBAAqB,QAAQ,MAAM,OAAO,aAAqB;AACnE,QAAM,GAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;GAC7E;;;AAIJ,SAAgB,gBAAgB,GAAW;AACzC,GAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK;AAExC,MAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,IACvC,KAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,MAAM;AAC1C,cAAY,GAAG,EAAE;AACjB;;;AAKN,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,KAAK;CAC7D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,GAAG,GAAG;AAEvD,GAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;EAAI,CAAC;;AAGrC,SAAgB,WAAW,GAAW;AACpC,QAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;;AAGxE,SAAgB,eAAe,WAAmB,OAA4B;AAC5E,MAAK,MAAM,KAAK,UAAU,OACxB,KAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,GACxC,QAAO;AAIX,QAAO;;AAGT,SAAgB,SAAS,GAAW,OAAmB;AACrD,GAAE,OAAO,KAAK,MAAM;AACpB,iBAAgB,EAAE;AAElB,QAAO"}
1	+ {"version":3,"file":"ranges.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. /\nconst Ranges = z.object({\n ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n let ranges: Ranges = { ranges: [] };\n try {\n const file = await fs.readFile(path, \"utf8\");\n ranges = Ranges.parse(JSON.parse(file));\n } catch (e: unknown) {\n if (e instanceof SyntaxError \|\| e instanceof z.ZodError) {\n const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n logger.error(msg);\n throw new CorruptedRangesError(msg);\n }\n\n if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n throw e;\n }\n\n // If the file does not exist, assume the ranges are empty.\n }\n\n normalizeRanges(ranges);\n\n return ranges;\n}\n\n/* Writes to a temporal file and then renames it atomically. /\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n await createPathAtomically(logger, path, async (tempPath: string) => {\n await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n });\n}\n\n/* Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n s.ranges.sort((a, b) => a.from - b.from);\n\n for (let i = 0; i < s.ranges.length - 1; i++) {\n if (s.ranges[i].to >= s.ranges[i + 1].from) {\n mergeRanges(s, i);\n i--;\n }\n }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n for (const r of allRanges.ranges) {\n if (r.from <= range.from && range.to <= r.to) {\n return true;\n }\n }\n\n return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n s.ranges.push(range);\n normalizeRanges(s);\n\n return s;\n}\n"],"mappings":";;;;;;;;;AASA,MAAM,SAAS,EAAE,OAAO,EACtB,QAAQ,EAAE,MAAM,WAAW,EAC5B,CAAC;AAIF,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;AACpD,QAAO,QAAQ;;AAGjB,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,EAAE,EAAE;AACnC,KAAI;EACF,MAAM,OAAO,MAAM,GAAG,SAAS,MAAM,OAAO;AAC5C,WAAS,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC;UAChC,GAAY;AACnB,MAAI,aAAa,eAAe,aAAa,EAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;AAC/D,UAAO,MAAM,IAAI;AACjB,SAAM,IAAI,qBAAqB,IAAI;;AAGrC,MAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,UACpD,OAAM;;AAMV,iBAAgB,OAAO;AAEvB,QAAO;;;AAIT,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;AACpF,OAAM,qBAAqB,QAAQ,MAAM,OAAO,aAAqB;AACnE,QAAM,GAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;GAC7E;;;AAIJ,SAAgB,gBAAgB,GAAW;AACzC,GAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK;AAExC,MAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,IACvC,KAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,MAAM;AAC1C,cAAY,GAAG,EAAE;AACjB;;;AAKN,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,KAAK;CAC7D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,GAAG,GAAG;AAEvD,GAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;EAAI,CAAC;;AAGrC,SAAgB,WAAW,GAAW;AACpC,QAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;;AAGxE,SAAgB,eAAe,WAAmB,OAA4B;AAC5E,MAAK,MAAM,KAAK,UAAU,OACxB,KAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,GACxC,QAAO;AAIX,QAAO;;AAGT,SAAgB,SAAS,GAAW,OAAmB;AACrD,GAAE,OAAO,KAAK,MAAM;AACpB,iBAAgB,EAAE;AAElB,QAAO"}

package/dist/drivers/download_blob_url/driver.cjs CHANGED Viewed

@@ -1,10 +1,10 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_download_blob_task = require('../download_blob/download_blob_task.cjs');
-const require_files_cache = require('../helpers/files_cache.cjs');
-const require_snapshot = require('./snapshot.cjs');
-const require_url = require('../urls/url.cjs');
-const require_task = require('./task.cjs');
-const require_driver_id = require('./driver_id.cjs');
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_download_blob_task = require("../download_blob/download_blob_task.cjs");
+const require_files_cache = require("../helpers/files_cache.cjs");
+const require_snapshot = require("./snapshot.cjs");
+const require_url = require("../urls/url.cjs");
+const require_task = require("./task.cjs");
+const require_driver_id = require("./driver_id.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_path = require("node:path");
@@ -13,7 +13,6 @@ let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 let node_crypto = require("node:crypto");
 //#region src/drivers/download_blob_url/driver.ts
 /** Downloads .tar, .tar.gz or zip archives,
 * extracts them into saveDir and gets a url for it. */
@@ -123,7 +122,7 @@ var DownloadBlobToURLDriver = class {
 		return node_path.join(this.saveDir, `${String(BigInt(id))}_${format}`);
 	}
 };
 //#endregion
 exports.DownloadBlobToURLDriver = DownloadBlobToURLDriver;
 //# sourceMappingURL=driver.cjs.map

package/dist/drivers/download_blob_url/driver.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","getPathForFolderURL","Computable","makeDownloadableBlobSnapshot","newId","nonRecoverableError","rmRFDir","DownloadAndUnarchiveTask","path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { ResourceId } from \"@milaboratories/pl-client\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport {\n type ArchiveFormat,\n type BlobToURLDriver,\n type FolderURL,\n isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: number;\n nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. /\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientDownload: ClientDownload,\n private readonly saveDir: string,\n private readonly opts: DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: 50 1024 * 1024,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 10000,\n maxDelay: 30000,\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n }\n\n public info(): any {\n return {\n saveDir: this.saveDir,\n opts: this.opts,\n idToDownloadSize: this.idToDownload.size,\n idToDownloadKeys: this.idToDownload.keys(),\n idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n };\n }\n\n /*\n @returns full path to the referenced file\n /\n getPathForCustomProtocol(url: FolderURL): string {\n if (isFolderURL(url)) {\n return getPathForFolderURL(this.signer, url, this.saveDir);\n }\n\n throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n }\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx: ComputableCtx,\n ): FolderURL \| undefined;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ): Computable<FolderURL \| undefined>;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx?: ComputableCtx,\n ): Computable<FolderURL \| undefined> \| FolderURL \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined)\n return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n ? makeDownloadableBlobSnapshot(res, ctx)\n : res;\n\n const callerId = randomUUID();\n\n ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded archive might be undefined. The current result: ${result}`,\n );\n\n if (result?.error !== undefined) throw result?.error;\n\n return result?.url;\n }\n\n private extractArchiveAndGetURLNoCtx(\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n w: Watcher,\n callerId: string,\n ) {\n const task = this.idToDownload.get(newId(rInfo.id, format));\n\n if (task != undefined) {\n task.attach(w, callerId);\n return task.getURL();\n }\n\n const newTask = this.setNewTask(w, rInfo, format, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask.getURL();\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n await task.download();\n // Might be undefined if a error happened\n if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n const task = this.idToDownload.get(newId(id, format));\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(\n w: Watcher,\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n callerId: string,\n ) {\n const result = new DownloadAndUnarchiveTask(\n this.logger,\n this.signer,\n this.saveDir,\n this.getFilePath(rInfo.id, format),\n rInfo,\n format,\n this.clientDownload,\n );\n result.attach(w, callerId);\n this.idToDownload.set(newId(rInfo.id, format), result);\n\n return result;\n }\n\n private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n this.idToDownload.delete(newId(task.rInfo.id, task.format));\n }\n\n private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAiCA,IAAa,0BAAb,MAAgE;CAC9D,AAAQ,+BAAkD,IAAI,KAAK;CACnE,AAAQ;;;CAIR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAiB,gBACjB,AAAiB,SACjB,AAAiB,OAAmC;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;EARiB;EACA;EACA;EACA;EACA;AAKjB,OAAK,gBAAgB,IAAIA,yCAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAIC,+BAAW,KAAK,KAAK,mBAAmB;;CAG3D,AAAO,OAAY;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,uDAAgB,IAAI,CAClB,QAAOC,gCAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,OACV,QAAOC,sCAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,mDAAgD,IAAI,GACtDC,8CAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,wCAAuB;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,OAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,OAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,AAAQ,6BACN,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAIC,wBAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,QAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAACC,+CAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,OAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAgB,QAAuB,UAAiC;EACxF,MAAM,OAAO,KAAK,aAAa,IAAID,wBAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,OAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAME,qBAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,2FACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAMA,qBAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,AAAQ,WACN,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAIC,sCACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAIH,wBAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,AAAQ,WAAW,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,8DAA+B,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAOA,wBAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,AAAQ,YAAY,IAAgB,QAA+B;AACjE,SAAOI,UAAK,KAAK,KAAK,SAAS,GAAG,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG,SAAS"}
1	+ {"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","getPathForFolderURL","Computable","makeDownloadableBlobSnapshot","newId","nonRecoverableError","rmRFDir","DownloadAndUnarchiveTask","path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { ResourceId } from \"@milaboratories/pl-client\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport {\n type ArchiveFormat,\n type BlobToURLDriver,\n type FolderURL,\n isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: number;\n nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. /\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientDownload: ClientDownload,\n private readonly saveDir: string,\n private readonly opts: DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: 50 1024 * 1024,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 10000,\n maxDelay: 30000,\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n }\n\n public info(): any {\n return {\n saveDir: this.saveDir,\n opts: this.opts,\n idToDownloadSize: this.idToDownload.size,\n idToDownloadKeys: this.idToDownload.keys(),\n idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n };\n }\n\n /*\n @returns full path to the referenced file\n /\n getPathForCustomProtocol(url: FolderURL): string {\n if (isFolderURL(url)) {\n return getPathForFolderURL(this.signer, url, this.saveDir);\n }\n\n throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n }\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx: ComputableCtx,\n ): FolderURL \| undefined;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ): Computable<FolderURL \| undefined>;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx?: ComputableCtx,\n ): Computable<FolderURL \| undefined> \| FolderURL \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined)\n return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n ? makeDownloadableBlobSnapshot(res, ctx)\n : res;\n\n const callerId = randomUUID();\n\n ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded archive might be undefined. The current result: ${result}`,\n );\n\n if (result?.error !== undefined) throw result?.error;\n\n return result?.url;\n }\n\n private extractArchiveAndGetURLNoCtx(\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n w: Watcher,\n callerId: string,\n ) {\n const task = this.idToDownload.get(newId(rInfo.id, format));\n\n if (task != undefined) {\n task.attach(w, callerId);\n return task.getURL();\n }\n\n const newTask = this.setNewTask(w, rInfo, format, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask.getURL();\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n await task.download();\n // Might be undefined if a error happened\n if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n const task = this.idToDownload.get(newId(id, format));\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(\n w: Watcher,\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n callerId: string,\n ) {\n const result = new DownloadAndUnarchiveTask(\n this.logger,\n this.signer,\n this.saveDir,\n this.getFilePath(rInfo.id, format),\n rInfo,\n format,\n this.clientDownload,\n );\n result.attach(w, callerId);\n this.idToDownload.set(newId(rInfo.id, format), result);\n\n return result;\n }\n\n private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n this.idToDownload.delete(newId(task.rInfo.id, task.format));\n }\n\n private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAiCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,OAAA,GAAA,gCAAA,aAAgB,IAAI,CAClB,QAAOC,YAAAA,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,SAAA,GAAA,wBAAA,eAAgD,IAAI,GACtDC,iBAAAA,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAIC,kBAAAA,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAgB,QAAuB,UAAiC;EACxF,MAAM,OAAO,KAAK,aAAa,IAAID,kBAAAA,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAME,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAIC,aAAAA,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAIH,kBAAAA,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,aAAA,GAAA,0BAAA,oBAA+B,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAOA,kBAAAA,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAgB,QAA+B;AACjE,SAAOI,UAAK,KAAK,KAAK,SAAS,GAAG,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG,SAAS"}

package/dist/drivers/download_blob_url/driver.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"mappings":";;;;;;;;;;KA0BY,0BAAA;EACV,kBAAA;EACA,oBAAA;AAAA;;;cAKW,uBAAA,YAAmC,eAAA;EAAA,iBAS3B,MAAA;EAAA,iBACA,MAAA;EAAA,iBACA,cAAA;EAAA,iBACA,OAAA;EAAA,iBACA,IAAA;EAAA,QAZX,YAAA;EAAA,QACA,aAAA;EAgDD;;EAAA,QA5CC,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EACR,cAAA,EAAgB,cAAA,EAChB,OAAA,UACA,IAAA,GAAM,0BAAA;EAelB,IAAA,CAAA;EA4BA;;;EAfP,wBAAA,CAAyB,GAAA,EAAK,SAAA;EAQ9B,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,EACR,GAAA,EAAK,aAAA,GACJ,SAAA;EAEH,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,GACP,UAAA,CAAW,SAAA;EAAA,QA8BN,4BAAA;EA+Bc;EARhB,WAAA,CAAY,IAAA,EAAM,wBAAA,EAA0B,QAAA,WAAgB,OAAA;EAQU;;EAAtE,WAAA,CAAY,EAAA,EAAI,UAAA,EAAY,MAAA,EAAQ,aAAA,EAAe,QAAA,WAAmB,OAAA;EAvHf;EAsJvD,UAAA,CAAA,GAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAqBA,UAAA;EAAA,QAMA,WAAA;AAAA"}

package/dist/drivers/download_blob_url/driver.js CHANGED Viewed

@@ -11,7 +11,6 @@ import { Computable } from "@milaboratories/computable";
 import { isFolderURL } from "@milaboratories/pl-model-common";
 import { isPlTreeEntry } from "@milaboratories/pl-tree";
 import { randomUUID } from "node:crypto";
 //#region src/drivers/download_blob_url/driver.ts
 /** Downloads .tar, .tar.gz or zip archives,
 * extracts them into saveDir and gets a url for it. */
@@ -121,7 +120,7 @@ var DownloadBlobToURLDriver = class {
 		return path$1.join(this.saveDir, `${String(BigInt(id))}_${format}`);
 	}
 };
 //#endregion
 export { DownloadBlobToURLDriver };
 //# sourceMappingURL=driver.js.map

package/dist/drivers/download_blob_url/driver.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { ResourceId } from \"@milaboratories/pl-client\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport {\n type ArchiveFormat,\n type BlobToURLDriver,\n type FolderURL,\n isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: number;\n nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. /\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientDownload: ClientDownload,\n private readonly saveDir: string,\n private readonly opts: DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: 50 1024 * 1024,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 10000,\n maxDelay: 30000,\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n }\n\n public info(): any {\n return {\n saveDir: this.saveDir,\n opts: this.opts,\n idToDownloadSize: this.idToDownload.size,\n idToDownloadKeys: this.idToDownload.keys(),\n idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n };\n }\n\n /*\n @returns full path to the referenced file\n /\n getPathForCustomProtocol(url: FolderURL): string {\n if (isFolderURL(url)) {\n return getPathForFolderURL(this.signer, url, this.saveDir);\n }\n\n throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n }\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx: ComputableCtx,\n ): FolderURL \| undefined;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ): Computable<FolderURL \| undefined>;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx?: ComputableCtx,\n ): Computable<FolderURL \| undefined> \| FolderURL \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined)\n return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n ? makeDownloadableBlobSnapshot(res, ctx)\n : res;\n\n const callerId = randomUUID();\n\n ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded archive might be undefined. The current result: ${result}`,\n );\n\n if (result?.error !== undefined) throw result?.error;\n\n return result?.url;\n }\n\n private extractArchiveAndGetURLNoCtx(\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n w: Watcher,\n callerId: string,\n ) {\n const task = this.idToDownload.get(newId(rInfo.id, format));\n\n if (task != undefined) {\n task.attach(w, callerId);\n return task.getURL();\n }\n\n const newTask = this.setNewTask(w, rInfo, format, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask.getURL();\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n await task.download();\n // Might be undefined if a error happened\n if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n const task = this.idToDownload.get(newId(id, format));\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(\n w: Watcher,\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n callerId: string,\n ) {\n const result = new DownloadAndUnarchiveTask(\n this.logger,\n this.signer,\n this.saveDir,\n this.getFilePath(rInfo.id, format),\n rInfo,\n format,\n this.clientDownload,\n );\n result.attach(w, callerId);\n this.idToDownload.set(newId(rInfo.id, format), result);\n\n return result;\n }\n\n private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n this.idToDownload.delete(newId(task.rInfo.id, task.format));\n }\n\n private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAiCA,IAAa,0BAAb,MAAgE;CAC9D,AAAQ,+BAAkD,IAAI,KAAK;CACnE,AAAQ;;;CAIR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAiB,gBACjB,AAAiB,SACjB,AAAiB,OAAmC;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;EARiB;EACA;EACA;EACA;EACA;AAKjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,AAAO,OAAY;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,MAAI,YAAY,IAAI,CAClB,QAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,OACV,QAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,QAAkC,cAAc,IAAI,GACtD,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,WAAW,YAAY;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,OAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,OAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,AAAQ,6BACN,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,QAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,OAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAgB,QAAuB,UAAiC;EACxF,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,OAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,AAAQ,WACN,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,AAAQ,WAAW,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,AAAQ,YAAY,IAAgB,QAA+B;AACjE,SAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG,SAAS"}
1	+ {"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { ResourceId } from \"@milaboratories/pl-client\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport {\n type ArchiveFormat,\n type BlobToURLDriver,\n type FolderURL,\n isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: number;\n nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. /\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n private readonly clientDownload: ClientDownload,\n private readonly saveDir: string,\n private readonly opts: DownloadBlobToURLDriverOps = {\n cacheSoftSizeBytes: 50 1024 * 1024,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n type: \"exponentialWithMaxDelayBackoff\",\n initialDelay: 10000,\n maxDelay: 30000,\n backoffMultiplier: 1.5,\n jitter: 0.5,\n });\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n }\n\n public info(): any {\n return {\n saveDir: this.saveDir,\n opts: this.opts,\n idToDownloadSize: this.idToDownload.size,\n idToDownloadKeys: this.idToDownload.keys(),\n idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n };\n }\n\n /*\n @returns full path to the referenced file\n /\n getPathForCustomProtocol(url: FolderURL): string {\n if (isFolderURL(url)) {\n return getPathForFolderURL(this.signer, url, this.saveDir);\n }\n\n throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n }\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx: ComputableCtx,\n ): FolderURL \| undefined;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ): Computable<FolderURL \| undefined>;\n\n extractArchiveAndGetURL(\n res: DownloadableBlobSnapshot \| PlTreeEntry,\n format: ArchiveFormat,\n ctx?: ComputableCtx,\n ): Computable<FolderURL \| undefined> \| FolderURL \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined)\n return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n ? makeDownloadableBlobSnapshot(res, ctx)\n : res;\n\n const callerId = randomUUID();\n\n ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded archive might be undefined. The current result: ${result}`,\n );\n\n if (result?.error !== undefined) throw result?.error;\n\n return result?.url;\n }\n\n private extractArchiveAndGetURLNoCtx(\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n w: Watcher,\n callerId: string,\n ) {\n const task = this.idToDownload.get(newId(rInfo.id, format));\n\n if (task != undefined) {\n task.attach(w, callerId);\n return task.getURL();\n }\n\n const newTask = this.setNewTask(w, rInfo, format, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask.getURL();\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n await task.download();\n // Might be undefined if a error happened\n if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n const task = this.idToDownload.get(newId(id, format));\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(\n w: Watcher,\n rInfo: DownloadableBlobSnapshot,\n format: ArchiveFormat,\n callerId: string,\n ) {\n const result = new DownloadAndUnarchiveTask(\n this.logger,\n this.signer,\n this.saveDir,\n this.getFilePath(rInfo.id, format),\n rInfo,\n format,\n this.clientDownload,\n );\n result.attach(w, callerId);\n this.idToDownload.set(newId(rInfo.id, format), result);\n\n return result;\n }\n\n private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n this.idToDownload.delete(newId(task.rInfo.id, task.format));\n }\n\n private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,MAAI,YAAY,IAAI,CAClB,QAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,QAAkC,cAAc,IAAI,GACtD,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,WAAW,YAAY;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAgB,QAAuB,UAAiC;EACxF,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAgB,QAA+B;AACjE,SAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG,SAAS"}

package/dist/drivers/download_blob_url/driver_id.cjs CHANGED Viewed

@@ -1,9 +1,8 @@
 //#region src/drivers/download_blob_url/driver_id.ts
 function newId(id, format) {
 	return `id:${String(BigInt(id))}-${format}`;
 }
 //#endregion
 exports.newId = newId;
 //# sourceMappingURL=driver_id.cjs.map

package/dist/drivers/download_blob_url/driver_id.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { ResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: ResourceId, format: ArchiveFormat): Id {\n return `id:${String(BigInt(id))}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAMA,SAAgB,MAAM,IAAgB,QAA2B;AAC/D,QAAO,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG"}
1	+ {"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { ResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: ResourceId, format: ArchiveFormat): Id {\n return `id:${String(BigInt(id))}-${format}`;\n}\n\n// export function\n"],"mappings":";AAMA,SAAgB,MAAM,IAAgB,QAA2B;AAC/D,QAAO,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG"}

package/dist/drivers/download_blob_url/driver_id.js CHANGED Viewed

@@ -2,7 +2,7 @@
 function newId(id, format) {
 	return `id:${String(BigInt(id))}-${format}`;
 }
 //#endregion
 export { newId };
 //# sourceMappingURL=driver_id.js.map

package/dist/drivers/download_blob_url/snapshot.cjs CHANGED Viewed

@@ -1,14 +1,13 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
+require("../../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 //#region src/drivers/download_blob_url/snapshot.ts
 /** We need only resource type for this driver. */
 const DownloadableBlobSnapshot = (0, _milaboratories_pl_tree.rsSchema)({});
 function makeDownloadableBlobSnapshot(entryOrAccessor, ctx) {
 	return (0, _milaboratories_pl_tree.makeResourceSnapshot)((0, _milaboratories_pl_tree.isPlTreeEntry)(entryOrAccessor) ? ctx.accessor(entryOrAccessor).node() : (0, _milaboratories_pl_tree.isPlTreeEntryAccessor)(entryOrAccessor) ? entryOrAccessor.node() : entryOrAccessor, DownloadableBlobSnapshot);
 }
 //#endregion
 exports.DownloadableBlobSnapshot = DownloadableBlobSnapshot;
 exports.makeDownloadableBlobSnapshot = makeDownloadableBlobSnapshot;
 //# sourceMappingURL=snapshot.cjs.map

package/dist/drivers/download_blob_url/snapshot.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"snapshot.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n InferSnapshot,\n PlTreeEntry,\n PlTreeEntryAccessor,\n PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":"~~;;;;;~~AAeA,MAAa,~~iEAAoC~~,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,~~qGAN2B~~,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,~~sDACd~~,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}
1	+ {"version":3,"file":"snapshot.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n InferSnapshot,\n PlTreeEntry,\n PlTreeEntryAccessor,\n PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;;AAeA,MAAa,4BAAA,GAAA,wBAAA,UAAoC,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,SAAA,GAAA,wBAAA,uBAAA,GAAA,wBAAA,eAN2B,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,IAAA,GAAA,wBAAA,uBACd,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}

package/dist/drivers/download_blob_url/snapshot.d.ts CHANGED Viewed

@@ -1,10 +1,10 @@
 import { ComputableCtx } from "@milaboratories/computable";
-import * as _milaboratories_pl_tree0 from "@milaboratories/pl-tree";
+import * as _$_milaboratories_pl_tree0 from "@milaboratories/pl-tree";
 import { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 //#region src/drivers/download_blob_url/snapshot.d.ts
 /** We need only resource type for this driver. */
-declare const DownloadableBlobSnapshot: _milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, undefined, undefined>;
+declare const DownloadableBlobSnapshot: _$_milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, undefined, undefined>;
 type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;
 declare function makeDownloadableBlobSnapshot(entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor, ctx: ComputableCtx): DownloadableBlobSnapshot;
 //#endregion

package/dist/drivers/download_blob_url/snapshot.d.ts.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"snapshot.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"mappings":";;;;;;cAea,wBAAA,EAAuC,0BAAA,CAAf,sBAAA;AAAA,KACzB,wBAAA,GAA2B,aAAA,QAAqB,wBAAA;AAAA,iBAE5C,4BAAA,CACd,eAAA,EAAiB,WAAA,GAAc,kBAAA,GAAqB,mBAAA,EACpD,GAAA,EAAK,aAAA,GACJ,wBAAA"}

package/dist/drivers/download_blob_url/snapshot.js CHANGED Viewed

@@ -1,12 +1,11 @@
 import { isPlTreeEntry, isPlTreeEntryAccessor, makeResourceSnapshot, rsSchema } from "@milaboratories/pl-tree";
 //#region src/drivers/download_blob_url/snapshot.ts
 /** We need only resource type for this driver. */
 const DownloadableBlobSnapshot = rsSchema({});
 function makeDownloadableBlobSnapshot(entryOrAccessor, ctx) {
 	return makeResourceSnapshot(isPlTreeEntry(entryOrAccessor) ? ctx.accessor(entryOrAccessor).node() : isPlTreeEntryAccessor(entryOrAccessor) ? entryOrAccessor.node() : entryOrAccessor, DownloadableBlobSnapshot);
 }
 //#endregion
 export { DownloadableBlobSnapshot, makeDownloadableBlobSnapshot };
 //# sourceMappingURL=snapshot.js.map

package/dist/drivers/download_blob_url/snapshot.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"snapshot.js","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n InferSnapshot,\n PlTreeEntry,\n PlTreeEntryAccessor,\n PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":"~~;;;;~~AAeA,MAAa,2BAA2B,SAAS,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,QAAO,qBANM,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}
1	+ {"version":3,"file":"snapshot.js","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n InferSnapshot,\n PlTreeEntry,\n PlTreeEntryAccessor,\n PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n isPlTreeEntryAccessor,\n makeResourceSnapshot,\n rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n entryOrAccessor: PlTreeEntry \| PlTreeNodeAccessor \| PlTreeEntryAccessor,\n ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n const node = isPlTreeEntry(entryOrAccessor)\n ? ctx.accessor(entryOrAccessor).node()\n : isPlTreeEntryAccessor(entryOrAccessor)\n ? entryOrAccessor.node()\n : entryOrAccessor;\n\n return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;AAeA,MAAa,2BAA2B,SAAS,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,QAAO,qBANM,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}

package/dist/drivers/download_blob_url/task.cjs CHANGED Viewed

@@ -1,7 +1,7 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_download_errors = require('../../helpers/download_errors.cjs');
-const require_download = require('../../clients/download.cjs');
-const require_url = require('../urls/url.cjs');
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_download_errors = require("../../helpers/download_errors.cjs");
+const require_download = require("../../clients/download.cjs");
+const require_url = require("../urls/url.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _protobuf_ts_runtime = require("@protobuf-ts/runtime");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
@@ -19,7 +19,6 @@ let fs_promises = require("fs/promises");
 fs_promises = require_runtime.__toESM(fs_promises);
 let decompress = require("decompress");
 decompress = require_runtime.__toESM(decompress);
 //#region src/drivers/download_blob_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadAndUnarchiveTask = class {
@@ -153,8 +152,8 @@ var URLAborted = class extends Error {
 function nonRecoverableError(e) {
 	return e instanceof URLAborted || require_download_errors.isDownloadNetworkError400(e) || e instanceof require_download.UnknownStorageError || e instanceof require_download.WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED") || String(e).includes("incorrect header check");
 }
 //#endregion
 exports.DownloadAndUnarchiveTask = DownloadAndUnarchiveTask;
 exports.rmRFDir = rmRFDir;
 //# sourceMappingURL=task.cjs.map

package/dist/drivers/download_blob_url/task.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,AAAS,UAAU,IAAIA,2CAAgB;CACvC,AAAS,SAAS,IAAIC,yCAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP,AAAQ;CACR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAS,SACT,AAASC,QACT,AAAS,OACT,AAAS,QACT,AAAiB,gBACjB;EAPiB;EACA;EACR;EACA;EACA;EACA;EACQ;;;CAInB,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,iFAAkD,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,oFAAqD,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,gFAAiD,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,iFAAkD,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,aAAK,QAAQ,KAAK,KAAK;AAC9C,wDAAsB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,iDAAiB,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,8DAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAMC,YAAI,MAAM,MAAM;MACtB,MAAM,cAAcC,qBAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAMF,YAAI,MAAM,MAAM;MACtB,MAAM,SAASG,sBAAU,MAAMC,UAAK,cAAc,CAAC;MACnD,MAAM,QAAQH,qBAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,qBAAS,MAAM,WAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,oCAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,WAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,uCAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,8CAAc,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAMI,yBAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,aAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMA,YAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,QAAc;AAC1C,OAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACbO,kDAA0B,EAAE,IAC5B,aAAaC,wCACb,aAAaC,sCAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
1	+ {"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","path","fsp","Writable","tar","Transform","zlib","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,UAAmB,IAAIA,2BAAAA,gBAAgB;CACvC,SAAkB,IAAIC,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,QACA,OACA,QACA,gBACA;AAPiB,OAAA,SAAA;AACA,OAAA,SAAA;AACR,OAAA,UAAA;AACA,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,SAAA;AACQ,OAAA,iBAAA;;;CAInB,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,oBAAkD,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,8BAAA,GAAA,0BAAA,yBAAqD,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,0BAAA,GAAA,0BAAA,yBAAiD,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,gCAAA,GAAA,0BAAA,oBAAkD,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAA,QAAK,QAAQ,KAAK,KAAK;AAC9C,SAAA,GAAA,2BAAA,iBAAsB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAME,YAAI,MAAM,MAAM;MACtB,MAAM,cAAcC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAMF,YAAI,MAAM,MAAM;MACtB,MAAM,SAASG,YAAAA,UAAU,MAAMC,UAAK,cAAc,CAAC;MACnD,MAAM,QAAQH,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAID,YAAAA,SAAS,MAAM,GAAA,QAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,aAAA,GAAA,WAAA,SAAiB,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAA,QAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,EAAA,GAAA,qBAAA,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,MAAA,GAAA,2BAAA,UAAc,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAMI,YAAAA,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAML,YAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAA,QAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMA,YAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,QAAc;AAC1C,OAAMA,YAAI,GAAGD,QAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACbO,wBAAAA,0BAA0B,EAAE,IAC5B,aAAaC,iBAAAA,uBACb,aAAaC,iBAAAA,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}

package/dist/drivers/download_blob_url/task.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"mappings":";;;;;;;KA6BY,SAAA;EACV,GAAA,GAAM,SAAA;EACN,KAAA;AAAA;;cAIW,wBAAA;EAAA,iBAWQ,MAAA;EAAA,iBACA,MAAA;EAAA,SACR,OAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA,EAAO,wBAAA;EAAA,SACP,MAAA,EAAQ,aAAA;EAAA,iBACA,cAAA;EAAA,SAhBV,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;EAAA,QACQ,GAAA;EAAA,QACA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EAChB,OAAA,UACA,IAAA,UACA,KAAA,EAAO,wBAAA,EACP,MAAA,EAAQ,aAAA,EACA,cAAA,EAAgB,cAAA;;EAI5B,IAAA,CAAA;;;;;;;;;EAYP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAA,GAAQ,OAAA;EAvBI;EAAA,QAoDJ,qBAAA;EAuEd,MAAA,CAAA,GAAU,SAAA;EAAA,QAQF,OAAA;EAAA,QAMA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA;;KA4BH,WAAA;EACH,SAAA;EACA,WAAA;EACA,UAAA;EACA,aAAA,GAAgB,aAAA;EAChB,QAAA;EACA,OAAA;EACA,cAAA;EACA,eAAA;EACA,cAAA;EACA,WAAA;AAAA"}

package/dist/drivers/download_blob_url/task.js CHANGED Viewed

@@ -12,7 +12,6 @@ import * as tar from "tar-fs";
 import path from "path";
 import * as fsp from "fs/promises";
 import decompress from "decompress";
 //#region src/drivers/download_blob_url/task.ts
 /** Downloads and extracts an archive to a directory. */
 var DownloadAndUnarchiveTask = class {
@@ -146,7 +145,7 @@ var URLAborted = class extends Error {
 function nonRecoverableError(e) {
 	return e instanceof URLAborted || isDownloadNetworkError400(e) || e instanceof UnknownStorageError || e instanceof WrongLocalFileUrl || e?.code == "ENOENT" || e.name == "RpcError" && (e.code == "NOT_FOUND" || e.code == "ABORTED" || e.code == "UNIMPLEMENTED") || String(e).includes("incorrect header check");
 }
 //#endregion
 export { DownloadAndUnarchiveTask, rmRFDir };
 //# sourceMappingURL=task.js.map

package/dist/drivers/download_blob_url/task.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,AAAS,UAAU,IAAI,gBAAgB;CACvC,AAAS,SAAS,IAAI,cAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP,AAAQ;CACR,AAAQ;CAER,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAS,SACT,AAAS,MACT,AAAS,OACT,AAAS,QACT,AAAiB,gBACjB;EAPiB;EACA;EACR;EACA;EACA;EACA;EACQ;;;CAInB,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,AAAQ,QAAQ,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,AAAQ,SAAS,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}
1	+ {"version":3,"file":"task.js","names":[],"sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport path from \"path\";\nimport fs from \"fs\";\nimport * as fsp from \"fs/promises\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport {\n UnknownStorageError,\n WrongLocalFileUrl,\n type ClientDownload,\n} from \"../../clients/download\";\nimport type { ArchiveFormat, FolderURL } from \"@milaboratories/pl-model-common\";\nimport { newFolderURL } from \"../urls/url\";\nimport decompress from \"decompress\";\nimport { assertNever } from \"@protobuf-ts/runtime\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} finished`,\n );\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(\n `a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`,\n );\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(\n `download and decompress for ${resourceIdToString(this.rInfo.id)} failed`,\n );\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo,\n {},\n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case \"tar\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case \"tgz\":\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content.pipeThrough(gunzip, { signal }).pipeTo(untar, { signal });\n return;\n\n case \"zip\":\n this.state!.zipPath = this.path + \".zip\";\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: (file) => !file.path.endsWith(\"/\"),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n },\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = \"URLAborted\";\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted \|\|\n isDownloadNetworkError400(e) \|\|\n e instanceof UnknownStorageError \|\|\n e instanceof WrongLocalFileUrl \|\|\n // file that we downloads from was moved or deleted.\n e?.code == \"ENOENT\" \|\|\n // A resource was deleted.\n (e.name == \"RpcError\" &&\n (e.code == \"NOT_FOUND\" \|\| e.code == \"ABORTED\" \|\| e.code == \"UNIMPLEMENTED\")) \|\|\n // wrong archive format\n String(e).includes(\"incorrect header check\")\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAmCA,IAAa,2BAAb,MAAsC;CACpC,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CACP;CACA;CAEA,YACE,QACA,QACA,SACA,MACA,OACA,QACA,gBACA;AAPiB,OAAA,SAAA;AACA,OAAA,SAAA;AACR,OAAA,UAAA;AACA,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,SAAA;AACQ,OAAA,iBAAA;;;CAInB,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,QAAQ,KAAK;GACb,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,aAAa,KAAK;GACnB;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,WAAW;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,sBAAsB,KAAK,UAAU,OAAO;AACpE,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAClE;AAED,QAAK,OAAO,KAAK,6BAA6B,wBAAwB,KAAK,MAAM,CAAC,GAAG;WAC9E,GAAQ;AACf,QAAK,OAAO,KACV,yBAAyB,EAAE,yBAAyB,wBAAwB,KAAK,MAAM,CAAC,GACzF;AAED,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YACV,+BAA+B,mBAAmB,KAAK,MAAM,GAAG,CAAC,SAClE;AAED,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;;CAKV,MAAc,sBAAsB,QAAsC;AACxE,OAAK,QAAQ,EAAE;AAEf,OAAK,MAAM,YAAY,KAAK,QAAQ,KAAK,KAAK;AAC9C,QAAM,gBAAgB,KAAK,MAAM,UAAU;AAE3C,OAAK,MAAM,cAAc,MAAM,WAAW,KAAK,KAAK;AACpD,MAAI,KAAK,MAAM,YACb,QAAO,MAAM,QAAQ,KAAK,KAAK;AA4DjC,SAzDa,MAAM,KAAK,eAAe,gBACrC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;AACvB,QAAK,MAAO,aAAa;AAEzB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,SAAK,MAAO,WAAW;AACvB,SAAK,MAAO,gBAAgB,KAAK;AAEjC,YAAQ,KAAK,QAAb;KACE,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,cAAc,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AACtD,YAAM,QAAQ,OAAO,aAAa,EAAE,QAAQ,CAAC;AAC7C;KAEF,KAAK;AACH,YAAM,IAAI,MAAM,MAAM;MACtB,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;MACnD,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAEhD,YAAM,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,OAAO,EAAE,QAAQ,CAAC;AACvE;KAEF,KAAK;AACH,WAAK,MAAO,UAAU,KAAK,OAAO;MAElC,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,KAAK,MAAO,QAAQ,CAAC;AACnE,YAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,CAAC;AACnC,WAAK,MAAO,iBAAiB;AAM7B,YAAM,WAAW,KAAK,MAAO,SAAS,OAAO,EAC3C,SAAS,SAAS,CAAC,KAAK,KAAK,SAAS,IAAI,EAC3C,CAAC;AACF,WAAK,MAAO,kBAAkB;AAE9B,YAAM,GAAG,SAAS,GAAG,KAAK,MAAO,QAAQ;AACzC,WAAK,MAAO,iBAAiB;AAE7B;KAEF,QACE,aAAY,KAAK,OAAO;;KAE5B;AAEF,QAAK,MAAO,cAAc;AAC1B,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KAAM,QAAO,EAAE,KAAK,SAAS,KAAK,IAAI,EAAE;AAEjD,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;AACZ,OAAK,MAAM,aAAa,KAAK,QAAQ,KAAK,SAAS,KAAK,KAAK;;CAG/D,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQ,KAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC;;;AAkBtD,IAAM,aAAN,cAAyB,MAAM;CAC7B,OAAO;;AAGT,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eACR,EAAE,QAAQ,eAAe,EAAE,QAAQ,aAAa,EAAE,QAAQ,oBAE7D,OAAO,EAAE,CAAC,SAAS,yBAAyB"}

package/dist/drivers/download_url/driver.cjs CHANGED Viewed

@@ -1,9 +1,9 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_download_errors = require('../../helpers/download_errors.cjs');
-const require_download = require('../../helpers/download.cjs');
-const require_files_cache = require('../helpers/files_cache.cjs');
-const require_url = require('../urls/url.cjs');
-const require_task = require('./task.cjs');
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_download_errors = require("../../helpers/download_errors.cjs");
+const require_download = require("../../helpers/download.cjs");
+const require_files_cache = require("../helpers/files_cache.cjs");
+const require_url = require("../urls/url.cjs");
+const require_task = require("./task.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_path = require("node:path");
@@ -11,7 +11,6 @@ node_path = require_runtime.__toESM(node_path);
 let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let node_crypto = require("node:crypto");
 //#region src/drivers/download_url/driver.ts
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
@@ -107,7 +106,7 @@ var DownloadUrlDriver = class {
 		return node_path.join(this.saveDir, sha256);
 	}
 };
 //#endregion
 exports.DownloadUrlDriver = DownloadUrlDriver;
 //# sourceMappingURL=driver.cjs.map

package/dist/drivers/download_url/driver.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n /** Returns a Computable that (when the time will come)\n * downloads an archive from an URL,\n * extracts it to the local dir and returns a path to that dir. /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n}\n\nexport interface UrlResult {\n /* Path to the downloadable blob along with the signature and a custom protocol,\n * might be undefined when the error happened. /\n url?: BlockUIURL;\n /* Error that happened when the archive were downloaded. /\n error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n /* A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /* Whether to gunzip the downloaded archive (it will be untared automatically). /\n withGunzip: boolean;\n\n /* Max number of concurrent downloads while calculating computable states\n * derived from this driver.\n * /\n nConcurrentDownloads: number;\n};\n\n/* Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. /\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n private readonly downloadHelper: RemoteFileDownloader;\n\n private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadByUrlTask>;\n\n constructor(\n private readonly logger: MiLogger,\n httpClient: Dispatcher,\n private readonly saveDir: string,\n private readonly signer: Signer,\n private readonly opts: DownloadUrlDriverOps = {\n cacheSoftSizeBytes: 1 1024 * 1024 * 1024, // 1 GB\n withGunzip: true,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n this.downloadHelper = new RemoteFileDownloader(httpClient);\n }\n\n /** Use to get a path result inside a computable context /\n getUrl(url: URL, ctx: ComputableCtx): UrlResult \| undefined;\n\n /* Returns a Computable that do the work /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n\n /* Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. /\n getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n const callerId = randomUUID();\n\n // read as ~ golang's defer\n ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n );\n\n return result;\n }\n\n getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n\n if (task !== undefined) {\n task.attach(w, callerId);\n return task.getUrl();\n }\n\n const newTask = this.setNewTask(w, url, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n });\n\n return newTask.getUrl();\n }\n\n getPathForBlockUI(url: string): string {\n if (!isBlockUIURL(url)) {\n throw new Error(`getPathForBlockUI: ${url} is invalid`);\n }\n\n return getPathForBlockUIURL(this.signer, url, this.saveDir);\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n await task.download(this.downloadHelper, this.opts.withGunzip);\n // Might be undefined if a error happened\n if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(url: URL, callerId: string): Promise<void> {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadByUrlTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(w: Watcher, url: URL, callerId: string) {\n const result = new DownloadByUrlTask(\n this.logger,\n this.getFilePath(url),\n url,\n this.signer,\n this.saveDir,\n );\n result.attach(w, callerId);\n this.urlToDownload.set(url.toString(), result);\n\n return result;\n }\n\n private removeTask(task: DownloadByUrlTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n this.urlToDownload.delete(task.url.toString());\n }\n\n private getFilePath(url: URL): string {\n const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n return path.join(this.saveDir, sha256);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E,AAAiB;CAEjB,AAAQ,gCAAgD,IAAI,KAAK;CACjE,AAAQ;;;CAIR,AAAQ;CAER,YACE,AAAiB,QACjB,YACA,AAAiB,SACjB,AAAiB,QACjB,AAAiB,OAA6B;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;EATiB;EAEA;EACA;EACA;AAMjB,OAAK,gBAAgB,IAAIA,yCAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAIC,+BAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAIC,sCAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,OAAW,QAAOC,sCAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,wCAAuB;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,OAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,QAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAaC,4BAAe,CAACC,kDAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,mDAAc,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAOC,iCAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,OAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,OAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAMC,qBAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,2FACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAMA,qBAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,mEAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,AAAQ,WAAW,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAIC,+BACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,AAAQ,WAAW,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,AAAQ,YAAY,KAAkB;EACpC,MAAM,qCAAoB,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOC,UAAK,KAAK,KAAK,SAAS,OAAO"}
1	+ {"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","RemoteFileDownloader","Computable","URLAborted","isDownloadNetworkError400","getPathForBlockUIURL","rmRFDir","DownloadByUrlTask","path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n /** Returns a Computable that (when the time will come)\n * downloads an archive from an URL,\n * extracts it to the local dir and returns a path to that dir. /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n}\n\nexport interface UrlResult {\n /* Path to the downloadable blob along with the signature and a custom protocol,\n * might be undefined when the error happened. /\n url?: BlockUIURL;\n /* Error that happened when the archive were downloaded. /\n error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n /* A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /* Whether to gunzip the downloaded archive (it will be untared automatically). /\n withGunzip: boolean;\n\n /* Max number of concurrent downloads while calculating computable states\n * derived from this driver.\n * /\n nConcurrentDownloads: number;\n};\n\n/* Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. /\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n private readonly downloadHelper: RemoteFileDownloader;\n\n private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n private downloadQueue: TaskProcessor;\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadByUrlTask>;\n\n constructor(\n private readonly logger: MiLogger,\n httpClient: Dispatcher,\n private readonly saveDir: string,\n private readonly signer: Signer,\n private readonly opts: DownloadUrlDriverOps = {\n cacheSoftSizeBytes: 1 1024 * 1024 * 1024, // 1 GB\n withGunzip: true,\n nConcurrentDownloads: 50,\n },\n ) {\n this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n this.downloadHelper = new RemoteFileDownloader(httpClient);\n }\n\n /** Use to get a path result inside a computable context /\n getUrl(url: URL, ctx: ComputableCtx): UrlResult \| undefined;\n\n /* Returns a Computable that do the work /\n getUrl(url: URL): Computable<UrlResult \| undefined>;\n\n /* Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. /\n getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult \| undefined> \| UrlResult \| undefined {\n // wrap result as computable, if we were not given an existing computable context\n if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n const callerId = randomUUID();\n\n // read as ~ golang's defer\n ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n if (result?.url === undefined)\n ctx.markUnstable(\n `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n );\n\n return result;\n }\n\n getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n\n if (task !== undefined) {\n task.attach(w, callerId);\n return task.getUrl();\n }\n\n const newTask = this.setNewTask(w, url, callerId);\n this.downloadQueue.push({\n fn: async () => this.downloadUrl(newTask, callerId),\n recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n });\n\n return newTask.getUrl();\n }\n\n getPathForBlockUI(url: string): string {\n if (!isBlockUIURL(url)) {\n throw new Error(`getPathForBlockUI: ${url} is invalid`);\n }\n\n return getPathForBlockUIURL(this.signer, url, this.saveDir);\n }\n\n /* Downloads and extracts a tar archive if it wasn't downloaded yet. /\n async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n await task.download(this.downloadHelper, this.opts.withGunzip);\n // Might be undefined if a error happened\n if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n }\n\n /* Removes a directory and aborts a downloading task when all callers\n * are not interested in it. /\n async releasePath(url: URL, callerId: string): Promise<void> {\n const key = url.toString();\n const task = this.urlToDownload.get(key);\n if (task == undefined) return;\n\n if (this.cache.existsFile(task.path)) {\n const toDelete = this.cache.removeFile(task.path, callerId);\n\n await Promise.all(\n toDelete.map(async (task: DownloadByUrlTask) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted)\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n\n /* Removes all files from a hard drive. */\n async releaseAll() {\n this.downloadQueue.stop();\n\n await Promise.all(\n Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n await rmRFDir(task.path);\n this.cache.removeCache(task);\n\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n );\n }),\n );\n }\n\n private setNewTask(w: Watcher, url: URL, callerId: string) {\n const result = new DownloadByUrlTask(\n this.logger,\n this.getFilePath(url),\n url,\n this.signer,\n this.saveDir,\n );\n result.attach(w, callerId);\n this.urlToDownload.set(url.toString(), result);\n\n return result;\n }\n\n private removeTask(task: DownloadByUrlTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n this.urlToDownload.delete(task.url.toString());\n }\n\n private getFilePath(url: URL): string {\n const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n return path.join(this.saveDir, sha256);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E;CAEA,gCAAwD,IAAI,KAAK;CACjE;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;AATiB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,OAAA;AAMjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAIC,iBAAAA,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,KAAA,EAAW,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,YAAA,GAAA,YAAA,aAAuB;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,KAAA,GAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAaC,aAAAA,eAAe,CAACC,wBAAAA,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,EAAA,GAAA,gCAAA,cAAc,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAOC,YAAAA,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAMC,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WAAmB,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAIC,aAAAA,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,WAAmB,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,YAAoB,KAAkB;EACpC,MAAM,UAAA,GAAA,YAAA,YAAoB,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOC,UAAK,KAAK,KAAK,SAAS,OAAO"}

package/dist/drivers/download_url/driver.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_url/driver.ts"],"mappings":";;;;;;;UAgBiB,qBAAA;EAAA;;;EAIf,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;AAAA;AAAA,UAGd,SAAA;EAHa;;EAM5B,GAAA,GAAM,UAAA;EANM;EAQZ,KAAA;AAAA;AAAA,KAGU,oBAAA;EAX4B;;AAGxC;;EAaE,kBAAA,UAVgB;EAahB,UAAA;EAbM;;;EAkBN,oBAAA;AAAA;;;cAKW,iBAAA,YAA6B,qBAAA,EAAuB,cAAA;EAAA,iBAW5C,MAAA;EAAA,iBAEA,OAAA;EAAA,iBACA,MAAA;EAAA,iBACA,IAAA;EAAA,iBAdF,cAAA;EAAA,QAET,aAAA;EAAA,QACA,aAAA;;;UAIA,KAAA;cAGW,MAAA,EAAQ,QAAA,EACzB,UAAA,EAAY,UAAA,EACK,OAAA,UACA,MAAA,EAAQ,MAAA,EACR,IAAA,GAAM,oBAAA;EAAA;EAYzB,MAAA,CAAO,GAAA,EAAK,GAAA,EAAK,GAAA,EAAK,aAAA,GAAgB,SAAA;EAAhB;EAGtB,MAAA,CAAO,GAAA,EAAK,GAAA,GAAM,UAAA,CAAW,SAAA;EAqB7B,WAAA,CAAY,GAAA,EAAK,GAAA,EAAK,CAAA,EAAG,OAAA,EAAS,QAAA,WAAgB,SAAA;EAkBlD,iBAAA,CAAkB,GAAA;EAvCA;EAgDZ,WAAA,CAAY,IAAA,EAAM,iBAAA,EAAmB,QAAA,WAAgB,OAAA;EA3BlC;;EAmCnB,WAAA,CAAY,GAAA,EAAK,GAAA,EAAK,QAAA,WAAmB,OAAA;EARY;EAwCrD,UAAA,CAAA,GAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAcA,UAAA;EAAA,QAMA,WAAA;AAAA"}

package/dist/drivers/download_url/driver.js CHANGED Viewed

@@ -9,7 +9,6 @@ import * as path$1 from "node:path";
 import { Computable } from "@milaboratories/computable";
 import { isBlockUIURL } from "@milaboratories/pl-model-common";
 import { createHash, randomUUID } from "node:crypto";
 //#region src/drivers/download_url/driver.ts
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
@@ -105,7 +104,7 @@ var DownloadUrlDriver = class {
 		return path$1.join(this.saveDir, sha256);
 	}
 };
 //#endregion
 export { DownloadUrlDriver };
 //# sourceMappingURL=driver.js.map