@milaboratories/pl-drivers 1.16.0 → 1.16.2

package/dist/drivers/download_url/driver.js

@@ -13,8 +13,12 @@ import { createHash, randomUUID } from "node:crypto";
 /** Downloads .tar or .tar.gz archives by given URLs
 * and extracts them into saveDir. */
 var DownloadUrlDriver = class {
+	logger;
+	saveDir;
+	signer;
+	opts;
 	downloadHelper;
-	urlToDownload = /* @__PURE__ */ new Map();
+	taskByKey = /* @__PURE__ */ new Map();
 	downloadQueue;
 	/** Writes and removes files to a hard drive and holds a counter for every
 	* file that should be kept. */
@@ -35,20 +39,37 @@ var DownloadUrlDriver = class {
 	/** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */
 	getUrl(url, ctx) {
 		if (ctx === void 0) return Computable.make((c) => this.getUrl(url, c));
+		return this.getResult({
+			type: "url",
+			url
+		}, url.toString(), ctx);
+	}
+	/** Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it
+	* into the recycled cache (so a live UI keeps the folder pinned) and returns
+	* a custom-protocol URL. Keyed on `path + ":" + mtime`, so the unpacked UI is
+	* re-derived when the source block changes. */
+	getLocalTgz(path, mtime, ctx) {
+		if (ctx === void 0) return Computable.make((c) => this.getLocalTgz(path, mtime, c));
+		return this.getResult({
+			type: "local-tgz",
+			path,
+			mtime
+		}, `${path}:${mtime}`, ctx);
+	}
+	getResult(source, key, ctx) {
 		const callerId = randomUUID();
-		ctx.addOnDestroy(() => this.releasePath(url, callerId));
-		const result = this.getUrlNoCtx(url, ctx.watcher, callerId);
+		ctx.addOnDestroy(() => this.releaseByKey(key, callerId));
+		const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);
 		if (result?.url === void 0) ctx.markUnstable(`a path to the downloaded and untared archive might be undefined. The current result: ${result}`);
 		return result;
 	}
-	getUrlNoCtx(url, w, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	getResultNoCtx(source, key, w, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task !== void 0) {
 			task.attach(w, callerId);
 			return task.getUrl();
 		}
-		const newTask = this.setNewTask(w, url, callerId);
+		const newTask = this.setNewTask(w, source, key, callerId);
 		this.downloadQueue.push({
 			fn: async () => this.downloadUrl(newTask, callerId),
 			recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e)
@@ -66,9 +87,8 @@ var DownloadUrlDriver = class {
 	}
 	/** Removes a directory and aborts a downloading task when all callers
 	* are not interested in it. */
-	async releasePath(url, callerId) {
-		const key = url.toString();
-		const task = this.urlToDownload.get(key);
+	async releaseByKey(key, callerId) {
+		const task = this.taskByKey.get(key);
 		if (task == void 0) return;
 		if (this.cache.existsFile(task.path)) {
 			const toDelete = this.cache.removeFile(task.path, callerId);
@@ -82,25 +102,27 @@ var DownloadUrlDriver = class {
 	/** Removes all files from a hard drive. */
 	async releaseAll() {
 		this.downloadQueue.stop();
-		await Promise.all(Array.from(this.urlToDownload.entries()).map(async ([, task]) => {
+		await Promise.all(Array.from(this.taskByKey.entries()).map(async ([, task]) => {
 			await rmRFDir(task.path);
 			this.cache.removeCache(task);
 			this.removeTask(task, `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`);
 		}));
 	}
-	setNewTask(w, url, callerId) {
-		const result = new DownloadByUrlTask(this.logger, this.getFilePath(url), url, this.signer, this.saveDir);
+	setNewTask(w, source, key, callerId) {
+		const result = new DownloadByUrlTask(this.logger, this.getExtractionPath(key), source, key, this.signer, this.saveDir);
 		result.attach(w, callerId);
-		this.urlToDownload.set(url.toString(), result);
+		this.taskByKey.set(key, result);
 		return result;
 	}
 	removeTask(task, reason) {
 		task.abort(reason);
-		task.change.markChanged(`task for url ${task.url} removed: ${reason}`);
-		this.urlToDownload.delete(task.url.toString());
+		task.change.markChanged(`task for ${task.key} removed: ${reason}`);
+		this.taskByKey.delete(task.key);
 	}
-	getFilePath(url) {
-		const sha256 = createHash("sha256").update(url.toString()).digest("hex");
+	/** Directory an archive (url or local file) is extracted into; keyed by a
+	* hash of the task key so the same source maps to a stable on-disk path. */
+	getExtractionPath(key) {
+		const sha256 = createHash("sha256").update(key).digest("hex");
 		return path$1.join(this.saveDir, sha256);
 	}
 };

package/dist/drivers/download_url/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releasePath(url, callerId));\n\n    const result = this.getUrlNoCtx(url, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  getUrlNoCtx(url: URL, w: Watcher, callerId: string) {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, url, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(url: URL, callerId: string): Promise<void> {\n    const key = url.toString();\n    const task = this.urlToDownload.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.urlToDownload.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, url: URL, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getFilePath(url),\n      url,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.urlToDownload.set(url.toString(), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for url ${task.url} removed: ${reason}`);\n    this.urlToDownload.delete(task.url.toString());\n  }\n\n  private getFilePath(url: URL): string {\n    const sha256 = createHash(\"sha256\").update(url.toString()).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiDA,IAAa,oBAAb,MAAgF;CAC9E;CAEA,gCAAwD,IAAI,KAAK;CACjE;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;EACvB,EACD;AATiB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,OAAA;AAMjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,qBAAqB;AACnF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;AACzD,OAAK,iBAAiB,IAAI,qBAAqB,WAAW;;;CAU5D,OAAO,KAAU,KAAgF;AAE/F,MAAI,QAAQ,KAAA,EAAW,QAAO,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,EAAE,CAAC;EAEzE,MAAM,WAAW,YAAY;AAG7B,MAAI,mBAAmB,KAAK,YAAY,KAAK,SAAS,CAAC;EAEvD,MAAM,SAAS,KAAK,YAAY,KAAK,IAAI,SAAS,SAAS;AAC3D,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,wFAAwF,SACzF;AAEH,SAAO;;CAGT,YAAY,KAAU,GAAY,UAAkB;EAClD,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AAExC,MAAI,SAAS,KAAA,GAAW;AACtB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,KAAK,SAAS;AACjD,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,EAAE,aAAa,eAAe,CAAC,0BAA0B,EAAE;GAC9F,CAAC;AAEF,SAAO,QAAQ,QAAQ;;CAGzB,kBAAkB,KAAqB;AACrC,MAAI,CAAC,aAAa,IAAI,CACpB,OAAM,IAAI,MAAM,sBAAsB,IAAI,aAAa;AAGzD,SAAO,qBAAqB,KAAK,QAAQ,KAAK,KAAK,QAAQ;;;CAI7D,MAAM,YAAY,MAAyB,UAAkB;AAC3D,QAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,WAAW;AAE9D,MAAI,KAAK,QAAQ,EAAE,QAAQ,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK3E,MAAM,YAAY,KAAU,UAAiC;EAC3D,MAAM,MAAM,IAAI,UAAU;EAC1B,MAAM,OAAO,KAAK,cAAc,IAAI,IAAI;AACxC,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;AAC9C,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,cAAc,SAAS,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WAAmB,GAAY,KAAU,UAAkB;EACzD,MAAM,SAAS,IAAI,kBACjB,KAAK,QACL,KAAK,YAAY,IAAI,EACrB,KACA,KAAK,QACL,KAAK,QACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,cAAc,IAAI,IAAI,UAAU,EAAE,OAAO;AAE9C,SAAO;;CAGT,WAAmB,MAAyB,QAAgB;AAC1D,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,gBAAgB,KAAK,IAAI,YAAY,SAAS;AACtE,OAAK,cAAc,OAAO,KAAK,IAAI,UAAU,CAAC;;CAGhD,YAAoB,KAAkB;EACpC,MAAM,SAAS,WAAW,SAAS,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,OAAO,MAAM;AACxE,SAAOA,OAAK,KAAK,KAAK,SAAS,OAAO"}
+{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { createHash, randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport type { BlockUIURL, FrontendDriver } from \"@milaboratories/pl-model-common\";\nimport { isBlockUIURL } from \"@milaboratories/pl-model-common\";\nimport { getPathForBlockUIURL } from \"../urls/url\";\nimport type { DownloadSource } from \"./task\";\nimport { DownloadByUrlTask, rmRFDir, URLAborted } from \"./task\";\n\nexport interface DownloadUrlSyncReader {\n  /** Returns a Computable that (when the time will come)\n   * downloads an archive from an URL,\n   * extracts it to the local dir and returns a path to that dir. */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n}\n\nexport interface UrlResult {\n  /** Path to the downloadable blob along with the signature and a custom protocol,\n   * might be undefined when the error happened. */\n  url?: BlockUIURL;\n  /** Error that happened when the archive were downloaded. */\n  error?: string;\n}\n\nexport type DownloadUrlDriverOps = {\n  /** A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /** Whether to gunzip the downloaded archive (it will be untared automatically). */\n  withGunzip: boolean;\n\n  /** Max number of concurrent downloads while calculating computable states\n   * derived from this driver.\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar or .tar.gz archives by given URLs\n * and extracts them into saveDir. */\nexport class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {\n  private readonly downloadHelper: RemoteFileDownloader;\n\n  private taskByKey: Map<string, DownloadByUrlTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadByUrlTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    httpClient: Dispatcher,\n    private readonly saveDir: string,\n    private readonly signer: Signer,\n    private readonly opts: DownloadUrlDriverOps = {\n      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB\n      withGunzip: true,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n    this.downloadHelper = new RemoteFileDownloader(httpClient);\n  }\n\n  /** Use to get a path result inside a computable context */\n  getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;\n\n  /** Returns a Computable that do the work */\n  getUrl(url: URL): Computable<UrlResult | undefined>;\n\n  /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */\n  getUrl(url: URL, ctx?: ComputableCtx): Computable<UrlResult | undefined> | UrlResult | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));\n    return this.getResult({ type: \"url\", url }, url.toString(), ctx);\n  }\n\n  getLocalTgz(path: string, mtime: string, ctx: ComputableCtx): UrlResult | undefined;\n  getLocalTgz(path: string, mtime: string): Computable<UrlResult | undefined>;\n\n  /** Mirror of {@link getUrl} for a local `.tgz` archive: lazily unpacks it\n   * into the recycled cache (so a live UI keeps the folder pinned) and returns\n   * a custom-protocol URL. Keyed on `path + \":\" + mtime`, so the unpacked UI is\n   * re-derived when the source block changes. */\n  getLocalTgz(\n    path: string,\n    mtime: string,\n    ctx?: ComputableCtx,\n  ): Computable<UrlResult | undefined> | UrlResult | undefined {\n    if (ctx === undefined) return Computable.make((c) => this.getLocalTgz(path, mtime, c));\n    return this.getResult({ type: \"local-tgz\", path, mtime }, `${path}:${mtime}`, ctx);\n  }\n\n  private getResult(\n    source: DownloadSource,\n    key: string,\n    ctx: ComputableCtx,\n  ): UrlResult | undefined {\n    const callerId = randomUUID();\n\n    // read as ~ golang's defer\n    ctx.addOnDestroy(() => this.releaseByKey(key, callerId));\n\n    const result = this.getResultNoCtx(source, key, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,\n      );\n\n    return result;\n  }\n\n  private getResultNoCtx(source: DownloadSource, key: string, w: Watcher, callerId: string) {\n    const task = this.taskByKey.get(key);\n\n    if (task !== undefined) {\n      task.attach(w, callerId);\n      return task.getUrl();\n    }\n\n    const newTask = this.setNewTask(w, source, key, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !(e instanceof URLAborted) && !isDownloadNetworkError400(e),\n    });\n\n    return newTask.getUrl();\n  }\n\n  getPathForBlockUI(url: string): string {\n    if (!isBlockUIURL(url)) {\n      throw new Error(`getPathForBlockUI: ${url} is invalid`);\n    }\n\n    return getPathForBlockUIURL(this.signer, url, this.saveDir);\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadByUrlTask, callerId: string) {\n    await task.download(this.downloadHelper, this.opts.withGunzip);\n    // Might be undefined if a error happened\n    if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releaseByKey(key: string, callerId: string): Promise<void> {\n    const task = this.taskByKey.get(key);\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadByUrlTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.taskByKey.entries()).map(async ([, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, source: DownloadSource, key: string, callerId: string) {\n    const result = new DownloadByUrlTask(\n      this.logger,\n      this.getExtractionPath(key),\n      source,\n      key,\n      this.signer,\n      this.saveDir,\n    );\n    result.attach(w, callerId);\n    this.taskByKey.set(key, result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadByUrlTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${task.key} removed: ${reason}`);\n    this.taskByKey.delete(task.key);\n  }\n\n  /** Directory an archive (url or local file) is extracted into; keyed by a\n   * hash of the task key so the same source maps to a stable on-disk path. */\n  private getExtractionPath(key: string): string {\n    const sha256 = createHash(\"sha256\").update(key).digest(\"hex\");\n    return path.join(this.saveDir, sha256);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAkDA,IAAa,oBAAb,MAAgF;CAW3D;CAEA;CACA;CACA;CAdnB;CAEA,4BAAoD,IAAI,IAAI;CAC5D;;;CAIA;CAEA,YACE,QACA,YACA,SACA,QACA,OAA8C;EAC5C,oBAAoB,IAAI,OAAO,OAAO;EACtC,YAAY;EACZ,sBAAsB;CACxB,GACA;EATiB,KAAA,SAAA;EAEA,KAAA,UAAA;EACA,KAAA,SAAA;EACA,KAAA,OAAA;EAMjB,KAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,oBAAoB;EAClF,KAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,kBAAkB;EACxD,KAAK,iBAAiB,IAAI,qBAAqB,UAAU;CAC3D;;CASA,OAAO,KAAU,KAAgF;EAE/F,IAAI,QAAQ,KAAA,GAAW,OAAO,WAAW,MAAM,MAAM,KAAK,OAAO,KAAK,CAAC,CAAC;EACxE,OAAO,KAAK,UAAU;GAAE,MAAM;GAAO;EAAI,GAAG,IAAI,SAAS,GAAG,GAAG;CACjE;;;;;CASA,YACE,MACA,OACA,KAC2D;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAO,WAAW,MAAM,MAAM,KAAK,YAAY,MAAM,OAAO,CAAC,CAAC;EACrF,OAAO,KAAK,UAAU;GAAE,MAAM;GAAa;GAAM;EAAM,GAAG,GAAG,KAAK,GAAG,SAAS,GAAG;CACnF;CAEA,UACE,QACA,KACA,KACuB;EACvB,MAAM,WAAW,WAAW;EAG5B,IAAI,mBAAmB,KAAK,aAAa,KAAK,QAAQ,CAAC;EAEvD,MAAM,SAAS,KAAK,eAAe,QAAQ,KAAK,IAAI,SAAS,QAAQ;EACrE,IAAI,QAAQ,QAAQ,KAAA,GAClB,IAAI,aACF,wFAAwF,QAC1F;EAEF,OAAO;CACT;CAEA,eAAuB,QAAwB,KAAa,GAAY,UAAkB;EACxF,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EAEnC,IAAI,SAAS,KAAA,GAAW;GACtB,KAAK,OAAO,GAAG,QAAQ;GACvB,OAAO,KAAK,OAAO;EACrB;EAEA,MAAM,UAAU,KAAK,WAAW,GAAG,QAAQ,KAAK,QAAQ;EACxD,KAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,QAAQ;GAClD,4BAA4B,MAAM,EAAE,aAAa,eAAe,CAAC,0BAA0B,CAAC;EAC9F,CAAC;EAED,OAAO,QAAQ,OAAO;CACxB;CAEA,kBAAkB,KAAqB;EACrC,IAAI,CAAC,aAAa,GAAG,GACnB,MAAM,IAAI,MAAM,sBAAsB,IAAI,YAAY;EAGxD,OAAO,qBAAqB,KAAK,QAAQ,KAAK,KAAK,OAAO;CAC5D;;CAGA,MAAM,YAAY,MAAyB,UAAkB;EAC3D,MAAM,KAAK,SAAS,KAAK,gBAAgB,KAAK,KAAK,UAAU;EAE7D,IAAI,KAAK,OAAO,CAAC,EAAE,QAAQ,KAAA,GAAW,KAAK,MAAM,SAAS,MAAM,QAAQ;CAC1E;;;CAIA,MAAM,aAAa,KAAa,UAAiC;EAC/D,MAAM,OAAO,KAAK,UAAU,IAAI,GAAG;EACnC,IAAI,QAAQ,KAAA,GAAW;EAEvB,IAAI,KAAK,MAAM,WAAW,KAAK,IAAI,GAAG;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ;GAE1D,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAA4B;IAC9C,MAAM,QAAQ,KAAK,IAAI;IACvB,KAAK,MAAM,YAAY,IAAI;IAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,GAClF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,wBACnD;CAEN;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,MAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,UAAU,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,UAAU;GAC3D,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK,MAAM,YAAY,IAAI;GAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,yCACnD;EACF,CAAC,CACH;CACF;CAEA,WAAmB,GAAY,QAAwB,KAAa,UAAkB;EACpF,MAAM,SAAS,IAAI,kBACjB,KAAK,QACL,KAAK,kBAAkB,GAAG,GAC1B,QACA,KACA,KAAK,QACL,KAAK,OACP;EACA,OAAO,OAAO,GAAG,QAAQ;EACzB,KAAK,UAAU,IAAI,KAAK,MAAM;EAE9B,OAAO;CACT;CAEA,WAAmB,MAAyB,QAAgB;EAC1D,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,YAAY,KAAK,IAAI,YAAY,QAAQ;EACjE,KAAK,UAAU,OAAO,KAAK,GAAG;CAChC;;;CAIA,kBAA0B,KAAqB;EAC7C,MAAM,SAAS,WAAW,QAAQ,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,KAAK;EAC5D,OAAOA,OAAK,KAAK,KAAK,SAAS,MAAM;CACvC;AACF"}

package/dist/drivers/download_url/task.cjs

@@ -2,35 +2,44 @@ const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
 const require_download_errors = require("../../helpers/download_errors.cjs");
 const require_url = require("../urls/url.cjs");
 let node_fs_promises = require("node:fs/promises");
-node_fs_promises = require_runtime.__toESM(node_fs_promises);
+node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
+let node_fs = require("node:fs");
 let node_path = require("node:path");
-node_path = require_runtime.__toESM(node_path);
+node_path = require_runtime.__toESM(node_path, 1);
 let node_stream = require("node:stream");
 let _milaboratories_computable = require("@milaboratories/computable");
 let node_zlib = require("node:zlib");
-node_zlib = require_runtime.__toESM(node_zlib);
+node_zlib = require_runtime.__toESM(node_zlib, 1);
 let tar_fs = require("tar-fs");
-tar_fs = require_runtime.__toESM(tar_fs);
+tar_fs = require_runtime.__toESM(tar_fs, 1);
 //#region src/drivers/download_url/task.ts
-/** Downloads and extracts an archive to a directory. */
+/** Fetches (over HTTP or from a local file) and extracts an archive to a
+* directory. `key` is the task's identity in the driver's task map and cache. */
 var DownloadByUrlTask = class {
+	logger;
+	path;
+	source;
+	key;
+	signer;
+	saveDir;
 	counter = new _milaboratories_ts_helpers.CallersCounter();
 	change = new _milaboratories_computable.ChangeSource();
 	signalCtl = new AbortController();
 	error;
 	done = false;
 	size = 0;
-	constructor(logger, path, url, signer, saveDir) {
+	constructor(logger, path, source, key, signer, saveDir) {
 		this.logger = logger;
 		this.path = path;
-		this.url = url;
+		this.source = source;
+		this.key = key;
 		this.signer = signer;
 		this.saveDir = saveDir;
 	}
 	info() {
 		return {
-			url: this.url.toString(),
+			key: this.key,
 			path: this.path,
 			done: this.done,
 			size: this.size,
@@ -43,35 +52,42 @@ var DownloadByUrlTask = class {
 	}
 	async download(clientDownload, withGunzip) {
 		try {
-			const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
+			const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
 			this.setDone(size);
-			this.change.markChanged(`download of ${this.url} finished`);
+			this.change.markChanged(`download of ${this.key} finished`);
 		} catch (e) {
 			if (e instanceof URLAborted || require_download_errors.isDownloadNetworkError400(e) || this.signalCtl.signal.aborted) {
 				this.setError(e);
-				this.change.markChanged(`download of ${this.url} failed`);
+				this.change.markChanged(`download of ${this.key} failed`);
 				await rmRFDir(this.path);
 				return;
 			}
 			throw e;
 		}
 	}
-	async downloadAndUntar(clientDownload, withGunzip, signal) {
+	async fetchAndUntar(clientDownload, withGunzip, signal) {
 		await (0, _milaboratories_ts_helpers.ensureDirExists)(node_path.dirname(this.path));
 		if (await (0, _milaboratories_ts_helpers.fileExists)(this.path)) return await dirSize(this.path);
-		return await clientDownload.withContent(this.url.toString(), {}, { signal }, async (content, size) => {
-			let processedContent = content;
-			if (withGunzip) {
-				const gunzip = node_stream.Transform.toWeb(node_zlib.createGunzip());
-				processedContent = content.pipeThrough(gunzip, { signal });
-			}
-			await (0, _milaboratories_ts_helpers.createPathAtomically)(this.logger, this.path, async (fPath) => {
-				await node_fs_promises.mkdir(fPath);
-				const untar = node_stream.Writable.toWeb(tar_fs.extract(fPath));
-				await processedContent.pipeTo(untar, { signal });
-			});
+		if (this.source.type === "url") return await clientDownload.withContent(this.source.url.toString(), {}, { signal }, async (content, size) => {
+			await this.untarStream(content, withGunzip, signal);
 			return size;
 		});
+		const content = node_stream.Readable.toWeb((0, node_fs.createReadStream)(this.source.path));
+		await this.untarStream(content, withGunzip, signal);
+		return await dirSize(this.path);
+	}
+	/** Optionally gunzips, then untars the content stream into `this.path`. */
+	async untarStream(content, withGunzip, signal) {
+		let processedContent = content;
+		if (withGunzip) {
+			const gunzip = node_stream.Transform.toWeb(node_zlib.createGunzip());
+			processedContent = content.pipeThrough(gunzip, { signal });
+		}
+		await (0, _milaboratories_ts_helpers.createPathAtomically)(this.logger, this.path, async (fPath) => {
+			await node_fs_promises.mkdir(fPath);
+			const untar = node_stream.Writable.toWeb(tar_fs.extract(fPath));
+			await processedContent.pipeTo(untar, { signal });
+		});
 	}
 	getUrl() {
 		if (this.done) return { url: require_url.newBlockUIURL(this.signer, this.saveDir, (0, _milaboratories_ts_helpers.notEmpty)(this.path)) };

package/dist/drivers/download_url/task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (\n        e instanceof URLAborted ||\n        isDownloadNetworkError400(e) ||\n        this.signalCtl.signal.aborted\n      ) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAIA,2BAAAA,gBAAgB;CACvC,SAAkB,IAAIC,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OACE,aAAa,cACbC,wBAAAA,0BAA0B,EAAE,IAC5B,KAAK,UAAU,OAAO,SACtB;AACA,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,SAAA,GAAA,2BAAA,iBAAsBC,UAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAASC,YAAAA,UAAU,MAAMC,UAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAMC,iBAAI,MAAM,MAAM;IACtB,MAAM,QAAQC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAKC,YAAAA,cAAc,KAAK,QAAQ,KAAK,UAAA,GAAA,2BAAA,UAAkB,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAMH,iBAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQH,UAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMG,iBAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAMA,iBAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}
+{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Readable","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport { createReadStream } from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable, Transform, Writable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Source an archive is fetched from before extraction: a remote URL or a\n * local `.tgz` file on disk. Both flow through the same untar machinery. */\nexport type DownloadSource =\n  | { type: \"url\"; url: URL }\n  | { type: \"local-tgz\"; path: string; mtime: string };\n\n/** Fetches (over HTTP or from a local file) and extracts an archive to a\n * directory. `key` is the task's identity in the driver's task map and cache. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly source: DownloadSource,\n    readonly key: string,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      key: this.key,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.key} finished`);\n    } catch (e: unknown) {\n      if (\n        e instanceof URLAborted ||\n        isDownloadNetworkError400(e) ||\n        this.signalCtl.signal.aborted\n      ) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.key} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async fetchAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    if (this.source.type === \"url\") {\n      return await clientDownload.withContent(\n        this.source.url.toString(),\n        {},\n        { signal },\n        async (content, size) => {\n          await this.untarStream(content, withGunzip, signal);\n          return size;\n        },\n      );\n    }\n\n    // local-tgz: read the archive from disk and untar through the same path.\n    const content = Readable.toWeb(createReadStream(this.source.path)) as ReadableStream;\n    await this.untarStream(content, withGunzip, signal);\n    return await dirSize(this.path);\n  }\n\n  /** Optionally gunzips, then untars the content stream into `this.path`. */\n  private async untarStream(\n    content: ReadableStream,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<void> {\n    let processedContent = content;\n    if (withGunzip) {\n      const gunzip = Transform.toWeb(zlib.createGunzip());\n      processedContent = content.pipeThrough(gunzip, { signal });\n    }\n\n    await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n      await fsp.mkdir(fPath); // throws if a directory already exists.\n      const untar = Writable.toWeb(tar.extract(fPath));\n      await processedContent.pipeTo(untar, { signal });\n    });\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA8BA,IAAa,oBAAb,MAA+B;CASV;CACR;CACA;CACA;CACA;CACA;CAbX,UAAmB,IAAIA,2BAAAA,eAAe;CACtC,SAAkB,IAAIC,2BAAAA,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,QACA,KACA,QACA,SACA;EANiB,KAAA,SAAA;EACR,KAAA,OAAA;EACA,KAAA,SAAA;EACA,KAAA,MAAA;EACA,KAAA,SAAA;EACA,KAAA,UAAA;CACR;CAEH,OAAc;EACZ,OAAO;GACL,KAAK,KAAK;GACV,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;EACd;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,SAAS,gBAAsC,YAAqB;EACxE,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,cAAc,gBAAgB,YAAY,KAAK,UAAU,MAAM;GACvF,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,UAAU;EAC5D,SAAS,GAAY;GACnB,IACE,aAAa,cACbC,wBAAAA,0BAA0B,CAAC,KAC3B,KAAK,UAAU,OAAO,SACtB;IACA,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,QAAQ;IAExD,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;CAEA,MAAc,cACZ,gBACA,YACA,QACiB;EACjB,OAAA,GAAA,2BAAA,gBAAA,CAAsBC,UAAK,QAAQ,KAAK,IAAI,CAAC;EAE7C,IAAI,OAAA,GAAA,2BAAA,WAAA,CAAiB,KAAK,IAAI,GAC5B,OAAO,MAAM,QAAQ,KAAK,IAAI;EAGhC,IAAI,KAAK,OAAO,SAAS,OACvB,OAAO,MAAM,eAAe,YAC1B,KAAK,OAAO,IAAI,SAAS,GACzB,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;GAClD,OAAO;EACT,CACF;EAIF,MAAM,UAAUC,YAAAA,SAAS,OAAA,GAAA,QAAA,iBAAA,CAAuB,KAAK,OAAO,IAAI,CAAC;EACjE,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;EAClD,OAAO,MAAM,QAAQ,KAAK,IAAI;CAChC;;CAGA,MAAc,YACZ,SACA,YACA,QACe;EACf,IAAI,mBAAmB;EACvB,IAAI,YAAY;GACd,MAAM,SAASC,YAAAA,UAAU,MAAMC,UAAK,aAAa,CAAC;GAClD,mBAAmB,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC;EAC3D;EAEA,OAAA,GAAA,2BAAA,qBAAA,CAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;GAC1E,MAAMC,iBAAI,MAAM,KAAK;GACrB,MAAM,QAAQC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,KAAK,CAAC;GAC/C,MAAM,iBAAiB,OAAO,OAAO,EAAE,OAAO,CAAC;EACjD,CAAC;CACH;CAEA,SAAgC;EAC9B,IAAI,KAAK,MACP,OAAO,EACL,KAAKC,YAAAA,cAAc,KAAK,QAAQ,KAAK,UAAA,GAAA,2BAAA,SAAA,CAAkB,KAAK,IAAI,CAAC,EACnE;EAEF,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;CACd;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;AACT;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAMH,iBAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQJ,UAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADYI,iBAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,MAAc;CAC1C,MAAMA,iBAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD"}

package/dist/drivers/download_url/task.d.ts

@@ -4,11 +4,23 @@ import { CallersCounter, MiLogger, Signer } from "@milaboratories/ts-helpers";
 import { ChangeSource, Watcher } from "@milaboratories/computable";
 
 //#region src/drivers/download_url/task.d.ts
-/** Downloads and extracts an archive to a directory. */
+/** Source an archive is fetched from before extraction: a remote URL or a
+ * local `.tgz` file on disk. Both flow through the same untar machinery. */
+type DownloadSource = {
+  type: "url";
+  url: URL;
+} | {
+  type: "local-tgz";
+  path: string;
+  mtime: string;
+};
+/** Fetches (over HTTP or from a local file) and extracts an archive to a
+ * directory. `key` is the task's identity in the driver's task map and cache. */
 declare class DownloadByUrlTask {
   private readonly logger;
   readonly path: string;
-  readonly url: URL;
+  readonly source: DownloadSource;
+  readonly key: string;
   readonly signer: Signer;
   readonly saveDir: string;
   readonly counter: CallersCounter;
@@ -17,9 +29,9 @@ declare class DownloadByUrlTask {
   error: string | undefined;
   done: boolean;
   size: number;
-  constructor(logger: MiLogger, path: string, url: URL, signer: Signer, saveDir: string);
+  constructor(logger: MiLogger, path: string, source: DownloadSource, key: string, signer: Signer, saveDir: string);
   info(): {
-    url: string;
+    key: string;
     path: string;
     done: boolean;
     size: number;
@@ -27,12 +39,14 @@ declare class DownloadByUrlTask {
   };
   attach(w: Watcher, callerId: string): void;
   download(clientDownload: RemoteFileDownloader, withGunzip: boolean): Promise<void>;
-  private downloadAndUntar;
+  private fetchAndUntar;
+  /** Optionally gunzips, then untars the content stream into `this.path`. */
+  private untarStream;
   getUrl(): UrlResult | undefined;
   private setDone;
   private setError;
   abort(reason: string): void;
 }
 //#endregion
-export { DownloadByUrlTask };
+export { DownloadByUrlTask, DownloadSource };
 //# sourceMappingURL=task.d.ts.map

package/dist/drivers/download_url/task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_url/task.ts"],"mappings":";;;;;;AAqBA;AAAA,cAAa,iBAAA;EAAA,iBASQ,MAAA;EAAA,SACR,IAAA;EAAA,SACA,GAAA,EAAK,GAAA;EAAA,SACL,MAAA,EAAQ,MAAA;EAAA,SACR,OAAA;EAAA,SAZF,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;cAGmB,MAAA,EAAQ,QAAA,EAChB,IAAA,UACA,GAAA,EAAK,GAAA,EACL,MAAA,EAAQ,MAAA,EACR,OAAA;EAGJ,IAAA,CAAA;;;;;;;EAUP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAS,cAAA,EAAgB,oBAAA,EAAsB,UAAA,YAAmB,OAAA;EAAA,QAsB1D,gBAAA;EAmCd,MAAA,CAAA,GAAU,SAAA;EAAA,QAWF,OAAA;EAAA,QAKA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA"}
+{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_url/task.ts"],"mappings":";;;;;;AAwBA;;AAAA,KAAY,cAAA;EACN,IAAA;EAAa,GAAA,EAAK,GAAG;AAAA;EACrB,IAAA;EAAmB,IAAA;EAAc,KAAA;AAAA;;AAAK;cAI/B,iBAAA;EAAA,iBASQ,MAAA;EAAA,SACR,IAAA;EAAA,SACA,MAAA,EAAQ,cAAA;EAAA,SACR,GAAA;EAAA,SACA,MAAA,EAAQ,MAAA;EAAA,SACR,OAAA;EAAA,SAbF,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;cAGmB,MAAA,EAAQ,QAAA,EAChB,IAAA,UACA,MAAA,EAAQ,cAAA,EACR,GAAA,UACA,MAAA,EAAQ,MAAA,EACR,OAAA;EAGJ,IAAA;;;;;;;EAUP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAS,cAAA,EAAgB,oBAAA,EAAsB,UAAA,YAAmB,OAAA;EAAA,QAsB1D,aAAA;EArDL;EAAA,QAmFK,WAAA;EAkBd,MAAA,IAAU,SAAA;EAAA,QAWF,OAAA;EAAA,QAKA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA"}

package/dist/drivers/download_url/task.js

@@ -2,30 +2,39 @@ import { isDownloadNetworkError400 } from "../../helpers/download_errors.js";
 import { newBlockUIURL } from "../urls/url.js";
 import * as fsp from "node:fs/promises";
 import { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from "@milaboratories/ts-helpers";
+import { createReadStream } from "node:fs";
 import * as path$1 from "node:path";
-import { Transform, Writable } from "node:stream";
+import { Readable, Transform, Writable } from "node:stream";
 import { ChangeSource } from "@milaboratories/computable";
 import * as zlib from "node:zlib";
 import * as tar from "tar-fs";
 //#region src/drivers/download_url/task.ts
-/** Downloads and extracts an archive to a directory. */
+/** Fetches (over HTTP or from a local file) and extracts an archive to a
+* directory. `key` is the task's identity in the driver's task map and cache. */
 var DownloadByUrlTask = class {
+	logger;
+	path;
+	source;
+	key;
+	signer;
+	saveDir;
 	counter = new CallersCounter();
 	change = new ChangeSource();
 	signalCtl = new AbortController();
 	error;
 	done = false;
 	size = 0;
-	constructor(logger, path, url, signer, saveDir) {
+	constructor(logger, path, source, key, signer, saveDir) {
 		this.logger = logger;
 		this.path = path;
-		this.url = url;
+		this.source = source;
+		this.key = key;
 		this.signer = signer;
 		this.saveDir = saveDir;
 	}
 	info() {
 		return {
-			url: this.url.toString(),
+			key: this.key,
 			path: this.path,
 			done: this.done,
 			size: this.size,
@@ -38,35 +47,42 @@ var DownloadByUrlTask = class {
 	}
 	async download(clientDownload, withGunzip) {
 		try {
-			const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
+			const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
 			this.setDone(size);
-			this.change.markChanged(`download of ${this.url} finished`);
+			this.change.markChanged(`download of ${this.key} finished`);
 		} catch (e) {
 			if (e instanceof URLAborted || isDownloadNetworkError400(e) || this.signalCtl.signal.aborted) {
 				this.setError(e);
-				this.change.markChanged(`download of ${this.url} failed`);
+				this.change.markChanged(`download of ${this.key} failed`);
 				await rmRFDir(this.path);
 				return;
 			}
 			throw e;
 		}
 	}
-	async downloadAndUntar(clientDownload, withGunzip, signal) {
+	async fetchAndUntar(clientDownload, withGunzip, signal) {
 		await ensureDirExists(path$1.dirname(this.path));
 		if (await fileExists(this.path)) return await dirSize(this.path);
-		return await clientDownload.withContent(this.url.toString(), {}, { signal }, async (content, size) => {
-			let processedContent = content;
-			if (withGunzip) {
-				const gunzip = Transform.toWeb(zlib.createGunzip());
-				processedContent = content.pipeThrough(gunzip, { signal });
-			}
-			await createPathAtomically(this.logger, this.path, async (fPath) => {
-				await fsp.mkdir(fPath);
-				const untar = Writable.toWeb(tar.extract(fPath));
-				await processedContent.pipeTo(untar, { signal });
-			});
+		if (this.source.type === "url") return await clientDownload.withContent(this.source.url.toString(), {}, { signal }, async (content, size) => {
+			await this.untarStream(content, withGunzip, signal);
 			return size;
 		});
+		const content = Readable.toWeb(createReadStream(this.source.path));
+		await this.untarStream(content, withGunzip, signal);
+		return await dirSize(this.path);
+	}
+	/** Optionally gunzips, then untars the content stream into `this.path`. */
+	async untarStream(content, withGunzip, signal) {
+		let processedContent = content;
+		if (withGunzip) {
+			const gunzip = Transform.toWeb(zlib.createGunzip());
+			processedContent = content.pipeThrough(gunzip, { signal });
+		}
+		await createPathAtomically(this.logger, this.path, async (fPath) => {
+			await fsp.mkdir(fPath);
+			const untar = Writable.toWeb(tar.extract(fPath));
+			await processedContent.pipeTo(untar, { signal });
+		});
 	}
 	getUrl() {
 		if (this.done) return { url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)) };

package/dist/drivers/download_url/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (\n        e instanceof URLAborted ||\n        isDownloadNetworkError400(e) ||\n        this.signalCtl.signal.aborted\n      ) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,KAAK,UAAU,OAAO,SACtB;AACA,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,QAAM,gBAAgBA,OAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,MAAM,WAAW,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAM,IAAI,MAAM,MAAM;IACtB,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAK,cAAc,KAAK,QAAQ,KAAK,SAAS,SAAS,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQA,OAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}
+{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport { createReadStream } from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable, Transform, Writable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Source an archive is fetched from before extraction: a remote URL or a\n * local `.tgz` file on disk. Both flow through the same untar machinery. */\nexport type DownloadSource =\n  | { type: \"url\"; url: URL }\n  | { type: \"local-tgz\"; path: string; mtime: string };\n\n/** Fetches (over HTTP or from a local file) and extracts an archive to a\n * directory. `key` is the task's identity in the driver's task map and cache. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly source: DownloadSource,\n    readonly key: string,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      key: this.key,\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.key} finished`);\n    } catch (e: unknown) {\n      if (\n        e instanceof URLAborted ||\n        isDownloadNetworkError400(e) ||\n        this.signalCtl.signal.aborted\n      ) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.key} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async fetchAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    if (this.source.type === \"url\") {\n      return await clientDownload.withContent(\n        this.source.url.toString(),\n        {},\n        { signal },\n        async (content, size) => {\n          await this.untarStream(content, withGunzip, signal);\n          return size;\n        },\n      );\n    }\n\n    // local-tgz: read the archive from disk and untar through the same path.\n    const content = Readable.toWeb(createReadStream(this.source.path)) as ReadableStream;\n    await this.untarStream(content, withGunzip, signal);\n    return await dirSize(this.path);\n  }\n\n  /** Optionally gunzips, then untars the content stream into `this.path`. */\n  private async untarStream(\n    content: ReadableStream,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<void> {\n    let processedContent = content;\n    if (withGunzip) {\n      const gunzip = Transform.toWeb(zlib.createGunzip());\n      processedContent = content.pipeThrough(gunzip, { signal });\n    }\n\n    await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n      await fsp.mkdir(fPath); // throws if a directory already exists.\n      const untar = Writable.toWeb(tar.extract(fPath));\n      await processedContent.pipeTo(untar, { signal });\n    });\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;AA8BA,IAAa,oBAAb,MAA+B;CASV;CACR;CACA;CACA;CACA;CACA;CAbX,UAAmB,IAAI,eAAe;CACtC,SAAkB,IAAI,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,QACA,KACA,QACA,SACA;EANiB,KAAA,SAAA;EACR,KAAA,OAAA;EACA,KAAA,SAAA;EACA,KAAA,MAAA;EACA,KAAA,SAAA;EACA,KAAA,UAAA;CACR;CAEH,OAAc;EACZ,OAAO;GACL,KAAK,KAAK;GACV,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;EACd;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,SAAS,gBAAsC,YAAqB;EACxE,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,cAAc,gBAAgB,YAAY,KAAK,UAAU,MAAM;GACvF,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,UAAU;EAC5D,SAAS,GAAY;GACnB,IACE,aAAa,cACb,0BAA0B,CAAC,KAC3B,KAAK,UAAU,OAAO,SACtB;IACA,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,QAAQ;IAExD,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;CAEA,MAAc,cACZ,gBACA,YACA,QACiB;EACjB,MAAM,gBAAgBA,OAAK,QAAQ,KAAK,IAAI,CAAC;EAE7C,IAAI,MAAM,WAAW,KAAK,IAAI,GAC5B,OAAO,MAAM,QAAQ,KAAK,IAAI;EAGhC,IAAI,KAAK,OAAO,SAAS,OACvB,OAAO,MAAM,eAAe,YAC1B,KAAK,OAAO,IAAI,SAAS,GACzB,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;GAClD,OAAO;EACT,CACF;EAIF,MAAM,UAAU,SAAS,MAAM,iBAAiB,KAAK,OAAO,IAAI,CAAC;EACjE,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;EAClD,OAAO,MAAM,QAAQ,KAAK,IAAI;CAChC;;CAGA,MAAc,YACZ,SACA,YACA,QACe;EACf,IAAI,mBAAmB;EACvB,IAAI,YAAY;GACd,MAAM,SAAS,UAAU,MAAM,KAAK,aAAa,CAAC;GAClD,mBAAmB,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC;EAC3D;EAEA,MAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;GAC1E,MAAM,IAAI,MAAM,KAAK;GACrB,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,KAAK,CAAC;GAC/C,MAAM,iBAAiB,OAAO,OAAO,EAAE,OAAO,CAAC;EACjD,CAAC;CACH;CAEA,SAAgC;EAC9B,IAAI,KAAK,MACP,OAAO,EACL,KAAK,cAAc,KAAK,QAAQ,KAAK,SAAS,SAAS,KAAK,IAAI,CAAC,EACnE;EAEF,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;CACd;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;AACT;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQA,OAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADY,IAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,MAAc;CAC1C,MAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD"}

package/dist/drivers/helpers/download_local_handle.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_local_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n  return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n  return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n  const parsed = handle.match(localHandleRegex);\n\n  if (parsed === null) {\n    throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { path, signature } = parsed.groups!;\n  signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n  return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;AAC5E,QAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,KAAK;;AAG3D,SAAgB,kBAAkB,QAA2C;AAC3E,QAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;;AAGhD,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,iBAAiB;AAE7C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,SAAS;CAG7E,MAAM,EAAE,MAAM,cAAc,OAAO;AACnC,QAAO,OAAO,MAAM,WAAW,sCAAsC,SAAS;AAE9E,QAAO;EAAE;EAAM;EAAW"}
+{"version":3,"file":"download_local_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n  return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n  return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n  const parsed = handle.match(localHandleRegex);\n\n  if (parsed === null) {\n    throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { path, signature } = parsed.groups!;\n  signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n  return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;CAC5E,OAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,IAAI;AAC1D;AAEA,SAAgB,kBAAkB,QAA2C;CAC3E,OAAO,QAAQ,OAAO,MAAM,gBAAgB,CAAC;AAC/C;AAEA,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,gBAAgB;CAE5C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,QAAQ;CAG5E,MAAM,EAAE,MAAM,cAAc,OAAO;CACnC,OAAO,OAAO,MAAM,WAAW,sCAAsC,QAAQ;CAE7E,OAAO;EAAE;EAAM;CAAU;AAC3B"}

package/dist/drivers/helpers/download_local_handle.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_local_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n  return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n  return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n  const parsed = handle.match(localHandleRegex);\n\n  if (parsed === null) {\n    throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { path, signature } = parsed.groups!;\n  signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n  return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;AAC5E,QAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,KAAK;;AAG3D,SAAgB,kBAAkB,QAA2C;AAC3E,QAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;;AAGhD,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,iBAAiB;AAE7C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,SAAS;CAG7E,MAAM,EAAE,MAAM,cAAc,OAAO;AACnC,QAAO,OAAO,MAAM,WAAW,sCAAsC,SAAS;AAE9E,QAAO;EAAE;EAAM;EAAW"}
+{"version":3,"file":"download_local_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n  return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n  return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n  const parsed = handle.match(localHandleRegex);\n\n  if (parsed === null) {\n    throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { path, signature } = parsed.groups!;\n  signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n  return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;CAC5E,OAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,IAAI;AAC1D;AAEA,SAAgB,kBAAkB,QAA2C;CAC3E,OAAO,QAAQ,OAAO,MAAM,gBAAgB,CAAC;AAC/C;AAEA,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,gBAAgB;CAE5C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,QAAQ;CAG5E,MAAM,EAAE,MAAM,cAAc,OAAO;CACnC,OAAO,OAAO,MAAM,WAAW,sCAAsC,QAAQ;CAE7E,OAAO;EAAE;EAAM;CAAU;AAC3B"}

package/dist/drivers/helpers/download_remote_handle.cjs

@@ -1,4 +1,3 @@
-require("../../_virtual/_rolldown/runtime.cjs");
 const require_types = require("../types.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 //#region src/drivers/helpers/download_remote_handle.ts

package/dist/drivers/helpers/download_remote_handle.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n  createSignedResourceId,\n  parseSignedResourceId,\n  signatureToBase64Url,\n  base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n  const sigSegment = signatureToBase64Url(resourceSig);\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n    parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n  return {\n    info: {\n      id: createSignedResourceId(BigInt(resourceId), sig),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,iBAAA,GAAA,0BAAA,uBAAsC,MAAM,GAAG;CAC5E,MAAM,cAAA,GAAA,0BAAA,sBAAkC,YAAY;CACpD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAGA,cAAAA,QAAQ,MAAM;AAElG,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;AAET,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;CAEhF,MAAM,MAAM,eAAA,GAAA,0BAAA,sBAAmC,YAAY,GAAG,KAAA;AAE9D,QAAO;EACL,MAAM;GACJ,KAAA,GAAA,0BAAA,wBAA2B,OAAO,WAAW,EAAE,IAAI;GACnD,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}
+{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n  createSignedResourceId,\n  parseSignedResourceId,\n  signatureToBase64Url,\n  base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n  const sigSegment = signatureToBase64Url(resourceSig);\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n    parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n  return {\n    info: {\n      id: createSignedResourceId(BigInt(resourceId), sig),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,iBAAA,GAAA,0BAAA,sBAAA,CAAsC,MAAM,EAAE;CAC3E,MAAM,cAAA,GAAA,0BAAA,qBAAA,CAAkC,WAAW;CACnD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAGA,cAAAA,QAAQ,KAAK;CAEjG,OAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,OAAO;AACjE;AAEA,SAAgB,mBAAmB,QAA4C;CAC7E,OAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;AAChD;AAEA,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,iBAAiB;CAC7C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,QAAQ;CAG7E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;CAET,OAAO,OAAO,SAAS,WAAW,qCAAqC,QAAQ;CAE/E,MAAM,MAAM,eAAA,GAAA,0BAAA,qBAAA,CAAmC,WAAW,IAAI,KAAA;CAE9D,OAAO;EACL,MAAM;GACJ,KAAA,GAAA,0BAAA,uBAAA,CAA2B,OAAO,UAAU,GAAG,GAAG;GAClD,MAAM;IAAE,MAAM;IAAc,SAAS;GAAgB;EACvD;EACA,MAAM,OAAO,IAAI;CACnB;AACF"}

package/dist/drivers/helpers/download_remote_handle.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n  createSignedResourceId,\n  parseSignedResourceId,\n  signatureToBase64Url,\n  base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n  const sigSegment = signatureToBase64Url(resourceSig);\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n    parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n  return {\n    info: {\n      id: createSignedResourceId(BigInt(resourceId), sig),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,gBAAgB,sBAAsB,MAAM,GAAG;CAC5E,MAAM,aAAa,qBAAqB,YAAY;CACpD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,MAAM;AAElG,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;AAET,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;CAEhF,MAAM,MAAM,cAAc,qBAAqB,YAAY,GAAG,KAAA;AAE9D,QAAO;EACL,MAAM;GACJ,IAAI,uBAAuB,OAAO,WAAW,EAAE,IAAI;GACnD,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}
+{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n  createSignedResourceId,\n  parseSignedResourceId,\n  signatureToBase64Url,\n  base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n  const sigSegment = signatureToBase64Url(resourceSig);\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n    parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n  return {\n    info: {\n      id: createSignedResourceId(BigInt(resourceId), sig),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,gBAAgB,sBAAsB,MAAM,EAAE;CAC3E,MAAM,aAAa,qBAAqB,WAAW;CACnD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,KAAK;CAEjG,OAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,OAAO;AACjE;AAEA,SAAgB,mBAAmB,QAA4C;CAC7E,OAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;AAChD;AAEA,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,iBAAiB;CAC7C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,QAAQ;CAG7E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;CAET,OAAO,OAAO,SAAS,WAAW,qCAAqC,QAAQ;CAE/E,MAAM,MAAM,cAAc,qBAAqB,WAAW,IAAI,KAAA;CAE9D,OAAO;EACL,MAAM;GACJ,IAAI,uBAAuB,OAAO,UAAU,GAAG,GAAG;GAClD,MAAM;IAAE,MAAM;IAAc,SAAS;GAAgB;EACvD;EACA,MAAM,OAAO,IAAI;CACnB;AACF"}

package/dist/drivers/helpers/files_cache.cjs

@@ -1,4 +1,3 @@
-require("../../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 //#region src/drivers/helpers/files_cache.ts
 /** Holds counters of how many callers need the file.
@@ -6,6 +5,7 @@ let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 * remove not needed blobs one by one.
 * If all the files are needed, do nothing. */
 var FilesCache = class {
+	softSizeBytes;
 	cache = /* @__PURE__ */ new Map();
 	totalSizeBytes = 0;
 	constructor(softSizeBytes) {