@milaboratories/pl-drivers 1.9.0 → 1.9.1

package/dist/drivers/download_blob/sparse_cache/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from '@milaboratories/pl-model-common';\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from '@milaboratories/ts-helpers';\nimport { promises as fs } from 'node:fs';\nimport path from 'node:path';\nimport { addRange, doesRangeExist, Ranges, rangesFileName, rangesFilePostfix, rangesSize, readRangesFile, writeRangesFile } from './ranges';\nimport { writeToSparseFile } from './file';\nimport { functions } from '@milaboratories/helpers';\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = 'CorruptedRangesError';\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = '.sparse.bin';\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock()\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    })\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    })\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number; }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: `\n        + `range: ${JSON.stringify(range)}, data: ${data.length}`\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"names":["path","fs"],"mappings":";;;;;;;AAQA;AACM,MAAO,oBAAqB,SAAQ,KAAK,CAAA;IAC7C,IAAI,GAAG,sBAAsB;AAC9B;AASD;MACa,mBAAmB,CAAA;AAEX,IAAA,MAAA;AACA,IAAA,QAAA;IAFnB,WAAA,CACmB,MAAgB,EAChB,QAAgB,EAAA;QADhB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,QAAQ,GAAR,QAAQ;IACxB;AAEK,IAAA,KAAK,CAAC,GAAW,EAAA;AACvB,QAAA,OAAOA,aAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IACtD;IAEA,MAAM,GAAG,CAAC,GAAW,EAAA;AACnB,QAAA,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3D;AAEA,IAAA,MAAM,GAAG,CAAC,GAAW,EAAE,MAAc,EAAA;AACnC,QAAA,OAAO,MAAM,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACpE;IAEA,MAAM,MAAM,CAAC,GAAW,EAAA;QACtB,MAAMC,QAAE,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B;AACD;AAWD;MACa,iBAAiB,CAAA;AAIT,IAAA,MAAA;AACA,IAAA,QAAA;IAJF,MAAM,GAAG,aAAa;IAEvC,WAAA,CACmB,MAAgB,EAChB,QAAgB,EAAA;QADhB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,QAAQ,GAAR,QAAQ;IACxB;AAEH,IAAA,MAAM,GAAG,GAAA;AACP,QAAA,MAAM,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;QACpC,MAAM,KAAK,GAAG,MAAMA,QAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;AAC7C,QAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrD;IAEA,MAAM,MAAM,CAAC,GAAW,EAAA;QACtB,OAAO,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC;AAEA,IAAA,IAAI,CAAC,GAAW,EAAA;AACd,QAAA,OAAOD,aAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;IACpD;AAEA,IAAA,MAAM,KAAK,CAAC,GAAW,EAAE,IAAgB,EAAE,IAAY,EAAA;AACrD,QAAA,MAAM,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;QACpC,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC;IACpF;IAEA,MAAM,MAAM,CAAC,GAAW,EAAA;QACtB,MAAMC,QAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B;AACD;AAED;MACa,WAAW,CAAA;AAUJ,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;;;AAVV,IAAA,IAAI,GAAG,IAAI,SAAS,CAAC,SAAS,EAAE;AAEjC,IAAA,mBAAmB,GAAG,IAAI,GAAG,EAAgB;IAC7C,IAAI,GAAG,CAAC;AAEf,IAAA,WAAA,CACkB,MAAgB;;IAEhB,OAAe,EACf,MAAyB,EACzB,OAA0B,EAAA;QAJ1B,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACtB;AAEH;AAC8B;AAC9B,IAAA,MAAM,KAAK,GAAA;QACT,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,YAAW;AACnC,YAAA,MAAM,IAAI,CAAC,WAAW,EAAE;AAC1B,QAAA,CAAC,CAAC;IACJ;AAEA;AAC8B;AAC9B,IAAA,MAAM,GAAG,CAAC,GAAW,EAAE,KAAiB,EAAA;QACtC,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,YAAW;YAC1C,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC;AACzC,QAAA,CAAC,CAAC;IACJ;AAEA;AAC8B;AAC9B,IAAA,MAAM,GAAG,CAAC,GAAW,EAAE,KAAiB,EAAE,IAAgB,EAAA;QACxD,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,YAAW;YACnC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC;AACxC,QAAA,CAAC,CAAC;IACJ;AAEQ,IAAA,MAAM,WAAW,GAAA;AACvB,QAAA,IAAI,CAAC,IAAI,GAAG,CAAC;AACb,QAAA,IAAI,CAAC,mBAAmB,GAAG,IAAI,GAAG,EAAgB;AAElD,QAAA,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE;;;;QAItB,KAAK,MAAM,GAAG,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE;YAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;AACzC,YAAA,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;YAC/B,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC;QACxC;IACF;AAEQ,IAAA,MAAM,SAAS,CAAC,GAAW,EAAE,KAAiB,EAAA;;;;;QAKpD,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAClC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC;YAE7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxC,YAAA,IAAI,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;gBACjC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;YAC/B;AAEA,YAAA,OAAO,SAAS;QAClB;AAEA,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,MAAM,SAAS,CAAC,GAAW,EAAE,KAAoC,EAAE,IAAgB,EAAA;QACzF,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC;AAC/C,QAAA,MAAM,IAAI,CAAC,aAAa,EAAE;IAC5B;;AAGA,IAAA,MAAM,kBAAkB,CAAC,GAAW,EAAE,KAAiB,EAAE,IAAgB,EAAA;AACvE,QAAA,IAAI,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE;AACzC,YAAA,MAAM,IAAI,KAAK,CACb,CAAA,+BAAA,EAAkC,GAAG,CAAA,0BAAA;AACnC,kBAAA,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA,QAAA,EAAW,IAAI,CAAC,MAAM,CAAA,CAAE,CAC1D;QACH;QAEA,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC;QAE7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxC,QAAA,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;AAE/B,QAAA,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QAE/C,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC;AACzC,QAAA,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,SAAS,CAAC;QAElC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC;IACvC;;AAGA,IAAA,MAAM,aAAa,GAAA;QACjB,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,mBAAmB,CAAC;AACnD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAE3E,OAAO,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE;YAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,UAAU,EAAE;gBACf;YACF;AACA,YAAA,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,UAAU;YAE3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxC,YAAA,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC;AAC/B,YAAA,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;QACjB;IACF;;IAGQ,MAAM,SAAS,CAAC,GAAW,EAAA;AACjC,QAAA,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACnC;QAAE,OAAO,CAAU,EAAE;AACnB,YAAA,IAAI,CAAC,YAAY,oBAAoB,EAAE;;;AAGrC,gBAAA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;AACrB,gBAAA,MAAM,IAAI,CAAC,WAAW,EAAE;gBAExB,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YACnC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,KAAK,CAAC,GAAW,EAAA;QAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;QAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;AAC7B,QAAA,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC;IACtC;AACD;AAED;AACA,eAAe,QAAQ,CAAI,IAAyB,EAAE,EAAoB,EAAA;AACxE,IAAA,IAAI;AACF,QAAA,MAAM,IAAI,CAAC,YAAY,EAAE;QACzB,OAAO,MAAM,EAAE,EAAE;IACnB;YAAU;QACR,IAAI,CAAC,OAAO,EAAE;IAChB;AACF;;;;"}

package/dist/drivers/download_blob/sparse_cache/file.cjs

@@ -0,0 +1,61 @@
+'use strict';
+
+var tsHelpers = require('@milaboratories/ts-helpers');
+var fsp = require('node:fs/promises');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var fsp__namespace = /*#__PURE__*/_interopNamespaceDefault(fsp);
+
+/** Creates a sparse file for all systems
+ * Table of what supports sparse files:
+ * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */
+async function createSparseFile(logger, path, platform) {
+    try {
+        const ensureCreated = await fsp__namespace.open(path, 'w');
+        await ensureCreated.close();
+        await ensureSparseOnWindows(path, platform);
+    }
+    catch (error) {
+        logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);
+    }
+}
+/** Sets a sparse flag on Windows.
+ * We could check the file is sparse by running:
+ * `fsutil sparse queryflag <path>`
+ * and
+ * `fsutil sparse queryrange <path>`
+ */
+async function ensureSparseOnWindows(path, platform) {
+    if (platform === 'win32') {
+        await tsHelpers.spawnAsync('fsutil', ['sparse', 'setflag', `"${path}"`], { stdio: 'pipe' });
+    }
+}
+/** Ensures the file is created and writes to it. */
+async function writeToSparseFile(logger, platform, path, data, from) {
+    if (!(await tsHelpers.fileExists(path))) {
+        await createSparseFile(logger, path, platform);
+    }
+    const fileHandle = await fsp__namespace.open(path, 'r+');
+    await fileHandle.write(data, 0, data.length, from);
+    await fileHandle.close();
+}
+
+exports.createSparseFile = createSparseFile;
+exports.writeToSparseFile = writeToSparseFile;
+//# sourceMappingURL=file.cjs.map

package/dist/drivers/download_blob/sparse_cache/file.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.cjs","sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from 'node:fs/promises';\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(\n  logger: MiLogger,\n  path: string,\n  platform: NodeJS.Platform,\n) {\n  try {\n    const ensureCreated = await fs.open(path, 'w');\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === 'win32') {\n    await spawnAsync('fsutil', ['sparse', 'setflag', `\"${path}\"`], { stdio: 'pipe' });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, 'r+');\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"names":["fs","spawnAsync","fileExists"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAGA;;AAE6F;AACtF,eAAe,gBAAgB,CACpC,MAAgB,EAChB,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI;QACF,MAAM,aAAa,GAAG,MAAMA,cAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC;AAC9C,QAAA,MAAM,aAAa,CAAC,KAAK,EAAE;AAE3B,QAAA,MAAM,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC7C;IAAE,OAAO,KAAc,EAAE;QACvB,MAAM,CAAC,KAAK,CAAC,CAAA,oBAAA,EAAuB,IAAI,CAAA,aAAA,EAAgB,QAAQ,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAC;IAC/E;AACF;AAEA;;;;;AAKG;AACH,eAAe,qBAAqB,CAAC,IAAY,EAAE,QAAyB,EAAA;AAC1E,IAAA,IAAI,QAAQ,KAAK,OAAO,EAAE;QACxB,MAAMC,oBAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAA,CAAA,EAAI,IAAI,CAAA,CAAA,CAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACnF;AACF;AAEA;AACO,eAAe,iBAAiB,CACrC,MAAgB,EAChB,QAAyB,EACzB,IAAY,EACZ,IAAgB,EAChB,IAAY,EAAA;IAEZ,IAAI,EAAE,MAAMC,oBAAU,CAAC,IAAI,CAAC,CAAC,EAAE;QAC7B,MAAM,gBAAgB,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC;IAChD;IAEA,MAAM,UAAU,GAAG,MAAMF,cAAE,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;AAC5C,IAAA,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAClD,IAAA,MAAM,UAAU,CAAC,KAAK,EAAE;AAC1B;;;;;"}

package/dist/drivers/download_blob/sparse_cache/file.d.ts

@@ -5,4 +5,3 @@ import { MiLogger } from '@milaboratories/ts-helpers';
 export declare function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform): Promise<void>;
 /** Ensures the file is created and writes to it. */
 export declare function writeToSparseFile(logger: MiLogger, platform: NodeJS.Platform, path: string, data: Uint8Array, from: number): Promise<void>;
-//# sourceMappingURL=file.d.ts.map

package/dist/drivers/download_blob/sparse_cache/file.js

@@ -0,0 +1,39 @@
+import { fileExists, spawnAsync } from '@milaboratories/ts-helpers';
+import * as fsp from 'node:fs/promises';
+
+/** Creates a sparse file for all systems
+ * Table of what supports sparse files:
+ * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */
+async function createSparseFile(logger, path, platform) {
+    try {
+        const ensureCreated = await fsp.open(path, 'w');
+        await ensureCreated.close();
+        await ensureSparseOnWindows(path, platform);
+    }
+    catch (error) {
+        logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);
+    }
+}
+/** Sets a sparse flag on Windows.
+ * We could check the file is sparse by running:
+ * `fsutil sparse queryflag <path>`
+ * and
+ * `fsutil sparse queryrange <path>`
+ */
+async function ensureSparseOnWindows(path, platform) {
+    if (platform === 'win32') {
+        await spawnAsync('fsutil', ['sparse', 'setflag', `"${path}"`], { stdio: 'pipe' });
+    }
+}
+/** Ensures the file is created and writes to it. */
+async function writeToSparseFile(logger, platform, path, data, from) {
+    if (!(await fileExists(path))) {
+        await createSparseFile(logger, path, platform);
+    }
+    const fileHandle = await fsp.open(path, 'r+');
+    await fileHandle.write(data, 0, data.length, from);
+    await fileHandle.close();
+}
+
+export { createSparseFile, writeToSparseFile };
+//# sourceMappingURL=file.js.map

package/dist/drivers/download_blob/sparse_cache/file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.js","sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from 'node:fs/promises';\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(\n  logger: MiLogger,\n  path: string,\n  platform: NodeJS.Platform,\n) {\n  try {\n    const ensureCreated = await fs.open(path, 'w');\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === 'win32') {\n    await spawnAsync('fsutil', ['sparse', 'setflag', `\"${path}\"`], { stdio: 'pipe' });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, 'r+');\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"names":["fs"],"mappings":";;;AAGA;;AAE6F;AACtF,eAAe,gBAAgB,CACpC,MAAgB,EAChB,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI;QACF,MAAM,aAAa,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC;AAC9C,QAAA,MAAM,aAAa,CAAC,KAAK,EAAE;AAE3B,QAAA,MAAM,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC7C;IAAE,OAAO,KAAc,EAAE;QACvB,MAAM,CAAC,KAAK,CAAC,CAAA,oBAAA,EAAuB,IAAI,CAAA,aAAA,EAAgB,QAAQ,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAC;IAC/E;AACF;AAEA;;;;;AAKG;AACH,eAAe,qBAAqB,CAAC,IAAY,EAAE,QAAyB,EAAA;AAC1E,IAAA,IAAI,QAAQ,KAAK,OAAO,EAAE;QACxB,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAA,CAAA,EAAI,IAAI,CAAA,CAAA,CAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACnF;AACF;AAEA;AACO,eAAe,iBAAiB,CACrC,MAAgB,EAChB,QAAyB,EACzB,IAAY,EACZ,IAAgB,EAChB,IAAY,EAAA;IAEZ,IAAI,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE;QAC7B,MAAM,gBAAgB,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC;IAChD;IAEA,MAAM,UAAU,GAAG,MAAMA,GAAE,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;AAC5C,IAAA,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAClD,IAAA,MAAM,UAAU,CAAC,KAAK,EAAE;AAC1B;;;;"}

package/dist/drivers/download_blob/sparse_cache/ranges.cjs

@@ -0,0 +1,104 @@
+'use strict';
+
+var zod = require('zod');
+var fsp = require('node:fs/promises');
+var plModelCommon = require('@milaboratories/pl-model-common');
+var tsHelpers = require('@milaboratories/ts-helpers');
+var cache = require('./cache.cjs');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var fsp__namespace = /*#__PURE__*/_interopNamespaceDefault(fsp);
+
+/** The content of the ranges file: ranges of bytes.
+ * The ranges should be normalized: sorted and no overlaps.
+ * For that, use `normalizeRanges` function. */
+const Ranges = zod.z.object({
+    ranges: zod.z.array(plModelCommon.RangeBytes),
+});
+const rangesFilePostfix = '.ranges.json';
+function rangesFileName(fPath) {
+    return fPath + rangesFilePostfix;
+}
+async function readRangesFile(logger, path) {
+    let ranges = { ranges: [] };
+    try {
+        const file = await fsp__namespace.readFile(path, 'utf8');
+        ranges = Ranges.parse(JSON.parse(file));
+    }
+    catch (e) {
+        if (e instanceof SyntaxError || e instanceof zod.z.ZodError) {
+            const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;
+            logger.error(msg);
+            throw new cache.CorruptedRangesError(msg);
+        }
+        if (!(e instanceof Error && 'code' in e && e.code === 'ENOENT')) {
+            throw e;
+        }
+        // If the file does not exist, assume the ranges are empty.
+    }
+    normalizeRanges(ranges);
+    return ranges;
+}
+/** Writes to a temporal file and then renames it atomically. */
+async function writeRangesFile(logger, path, ranges) {
+    await tsHelpers.createPathAtomically(logger, path, async (tempPath) => {
+        await fsp__namespace.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: 'wx' });
+    });
+}
+/** Sorts and merges overlapping ranges. */
+function normalizeRanges(s) {
+    s.ranges.sort((a, b) => a.from - b.from);
+    for (let i = 0; i < s.ranges.length - 1; i++) {
+        if (s.ranges[i].to >= s.ranges[i + 1].from) {
+            mergeRanges(s, i);
+            i--;
+        }
+    }
+}
+function mergeRanges(s, i) {
+    const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);
+    const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);
+    s.ranges.splice(i, 2, { from, to });
+}
+function rangesSize(s) {
+    return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);
+}
+function doesRangeExist(allRanges, range) {
+    for (const r of allRanges.ranges) {
+        if (r.from <= range.from && range.to <= r.to) {
+            return true;
+        }
+    }
+    return false;
+}
+function addRange(s, range) {
+    s.ranges.push(range);
+    normalizeRanges(s);
+    return s;
+}
+
+exports.addRange = addRange;
+exports.doesRangeExist = doesRangeExist;
+exports.normalizeRanges = normalizeRanges;
+exports.rangesFileName = rangesFileName;
+exports.rangesFilePostfix = rangesFilePostfix;
+exports.rangesSize = rangesSize;
+exports.readRangesFile = readRangesFile;
+exports.writeRangesFile = writeRangesFile;
+//# sourceMappingURL=ranges.cjs.map

package/dist/drivers/download_blob/sparse_cache/ranges.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranges.cjs","sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from 'node:fs/promises';\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = '.ranges.json';\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, 'utf8');\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && 'code' in e && e.code === 'ENOENT')) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: 'wx' });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n\n"],"names":["z","RangeBytes","fs","CorruptedRangesError","createPathAtomically"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAMA;;AAE+C;AAC/C,MAAM,MAAM,GAAGA,KAAC,CAAC,MAAM,CAAC;AACtB,IAAA,MAAM,EAAEA,KAAC,CAAC,KAAK,CAACC,wBAAU,CAAC;AAC5B,CAAA,CAAC;AAIK,MAAM,iBAAiB,GAAG;AAE3B,SAAU,cAAc,CAAC,KAAa,EAAA;IAC1C,OAAO,KAAK,GAAG,iBAAiB;AAClC;AAEO,eAAe,cAAc,CAAC,MAAgB,EAAE,IAAY,EAAA;AACjE,IAAA,IAAI,MAAM,GAAW,EAAE,MAAM,EAAE,EAAE,EAAE;AACnC,IAAA,IAAI;QACF,MAAM,IAAI,GAAG,MAAMC,cAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;AAC5C,QAAA,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC;IAAE,OAAO,CAAU,EAAE;QAEnB,IAAI,CAAC,YAAY,WAAW,IAAI,CAAC,YAAYF,KAAC,CAAC,QAAQ,EAAE;AACvD,YAAA,MAAM,GAAG,GAAG,CAAA,yBAAA,EAA4B,IAAI,CAAA,gBAAA,EAAmB,CAAC,EAAE;AAClE,YAAA,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AACjB,YAAA,MAAM,IAAIG,0BAAoB,CAAC,GAAG,CAAC;QACrC;AAEA,QAAA,IAAI,EAAE,CAAC,YAAY,KAAK,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,EAAE;AAC/D,YAAA,MAAM,CAAC;QACT;;IAGF;IAEA,eAAe,CAAC,MAAM,CAAC;AAEvB,IAAA,OAAO,MAAM;AACf;AAEA;AACO,eAAe,eAAe,CAAC,MAAgB,EAAE,IAAY,EAAE,MAAc,EAAA;IAClF,MAAMC,8BAAoB,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,QAAgB,KAAI;QAClE,MAAMF,cAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAC/E,IAAA,CAAC,CAAC;AACJ;AAEA;AACM,SAAU,eAAe,CAAC,CAAS,EAAA;IACvC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;AAExC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;QAC5C,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE;AAC1C,YAAA,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC;AACjB,YAAA,CAAC,EAAE;QACL;IACF;AACF;AAEA,SAAS,WAAW,CAAC,CAAS,EAAE,CAAS,EAAA;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;AAEvD,IAAA,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AACrC;AAEM,SAAU,UAAU,CAAC,CAAS,EAAA;IAClC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,KAAK,GAAG,GAAG,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE;AAEM,SAAU,cAAc,CAAC,SAAiB,EAAE,KAAiB,EAAA;AACjE,IAAA,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE;AAChC,QAAA,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE;AAC5C,YAAA,OAAO,IAAI;QACb;IACF;AAEA,IAAA,OAAO,KAAK;AACd;AAEM,SAAU,QAAQ,CAAC,CAAS,EAAE,KAAiB,EAAA;AACnD,IAAA,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;IACpB,eAAe,CAAC,CAAC,CAAC;AAElB,IAAA,OAAO,CAAC;AACV;;;;;;;;;;;"}

package/dist/drivers/download_blob/sparse_cache/ranges.d.ts

@@ -43,4 +43,3 @@ export declare function addRange(s: Ranges, range: RangeBytes): {
     }[];
 };
 export {};
-//# sourceMappingURL=ranges.d.ts.map

package/dist/drivers/download_blob/sparse_cache/ranges.js

@@ -0,0 +1,76 @@
+import { z } from 'zod';
+import * as fsp from 'node:fs/promises';
+import { RangeBytes } from '@milaboratories/pl-model-common';
+import { createPathAtomically } from '@milaboratories/ts-helpers';
+import { CorruptedRangesError } from './cache.js';
+
+/** The content of the ranges file: ranges of bytes.
+ * The ranges should be normalized: sorted and no overlaps.
+ * For that, use `normalizeRanges` function. */
+const Ranges = z.object({
+    ranges: z.array(RangeBytes),
+});
+const rangesFilePostfix = '.ranges.json';
+function rangesFileName(fPath) {
+    return fPath + rangesFilePostfix;
+}
+async function readRangesFile(logger, path) {
+    let ranges = { ranges: [] };
+    try {
+        const file = await fsp.readFile(path, 'utf8');
+        ranges = Ranges.parse(JSON.parse(file));
+    }
+    catch (e) {
+        if (e instanceof SyntaxError || e instanceof z.ZodError) {
+            const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;
+            logger.error(msg);
+            throw new CorruptedRangesError(msg);
+        }
+        if (!(e instanceof Error && 'code' in e && e.code === 'ENOENT')) {
+            throw e;
+        }
+        // If the file does not exist, assume the ranges are empty.
+    }
+    normalizeRanges(ranges);
+    return ranges;
+}
+/** Writes to a temporal file and then renames it atomically. */
+async function writeRangesFile(logger, path, ranges) {
+    await createPathAtomically(logger, path, async (tempPath) => {
+        await fsp.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: 'wx' });
+    });
+}
+/** Sorts and merges overlapping ranges. */
+function normalizeRanges(s) {
+    s.ranges.sort((a, b) => a.from - b.from);
+    for (let i = 0; i < s.ranges.length - 1; i++) {
+        if (s.ranges[i].to >= s.ranges[i + 1].from) {
+            mergeRanges(s, i);
+            i--;
+        }
+    }
+}
+function mergeRanges(s, i) {
+    const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);
+    const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);
+    s.ranges.splice(i, 2, { from, to });
+}
+function rangesSize(s) {
+    return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);
+}
+function doesRangeExist(allRanges, range) {
+    for (const r of allRanges.ranges) {
+        if (r.from <= range.from && range.to <= r.to) {
+            return true;
+        }
+    }
+    return false;
+}
+function addRange(s, range) {
+    s.ranges.push(range);
+    normalizeRanges(s);
+    return s;
+}
+
+export { addRange, doesRangeExist, normalizeRanges, rangesFileName, rangesFilePostfix, rangesSize, readRangesFile, writeRangesFile };
+//# sourceMappingURL=ranges.js.map

package/dist/drivers/download_blob/sparse_cache/ranges.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranges.js","sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from 'node:fs/promises';\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = '.ranges.json';\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, 'utf8');\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && 'code' in e && e.code === 'ENOENT')) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: 'wx' });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n\n"],"names":["fs"],"mappings":";;;;;;AAMA;;AAE+C;AAC/C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;AACtB,IAAA,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;AAC5B,CAAA,CAAC;AAIK,MAAM,iBAAiB,GAAG;AAE3B,SAAU,cAAc,CAAC,KAAa,EAAA;IAC1C,OAAO,KAAK,GAAG,iBAAiB;AAClC;AAEO,eAAe,cAAc,CAAC,MAAgB,EAAE,IAAY,EAAA;AACjE,IAAA,IAAI,MAAM,GAAW,EAAE,MAAM,EAAE,EAAE,EAAE;AACnC,IAAA,IAAI;QACF,MAAM,IAAI,GAAG,MAAMA,GAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;AAC5C,QAAA,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC;IAAE,OAAO,CAAU,EAAE;QAEnB,IAAI,CAAC,YAAY,WAAW,IAAI,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE;AACvD,YAAA,MAAM,GAAG,GAAG,CAAA,yBAAA,EAA4B,IAAI,CAAA,gBAAA,EAAmB,CAAC,EAAE;AAClE,YAAA,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;AACjB,YAAA,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC;QACrC;AAEA,QAAA,IAAI,EAAE,CAAC,YAAY,KAAK,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,EAAE;AAC/D,YAAA,MAAM,CAAC;QACT;;IAGF;IAEA,eAAe,CAAC,MAAM,CAAC;AAEvB,IAAA,OAAO,MAAM;AACf;AAEA;AACO,eAAe,eAAe,CAAC,MAAgB,EAAE,IAAY,EAAE,MAAc,EAAA;IAClF,MAAM,oBAAoB,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,QAAgB,KAAI;QAClE,MAAMA,GAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAC/E,IAAA,CAAC,CAAC;AACJ;AAEA;AACM,SAAU,eAAe,CAAC,CAAS,EAAA;IACvC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;AAExC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;QAC5C,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE;AAC1C,YAAA,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC;AACjB,YAAA,CAAC,EAAE;QACL;IACF;AACF;AAEA,SAAS,WAAW,CAAC,CAAS,EAAE,CAAS,EAAA;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;AAEvD,IAAA,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AACrC;AAEM,SAAU,UAAU,CAAC,CAAS,EAAA;IAClC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,KAAK,GAAG,GAAG,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE;AAEM,SAAU,cAAc,CAAC,SAAiB,EAAE,KAAiB,EAAA;AACjE,IAAA,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE;AAChC,QAAA,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE;AAC5C,YAAA,OAAO,IAAI;QACb;IACF;AAEA,IAAA,OAAO,KAAK;AACd;AAEM,SAAU,QAAQ,CAAC,CAAS,EAAE,KAAiB,EAAA;AACnD,IAAA,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;IACpB,eAAe,CAAC,CAAC,CAAC;AAElB,IAAA,OAAO,CAAC;AACV;;;;"}

package/dist/drivers/download_blob_url/driver.cjs

@@ -0,0 +1,169 @@
+'use strict';
+
+var computable = require('@milaboratories/computable');
+var tsHelpers = require('@milaboratories/ts-helpers');
+var node_crypto = require('node:crypto');
+var path = require('node:path');
+var files_cache = require('../helpers/files_cache.cjs');
+var plClient = require('@milaboratories/pl-client');
+var plModelCommon = require('@milaboratories/pl-model-common');
+var snapshot = require('./snapshot.cjs');
+var plTree = require('@milaboratories/pl-tree');
+var task = require('./task.cjs');
+var url = require('../urls/url.cjs');
+var driver_id = require('./driver_id.cjs');
+var download_blob_task = require('../download_blob/download_blob_task.cjs');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
+
+/** Downloads .tar, .tar.gz or zip archives,
+ * extracts them into saveDir and gets a url for it. */
+class DownloadBlobToURLDriver {
+    logger;
+    signer;
+    clientDownload;
+    saveDir;
+    opts;
+    idToDownload = new Map();
+    downloadQueue;
+    /** Writes and removes files to a hard drive and holds a counter for every
+     * file that should be kept. */
+    cache;
+    constructor(logger, signer, clientDownload, saveDir, opts = {
+        cacheSoftSizeBytes: 50 * 1024 * 1024,
+        nConcurrentDownloads: 50,
+    }) {
+        this.logger = logger;
+        this.signer = signer;
+        this.clientDownload = clientDownload;
+        this.saveDir = saveDir;
+        this.opts = opts;
+        this.downloadQueue = new tsHelpers.TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {
+            type: 'exponentialWithMaxDelayBackoff',
+            initialDelay: 10000,
+            maxDelay: 30000,
+            backoffMultiplier: 1.5,
+            jitter: 0.5,
+        });
+        this.cache = new files_cache.FilesCache(this.opts.cacheSoftSizeBytes);
+    }
+    info() {
+        return {
+            saveDir: this.saveDir,
+            opts: this.opts,
+            idToDownloadSize: this.idToDownload.size,
+            idToDownloadKeys: this.idToDownload.keys(),
+            idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),
+        };
+    }
+    /**
+     * @returns full path to the referenced file
+     */
+    getPathForCustomProtocol(url$1) {
+        if (plModelCommon.isFolderURL(url$1)) {
+            return url.getPathForFolderURL(this.signer, url$1, this.saveDir);
+        }
+        throw new Error(`getPathForCustomProtocol: ${url$1} is invalid`);
+    }
+    extractArchiveAndGetURL(res, format, ctx) {
+        // wrap result as computable, if we were not given an existing computable context
+        if (ctx === undefined)
+            return computable.Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));
+        const rInfo = plTree.isPlTreeEntry(res)
+            ? snapshot.makeDownloadableBlobSnapshot(res, ctx)
+            : res;
+        const callerId = node_crypto.randomUUID();
+        ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));
+        const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);
+        if (result?.url === undefined)
+            ctx.markUnstable(`a path to the downloaded archive might be undefined. The current result: ${result}`);
+        if (result?.error !== undefined)
+            throw result?.error;
+        return result?.url;
+    }
+    extractArchiveAndGetURLNoCtx(rInfo, format, w, callerId) {
+        const task = this.idToDownload.get(driver_id.newId(rInfo.id, format));
+        if (task != undefined) {
+            task.attach(w, callerId);
+            return task.getURL();
+        }
+        const newTask = this.setNewTask(w, rInfo, format, callerId);
+        this.downloadQueue.push({
+            fn: async () => this.downloadUrl(newTask, callerId),
+            recoverableErrorPredicate: (e) => !download_blob_task.nonRecoverableError(e),
+        });
+        return newTask.getURL();
+    }
+    /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
+    async downloadUrl(task, callerId) {
+        await task.download();
+        // Might be undefined if a error happened
+        if (task.getURL()?.url != undefined)
+            this.cache.addCache(task, callerId);
+    }
+    /** Removes a directory and aborts a downloading task when all callers
+     * are not interested in it. */
+    async releasePath(id, format, callerId) {
+        const task$1 = this.idToDownload.get(driver_id.newId(id, format));
+        if (task$1 == undefined)
+            return;
+        if (this.cache.existsFile(task$1.path)) {
+            const toDelete = this.cache.removeFile(task$1.path, callerId);
+            await Promise.all(toDelete.map(async (task$1) => {
+                await task.rmRFDir(task$1.path);
+                this.cache.removeCache(task$1);
+                this.removeTask(task$1, `the task ${plClient.stringifyWithResourceId(task$1.info())} was removed`
+                    + `from cache along with ${plClient.stringifyWithResourceId(toDelete.map((t) => t.info()))}`);
+            }));
+        }
+        else {
+            // The task is still in a downloading queue.
+            const deleted = task$1.counter.dec(callerId);
+            if (deleted)
+                this.removeTask(task$1, `the task ${plClient.stringifyWithResourceId(task$1.info())} was removed from cache`);
+        }
+    }
+    /** Removes all files from a hard drive. */
+    async releaseAll() {
+        this.downloadQueue.stop();
+        await Promise.all(Array.from(this.idToDownload.entries()).map(async ([_, task$1]) => {
+            await task.rmRFDir(task$1.path);
+            this.cache.removeCache(task$1);
+            this.removeTask(task$1, `the task ${plClient.stringifyWithResourceId(task$1.info())} was released when the driver was closed`);
+        }));
+    }
+    setNewTask(w, rInfo, format, callerId) {
+        const result = new task.DownloadAndUnarchiveTask(this.logger, this.signer, this.saveDir, this.getFilePath(rInfo.id, format), rInfo, format, this.clientDownload);
+        result.attach(w, callerId);
+        this.idToDownload.set(driver_id.newId(rInfo.id, format), result);
+        return result;
+    }
+    removeTask(task, reason) {
+        task.abort(reason);
+        task.change.markChanged(`task for ${plClient.resourceIdToString(task.rInfo.id)} removed: ${reason}`);
+        this.idToDownload.delete(driver_id.newId(task.rInfo.id, task.format));
+    }
+    getFilePath(id, format) {
+        return path__namespace.join(this.saveDir, `${String(BigInt(id))}_${format}`);
+    }
+}
+
+exports.DownloadBlobToURLDriver = DownloadBlobToURLDriver;
+//# sourceMappingURL=driver.cjs.map

package/dist/drivers/download_blob_url/driver.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.cjs","sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from '@milaboratories/computable';\nimport { Computable } from '@milaboratories/computable';\nimport type {\n  MiLogger,\n  Signer } from '@milaboratories/ts-helpers';\nimport {\n  TaskProcessor,\n} from '@milaboratories/ts-helpers';\nimport { randomUUID } from 'node:crypto';\nimport * as path from 'node:path';\nimport { FilesCache } from '../helpers/files_cache';\nimport type { ResourceId } from '@milaboratories/pl-client';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport { type ArchiveFormat, type BlobToURLDriver, type FolderURL, isFolderURL } from '@milaboratories/pl-model-common';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { makeDownloadableBlobSnapshot } from './snapshot';\nimport type { PlTreeEntry } from '@milaboratories/pl-tree';\nimport { isPlTreeEntry } from '@milaboratories/pl-tree';\nimport { DownloadAndUnarchiveTask, rmRFDir } from './task';\nimport type { ClientDownload } from '../../clients/download';\nimport { getPathForFolderURL } from '../urls/url';\nimport type { Id } from './driver_id';\nimport { newId } from './driver_id';\nimport { nonRecoverableError } from '../download_blob/download_blob_task';\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: 'exponentialWithMaxDelayBackoff',\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined)\n      throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed`\n            + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(w: Watcher, rInfo: DownloadableBlobSnapshot, format: ArchiveFormat, callerId: string) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n    return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n  }\n}\n"],"names":["TaskProcessor","FilesCache","url","isFolderURL","getPathForFolderURL","Computable","isPlTreeEntry","makeDownloadableBlobSnapshot","randomUUID","newId","nonRecoverableError","task","rmRFDir","stringifyWithResourceId","DownloadAndUnarchiveTask","resourceIdToString","path"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA;AACuD;MAC1C,uBAAuB,CAAA;AASf,IAAA,MAAA;AACA,IAAA,MAAA;AACA,IAAA,cAAA;AACA,IAAA,OAAA;AACA,IAAA,IAAA;AAZX,IAAA,YAAY,GAAsC,IAAI,GAAG,EAAE;AAC3D,IAAA,aAAa;AAErB;AAC+B;AACvB,IAAA,KAAK;IAEb,WAAA,CACmB,MAAgB,EAChB,MAAc,EACd,cAA8B,EAC9B,OAAe,EACf,IAAA,GAAmC;AAClD,QAAA,kBAAkB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;AACpC,QAAA,oBAAoB,EAAE,EAAE;AACzB,KAAA,EAAA;QAPgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;AAKrB,QAAA,IAAI,CAAC,aAAa,GAAG,IAAIA,uBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE;AAClF,YAAA,IAAI,EAAE,gCAAgC;AACtC,YAAA,YAAY,EAAE,KAAK;AACnB,YAAA,QAAQ,EAAE,KAAK;AACf,YAAA,iBAAiB,EAAE,GAAG;AACtB,YAAA,MAAM,EAAE,GAAG;AACZ,SAAA,CAAC;AACF,QAAA,IAAI,CAAC,KAAK,GAAG,IAAIC,sBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC;IAC3D;IAEO,IAAI,GAAA;QACT,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,gBAAgB,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;AACxC,YAAA,gBAAgB,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE;AAC1C,YAAA,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC7F;IACH;AAEA;;AAEG;AACH,IAAA,wBAAwB,CAACC,KAAc,EAAA;AACrC,QAAA,IAAIC,yBAAW,CAACD,KAAG,CAAC,EAAE;AACpB,YAAA,OAAOE,uBAAmB,CAAC,IAAI,CAAC,MAAM,EAAEF,KAAG,EAAE,IAAI,CAAC,OAAO,CAAC;QAC5D;AAEA,QAAA,MAAM,IAAI,KAAK,CAAC,6BAA6BA,KAAG,CAAA,WAAA,CAAa,CAAC;IAChE;AAaA,IAAA,uBAAuB,CACrB,GAA2C,EAC3C,MAAqB,EACrB,GAAmB,EAAA;;QAGnB,IAAI,GAAG,KAAK,SAAS;YACnB,OAAOG,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;AAE7E,QAAA,MAAM,KAAK,GAA6BC,oBAAa,CAAC,GAAG;AACvD,cAAEC,qCAA4B,CAAC,GAAG,EAAE,GAAG;cACrC,GAAG;AAEP,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAE7B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAEpE,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;AACtF,QAAA,IAAI,MAAM,EAAE,GAAG,KAAK,SAAS;AAC3B,YAAA,GAAG,CAAC,YAAY,CACd,4EAA4E,MAAM,CAAA,CAAE,CACrF;AAEH,QAAA,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS;YAC7B,MAAM,MAAM,EAAE,KAAK;QAErB,OAAO,MAAM,EAAE,GAAG;IACpB;AAEQ,IAAA,4BAA4B,CAClC,KAA+B,EAC/B,MAAqB,EACrB,CAAU,EACV,QAAgB,EAAA;AAEhB,QAAA,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAACC,eAAK,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AAE3D,QAAA,IAAI,IAAI,IAAI,SAAS,EAAE;AACrB,YAAA,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AACxB,YAAA,OAAO,IAAI,CAAC,MAAM,EAAE;QACtB;AAEA,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACtB,YAAA,EAAE,EAAE,YAAY,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;YACnD,yBAAyB,EAAE,CAAC,CAAC,KAAK,CAACC,sCAAmB,CAAC,CAAC,CAAC;AAC1D,SAAA,CAAC;AAEF,QAAA,OAAO,OAAO,CAAC,MAAM,EAAE;IACzB;;AAGA,IAAA,MAAM,WAAW,CAAC,IAA8B,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,IAAI,CAAC,QAAQ,EAAE;;AAErB,QAAA,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,IAAI,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAC1E;AAEA;AAC+B;AAC/B,IAAA,MAAM,WAAW,CAAC,EAAc,EAAE,MAAqB,EAAE,QAAgB,EAAA;AACvE,QAAA,MAAMC,MAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAACF,eAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACrD,IAAIE,MAAI,IAAI,SAAS;YAAE;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,CAAC,EAAE;AACpC,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAACA,MAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;AAE3D,YAAA,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,OAAOA,MAA8B,KAAI;AACpD,gBAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,YAAA;AAC9C,sBAAA,CAAA,sBAAA,EAAyBE,gCAAuB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAE,CACpF;YACH,CAAC,CAAC,CACH;QACH;aAAO;;YAEL,MAAM,OAAO,GAAGF,MAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AAC1C,YAAA,IAAI,OAAO;AACT,gBAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,uBAAA,CAAyB,CAC1E;QACL;IACF;;AAGA,IAAA,MAAM,UAAU,GAAA;AACd,QAAA,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAEzB,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAEA,MAAI,CAAC,KAAI;AAC9D,YAAA,MAAMC,YAAO,CAACD,MAAI,CAAC,IAAI,CAAC;AACxB,YAAA,IAAI,CAAC,KAAK,CAAC,WAAW,CAACA,MAAI,CAAC;AAE5B,YAAA,IAAI,CAAC,UAAU,CACbA,MAAI,EACJ,CAAA,SAAA,EAAYE,gCAAuB,CAACF,MAAI,CAAC,IAAI,EAAE,CAAC,CAAA,wCAAA,CAA0C,CAC3F;QACH,CAAC,CAAC,CACH;IACH;AAEQ,IAAA,UAAU,CAAC,CAAU,EAAE,KAA+B,EAAE,MAAqB,EAAE,QAAgB,EAAA;AACrG,QAAA,MAAM,MAAM,GAAG,IAAIG,6BAAwB,CACzC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAClC,KAAK,EACL,MAAM,EACN,IAAI,CAAC,cAAc,CACpB;AACD,QAAA,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC1B,QAAA,IAAI,CAAC,YAAY,CAAC,GAAG,CAACL,eAAK,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC;AAEtD,QAAA,OAAO,MAAM;IACf;IAEQ,UAAU,CAAC,IAA8B,EAAE,MAAc,EAAA;AAC/D,QAAA,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,SAAA,EAAYM,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,aAAa,MAAM,CAAA,CAAE,CAAC;AAC3F,QAAA,IAAI,CAAC,YAAY,CAAC,MAAM,CAACN,eAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7D;IAEQ,WAAW,CAAC,EAAc,EAAE,MAAqB,EAAA;QACvD,OAAOO,eAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA,EAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAA,CAAE,CAAC;IACnE;AACD;;;;"}

package/dist/drivers/download_blob_url/driver.d.ts

@@ -43,4 +43,3 @@ export declare class DownloadBlobToURLDriver implements BlobToURLDriver {
     private removeTask;
     private getFilePath;
 }
-//# sourceMappingURL=driver.d.ts.map