@milaboratories/pl-drivers 1.16.1 → 1.16.2

package/dist/drivers/download_blob/sparse_cache/cache.cjs

@@ -5,7 +5,7 @@ let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_helpers = require("@milaboratories/helpers");
 let node_fs = require("node:fs");
 let node_path = require("node:path");
-node_path = require_runtime.__toESM(node_path);
+node_path = require_runtime.__toESM(node_path, 1);
 //#region src/drivers/download_blob/sparse_cache/cache.ts
 /** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */
 var CorruptedRangesError = class extends Error {
@@ -13,6 +13,8 @@ var CorruptedRangesError = class extends Error {
 };
 /** Stores ranges in a directory as JSON files (the default implementation). */
 var SparseCacheFsRanges = class {
+	logger;
+	cacheDir;
 	constructor(logger, cacheDir) {
 		this.logger = logger;
 		this.cacheDir = cacheDir;
@@ -32,6 +34,8 @@ var SparseCacheFsRanges = class {
 };
 /** Stores sparse files in a directory (the default implementation). */
 var SparseCacheFsFile = class {
+	logger;
+	cacheDir;
 	suffix = ".sparse.bin";
 	constructor(logger, cacheDir) {
 		this.logger = logger;
@@ -57,6 +61,10 @@ var SparseCacheFsFile = class {
 };
 /** LRU cache for ranges of sparse files. */
 var SparseCache = class {
+	logger;
+	maxSize;
+	ranges;
+	storage;
 	/** Fields are public for tests. */
 	/** The lock to make sure cache requests are done one by one. */
 	lock = new _milaboratories_helpers.functions.AwaitLock();

package/dist/drivers/download_blob/sparse_cache/cache.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"cache.cjs","names":["path","rangesFileName","readRangesFile","writeRangesFile","fs","writeToSparseFile","functions","rangesSize","doesRangeExist","addRange"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;;;AAWT,IAAa,sBAAb,MAA8D;CAC5D,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAc,KAAqB;AACjC,SAAOA,UAAAA,QAAK,KAAK,KAAK,UAAUC,eAAAA,eAAe,IAAI,CAAC;;CAGtD,MAAM,IAAI,KAA8B;AACtC,SAAO,MAAMC,eAAAA,eAAe,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;;CAG3D,MAAM,IAAI,KAAa,QAAgB;AACrC,SAAO,MAAMC,eAAAA,gBAAgB,KAAK,QAAQ,KAAK,MAAM,IAAI,EAAE,OAAO;;CAGpE,MAAM,OAAO,KAAa;AACxB,QAAMC,QAAAA,SAAG,GAAG,KAAK,MAAM,IAAI,CAAC;;;;AAchC,IAAa,oBAAb,MAA4D;CAC1D,SAA0B;CAE1B,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAM,MAAyB;AAC7B,SAAA,GAAA,2BAAA,iBAAsB,KAAK,SAAS;AAEpC,UADc,MAAMA,QAAAA,SAAG,QAAQ,KAAK,SAAS,EAChC,QAAQ,MAAM,EAAE,SAAS,KAAK,OAAO,CAAC;;CAGrD,MAAM,OAAO,KAA+B;AAC1C,SAAO,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK,IAAI,CAAC;;CAGzC,KAAK,KAAqB;AACxB,SAAOJ,UAAAA,QAAK,KAAK,KAAK,UAAU,MAAM,KAAK,OAAO;;CAGpD,MAAM,MAAM,KAAa,MAAkB,MAA6B;AACtE,SAAA,GAAA,2BAAA,iBAAsB,KAAK,SAAS;AACpC,QAAMK,aAAAA,kBAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,IAAI,EAAE,MAAM,KAAK;;CAGpF,MAAM,OAAO,KAA4B;AACvC,QAAMD,QAAAA,SAAG,GAAG,KAAK,KAAK,IAAI,CAAC;;;;AAK/B,IAAa,cAAb,MAAoD;;;CAIlD,OAAe,IAAIE,wBAAAA,UAAU,WAAW;CAExC,sCAA6B,IAAI,KAAmB;CACpD,OAAc;CAEd,YACE,QAEA,SACA,QACA,SACA;AALgB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;;;CAKlB,MAAM,QAAQ;AACZ,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,aAAa;IACxB;;;;CAKJ,MAAM,IAAI,KAAa,OAAgD;AACrE,SAAO,MAAM,SAAS,KAAK,MAAM,YAAY;AAC3C,UAAO,MAAM,KAAK,UAAU,KAAK,MAAM;IACvC;;;;CAKJ,MAAM,IAAI,KAAa,OAAmB,MAAiC;AACzE,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,UAAU,KAAK,OAAO,KAAK;IACtC;;CAGJ,MAAc,cAAc;AAC1B,OAAK,OAAO;AACZ,OAAK,sCAAsB,IAAI,KAAmB;EAElD,MAAM,sBAAM,IAAI,MAAM;AAItB,OAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,EAAE;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,IAAI;AACzC,QAAK,QAAQC,eAAAA,WAAW,OAAO;AAC/B,QAAK,oBAAoB,IAAI,KAAK,IAAI;;;CAI1C,MAAc,UAAU,KAAa,OAAgD;AAKnF,MAAI,MAAM,KAAK,QAAQ,OAAO,IAAI,EAAE;AAClC,QAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;AAG7C,OAAIC,eAAAA,eADW,MAAM,KAAK,UAAU,IAAI,EACb,MAAM,CAC/B,QAAO,KAAK,QAAQ,KAAK,IAAI;AAG/B;;;CAMJ,MAAc,UAAU,KAAa,OAAqC,MAAkB;AAC1F,QAAM,KAAK,mBAAmB,KAAK,OAAO,KAAK;AAC/C,QAAM,KAAK,eAAe;;;CAI5B,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;AACxF,MAAI,MAAM,KAAK,MAAM,SAAS,KAAK,OACjC,OAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,MAAM,CAAC,UAAU,KAAK,SAClD;AAGH,OAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;EAE7C,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,OAAK,QAAQD,eAAAA,WAAW,OAAO;AAE/B,QAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,KAAK;EAE/C,MAAM,YAAYE,eAAAA,SAAS,QAAQ,MAAM;AACzC,OAAK,QAAQF,eAAAA,WAAW,UAAU;AAElC,QAAM,KAAK,OAAO,IAAI,KAAK,UAAU;;;CAIvC,MAAM,gBAA+B;EACnC,MAAM,UAAA,GAAA,2BAAA,YAAoB,KAAK,oBAAoB;AACnD,SAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;AAE3E,SAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,KAAK;AAC/B,OAAI,CAAC,WACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,QAAK,QAAQA,eAAAA,WAAW,OAAO;AAC/B,QAAK,MAAM,IAAI;;;;CAKnB,MAAc,UAAU,KAAa;AACnC,MAAI;AACF,UAAO,MAAM,KAAK,OAAO,IAAI,IAAI;WAC1B,GAAY;AACnB,OAAI,aAAa,sBAAsB;AAGrC,UAAM,KAAK,MAAM,IAAI;AACrB,UAAM,KAAK,aAAa;AAExB,WAAO,MAAM,KAAK,OAAO,IAAI,IAAI;;AAGnC,SAAM;;;;CAKV,MAAc,MAAM,KAAa;AAC/B,QAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,QAAM,KAAK,OAAO,OAAO,IAAI;AAC7B,OAAK,oBAAoB,OAAO,IAAI;;CAGtC,MAAM,UAAyB;AAC7B,QAAM,KAAK,KAAK,cAAc;;CAGhC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,eAAe,SAAY,MAA2B,IAAkC;AACtF,KAAI;AACF,QAAM,KAAK,cAAc;AACzB,SAAO,MAAM,IAAI;WACT;AACR,OAAK,SAAS"}
+{"version":3,"file":"cache.cjs","names":["path","rangesFileName","readRangesFile","writeRangesFile","fs","writeToSparseFile","functions","rangesSize","doesRangeExist","addRange"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;AACT;;AAUA,IAAa,sBAAb,MAA8D;CAEzC;CACA;CAFnB,YACE,QACA,UACA;EAFiB,KAAA,SAAA;EACA,KAAA,WAAA;CAChB;CAEH,MAAc,KAAqB;EACjC,OAAOA,UAAAA,QAAK,KAAK,KAAK,UAAUC,eAAAA,eAAe,GAAG,CAAC;CACrD;CAEA,MAAM,IAAI,KAA8B;EACtC,OAAO,MAAMC,eAAAA,eAAe,KAAK,QAAQ,KAAK,MAAM,GAAG,CAAC;CAC1D;CAEA,MAAM,IAAI,KAAa,QAAgB;EACrC,OAAO,MAAMC,eAAAA,gBAAgB,KAAK,QAAQ,KAAK,MAAM,GAAG,GAAG,MAAM;CACnE;CAEA,MAAM,OAAO,KAAa;EACxB,MAAMC,QAAAA,SAAG,GAAG,KAAK,MAAM,GAAG,CAAC;CAC7B;AACF;;AAYA,IAAa,oBAAb,MAA4D;CAIvC;CACA;CAJnB,SAA0B;CAE1B,YACE,QACA,UACA;EAFiB,KAAA,SAAA;EACA,KAAA,WAAA;CAChB;CAEH,MAAM,MAAyB;EAC7B,OAAA,GAAA,2BAAA,gBAAA,CAAsB,KAAK,QAAQ;EAEnC,QAAO,MADaA,QAAAA,SAAG,QAAQ,KAAK,QAAQ,EAAA,CAC/B,QAAQ,MAAM,EAAE,SAAS,KAAK,MAAM,CAAC;CACpD;CAEA,MAAM,OAAO,KAA+B;EAC1C,OAAO,OAAA,GAAA,2BAAA,WAAA,CAAiB,KAAK,KAAK,GAAG,CAAC;CACxC;CAEA,KAAK,KAAqB;EACxB,OAAOJ,UAAAA,QAAK,KAAK,KAAK,UAAU,MAAM,KAAK,MAAM;CACnD;CAEA,MAAM,MAAM,KAAa,MAAkB,MAA6B;EACtE,OAAA,GAAA,2BAAA,gBAAA,CAAsB,KAAK,QAAQ;EACnC,MAAMK,aAAAA,kBAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,GAAG,GAAG,MAAM,IAAI;CACnF;CAEA,MAAM,OAAO,KAA4B;EACvC,MAAMD,QAAAA,SAAG,GAAG,KAAK,KAAK,GAAG,CAAC;CAC5B;AACF;;AAGA,IAAa,cAAb,MAAoD;CAUhC;CAEA;CACA;CACA;;;CAVlB,OAAe,IAAIE,wBAAAA,UAAU,UAAU;CAEvC,sCAA6B,IAAI,IAAkB;CACnD,OAAc;CAEd,YACE,QAEA,SACA,QACA,SACA;EALgB,KAAA,SAAA;EAEA,KAAA,UAAA;EACA,KAAA,SAAA;EACA,KAAA,UAAA;CACf;;;CAIH,MAAM,QAAQ;EACZ,MAAM,SAAS,KAAK,MAAM,YAAY;GACpC,MAAM,KAAK,YAAY;EACzB,CAAC;CACH;;;CAIA,MAAM,IAAI,KAAa,OAAgD;EACrE,OAAO,MAAM,SAAS,KAAK,MAAM,YAAY;GAC3C,OAAO,MAAM,KAAK,UAAU,KAAK,KAAK;EACxC,CAAC;CACH;;;CAIA,MAAM,IAAI,KAAa,OAAmB,MAAiC;EACzE,MAAM,SAAS,KAAK,MAAM,YAAY;GACpC,MAAM,KAAK,UAAU,KAAK,OAAO,IAAI;EACvC,CAAC;CACH;CAEA,MAAc,cAAc;EAC1B,KAAK,OAAO;EACZ,KAAK,sCAAsB,IAAI,IAAkB;EAEjD,MAAM,sBAAM,IAAI,KAAK;EAIrB,KAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,IAAI,GAAG;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,GAAG;GACxC,KAAK,QAAQC,eAAAA,WAAW,MAAM;GAC9B,KAAK,oBAAoB,IAAI,KAAK,GAAG;EACvC;CACF;CAEA,MAAc,UAAU,KAAa,OAAgD;EAKnF,IAAI,MAAM,KAAK,QAAQ,OAAO,GAAG,GAAG;GAClC,KAAK,oBAAoB,IAAI,qBAAK,IAAI,KAAK,CAAC;GAG5C,IAAIC,eAAAA,eAAe,MADE,KAAK,UAAU,GAAG,GACZ,KAAK,GAC9B,OAAO,KAAK,QAAQ,KAAK,GAAG;GAG9B;EACF;CAGF;CAEA,MAAc,UAAU,KAAa,OAAqC,MAAkB;EAC1F,MAAM,KAAK,mBAAmB,KAAK,OAAO,IAAI;EAC9C,MAAM,KAAK,cAAc;CAC3B;;CAGA,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;EACxF,IAAI,MAAM,KAAK,MAAM,SAAS,KAAK,QACjC,MAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,KAAK,EAAE,UAAU,KAAK,QACnD;EAGF,KAAK,oBAAoB,IAAI,qBAAK,IAAI,KAAK,CAAC;EAE5C,MAAM,SAAS,MAAM,KAAK,UAAU,GAAG;EACvC,KAAK,QAAQD,eAAAA,WAAW,MAAM;EAE9B,MAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,IAAI;EAE9C,MAAM,YAAYE,eAAAA,SAAS,QAAQ,KAAK;EACxC,KAAK,QAAQF,eAAAA,WAAW,SAAS;EAEjC,MAAM,KAAK,OAAO,IAAI,KAAK,SAAS;CACtC;;CAGA,MAAM,gBAA+B;EACnC,MAAM,UAAA,GAAA,2BAAA,WAAA,CAAoB,KAAK,mBAAmB;EAClD,OAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,QAAQ,IAAI,MAAM,QAAQ,CAAC;EAE1E,OAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,IAAI;GAC9B,IAAI,CAAC,YACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,GAAG;GACvC,KAAK,QAAQA,eAAAA,WAAW,MAAM;GAC9B,KAAK,MAAM,GAAG;EAChB;CACF;;CAGA,MAAc,UAAU,KAAa;EACnC,IAAI;GACF,OAAO,MAAM,KAAK,OAAO,IAAI,GAAG;EAClC,SAAS,GAAY;GACnB,IAAI,aAAa,sBAAsB;IAGrC,MAAM,KAAK,MAAM,GAAG;IACpB,MAAM,KAAK,YAAY;IAEvB,OAAO,MAAM,KAAK,OAAO,IAAI,GAAG;GAClC;GAEA,MAAM;EACR;CACF;;CAGA,MAAc,MAAM,KAAa;EAC/B,MAAM,KAAK,QAAQ,OAAO,GAAG;EAC7B,MAAM,KAAK,OAAO,OAAO,GAAG;EAC5B,KAAK,oBAAoB,OAAO,GAAG;CACrC;CAEA,MAAM,UAAyB;EAC7B,MAAM,KAAK,KAAK,aAAa;CAC/B;CAEA,OAAO,OAAO,gBAA+B;EAC3C,MAAM,KAAK,QAAQ;CACrB;AACF;;AAGA,eAAe,SAAY,MAA2B,IAAkC;CACtF,IAAI;EACF,MAAM,KAAK,aAAa;EACxB,OAAO,MAAM,GAAG;CAClB,UAAU;EACR,KAAK,QAAQ;CACf;AACF"}

package/dist/drivers/download_blob/sparse_cache/cache.js

@@ -11,6 +11,8 @@ var CorruptedRangesError = class extends Error {
 };
 /** Stores ranges in a directory as JSON files (the default implementation). */
 var SparseCacheFsRanges = class {
+	logger;
+	cacheDir;
 	constructor(logger, cacheDir) {
 		this.logger = logger;
 		this.cacheDir = cacheDir;
@@ -30,6 +32,8 @@ var SparseCacheFsRanges = class {
 };
 /** Stores sparse files in a directory (the default implementation). */
 var SparseCacheFsFile = class {
+	logger;
+	cacheDir;
 	suffix = ".sparse.bin";
 	constructor(logger, cacheDir) {
 		this.logger = logger;
@@ -55,6 +59,10 @@ var SparseCacheFsFile = class {
 };
 /** LRU cache for ranges of sparse files. */
 var SparseCache = class {
+	logger;
+	maxSize;
+	ranges;
+	storage;
 	/** Fields are public for tests. */
 	/** The lock to make sure cache requests are done one by one. */
 	lock = new functions.AwaitLock();

package/dist/drivers/download_blob/sparse_cache/cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache.js","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;;;AAWT,IAAa,sBAAb,MAA8D;CAC5D,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAc,KAAqB;AACjC,SAAO,KAAK,KAAK,KAAK,UAAU,eAAe,IAAI,CAAC;;CAGtD,MAAM,IAAI,KAA8B;AACtC,SAAO,MAAM,eAAe,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;;CAG3D,MAAM,IAAI,KAAa,QAAgB;AACrC,SAAO,MAAM,gBAAgB,KAAK,QAAQ,KAAK,MAAM,IAAI,EAAE,OAAO;;CAGpE,MAAM,OAAO,KAAa;AACxB,QAAMA,SAAG,GAAG,KAAK,MAAM,IAAI,CAAC;;;;AAchC,IAAa,oBAAb,MAA4D;CAC1D,SAA0B;CAE1B,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAM,MAAyB;AAC7B,QAAM,gBAAgB,KAAK,SAAS;AAEpC,UADc,MAAMA,SAAG,QAAQ,KAAK,SAAS,EAChC,QAAQ,MAAM,EAAE,SAAS,KAAK,OAAO,CAAC;;CAGrD,MAAM,OAAO,KAA+B;AAC1C,SAAO,MAAM,WAAW,KAAK,KAAK,IAAI,CAAC;;CAGzC,KAAK,KAAqB;AACxB,SAAO,KAAK,KAAK,KAAK,UAAU,MAAM,KAAK,OAAO;;CAGpD,MAAM,MAAM,KAAa,MAAkB,MAA6B;AACtE,QAAM,gBAAgB,KAAK,SAAS;AACpC,QAAM,kBAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,IAAI,EAAE,MAAM,KAAK;;CAGpF,MAAM,OAAO,KAA4B;AACvC,QAAMA,SAAG,GAAG,KAAK,KAAK,IAAI,CAAC;;;;AAK/B,IAAa,cAAb,MAAoD;;;CAIlD,OAAe,IAAI,UAAU,WAAW;CAExC,sCAA6B,IAAI,KAAmB;CACpD,OAAc;CAEd,YACE,QAEA,SACA,QACA,SACA;AALgB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;;;CAKlB,MAAM,QAAQ;AACZ,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,aAAa;IACxB;;;;CAKJ,MAAM,IAAI,KAAa,OAAgD;AACrE,SAAO,MAAM,SAAS,KAAK,MAAM,YAAY;AAC3C,UAAO,MAAM,KAAK,UAAU,KAAK,MAAM;IACvC;;;;CAKJ,MAAM,IAAI,KAAa,OAAmB,MAAiC;AACzE,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,UAAU,KAAK,OAAO,KAAK;IACtC;;CAGJ,MAAc,cAAc;AAC1B,OAAK,OAAO;AACZ,OAAK,sCAAsB,IAAI,KAAmB;EAElD,MAAM,sBAAM,IAAI,MAAM;AAItB,OAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,EAAE;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,IAAI;AACzC,QAAK,QAAQ,WAAW,OAAO;AAC/B,QAAK,oBAAoB,IAAI,KAAK,IAAI;;;CAI1C,MAAc,UAAU,KAAa,OAAgD;AAKnF,MAAI,MAAM,KAAK,QAAQ,OAAO,IAAI,EAAE;AAClC,QAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;AAG7C,OAAI,eADW,MAAM,KAAK,UAAU,IAAI,EACb,MAAM,CAC/B,QAAO,KAAK,QAAQ,KAAK,IAAI;AAG/B;;;CAMJ,MAAc,UAAU,KAAa,OAAqC,MAAkB;AAC1F,QAAM,KAAK,mBAAmB,KAAK,OAAO,KAAK;AAC/C,QAAM,KAAK,eAAe;;;CAI5B,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;AACxF,MAAI,MAAM,KAAK,MAAM,SAAS,KAAK,OACjC,OAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,MAAM,CAAC,UAAU,KAAK,SAClD;AAGH,OAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;EAE7C,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,OAAK,QAAQ,WAAW,OAAO;AAE/B,QAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,KAAK;EAE/C,MAAM,YAAY,SAAS,QAAQ,MAAM;AACzC,OAAK,QAAQ,WAAW,UAAU;AAElC,QAAM,KAAK,OAAO,IAAI,KAAK,UAAU;;;CAIvC,MAAM,gBAA+B;EACnC,MAAM,SAAS,WAAW,KAAK,oBAAoB;AACnD,SAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;AAE3E,SAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,KAAK;AAC/B,OAAI,CAAC,WACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,QAAK,QAAQ,WAAW,OAAO;AAC/B,QAAK,MAAM,IAAI;;;;CAKnB,MAAc,UAAU,KAAa;AACnC,MAAI;AACF,UAAO,MAAM,KAAK,OAAO,IAAI,IAAI;WAC1B,GAAY;AACnB,OAAI,aAAa,sBAAsB;AAGrC,UAAM,KAAK,MAAM,IAAI;AACrB,UAAM,KAAK,aAAa;AAExB,WAAO,MAAM,KAAK,OAAO,IAAI,IAAI;;AAGnC,SAAM;;;;CAKV,MAAc,MAAM,KAAa;AAC/B,QAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,QAAM,KAAK,OAAO,OAAO,IAAI;AAC7B,OAAK,oBAAoB,OAAO,IAAI;;CAGtC,MAAM,UAAyB;AAC7B,QAAM,KAAK,KAAK,cAAc;;CAGhC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,eAAe,SAAY,MAA2B,IAAkC;AACtF,KAAI;AACF,QAAM,KAAK,cAAc;AACzB,SAAO,MAAM,IAAI;WACT;AACR,OAAK,SAAS"}
+{"version":3,"file":"cache.js","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;AACT;;AAUA,IAAa,sBAAb,MAA8D;CAEzC;CACA;CAFnB,YACE,QACA,UACA;EAFiB,KAAA,SAAA;EACA,KAAA,WAAA;CAChB;CAEH,MAAc,KAAqB;EACjC,OAAO,KAAK,KAAK,KAAK,UAAU,eAAe,GAAG,CAAC;CACrD;CAEA,MAAM,IAAI,KAA8B;EACtC,OAAO,MAAM,eAAe,KAAK,QAAQ,KAAK,MAAM,GAAG,CAAC;CAC1D;CAEA,MAAM,IAAI,KAAa,QAAgB;EACrC,OAAO,MAAM,gBAAgB,KAAK,QAAQ,KAAK,MAAM,GAAG,GAAG,MAAM;CACnE;CAEA,MAAM,OAAO,KAAa;EACxB,MAAMA,SAAG,GAAG,KAAK,MAAM,GAAG,CAAC;CAC7B;AACF;;AAYA,IAAa,oBAAb,MAA4D;CAIvC;CACA;CAJnB,SAA0B;CAE1B,YACE,QACA,UACA;EAFiB,KAAA,SAAA;EACA,KAAA,WAAA;CAChB;CAEH,MAAM,MAAyB;EAC7B,MAAM,gBAAgB,KAAK,QAAQ;EAEnC,QAAO,MADaA,SAAG,QAAQ,KAAK,QAAQ,EAAA,CAC/B,QAAQ,MAAM,EAAE,SAAS,KAAK,MAAM,CAAC;CACpD;CAEA,MAAM,OAAO,KAA+B;EAC1C,OAAO,MAAM,WAAW,KAAK,KAAK,GAAG,CAAC;CACxC;CAEA,KAAK,KAAqB;EACxB,OAAO,KAAK,KAAK,KAAK,UAAU,MAAM,KAAK,MAAM;CACnD;CAEA,MAAM,MAAM,KAAa,MAAkB,MAA6B;EACtE,MAAM,gBAAgB,KAAK,QAAQ;EACnC,MAAM,kBAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,GAAG,GAAG,MAAM,IAAI;CACnF;CAEA,MAAM,OAAO,KAA4B;EACvC,MAAMA,SAAG,GAAG,KAAK,KAAK,GAAG,CAAC;CAC5B;AACF;;AAGA,IAAa,cAAb,MAAoD;CAUhC;CAEA;CACA;CACA;;;CAVlB,OAAe,IAAI,UAAU,UAAU;CAEvC,sCAA6B,IAAI,IAAkB;CACnD,OAAc;CAEd,YACE,QAEA,SACA,QACA,SACA;EALgB,KAAA,SAAA;EAEA,KAAA,UAAA;EACA,KAAA,SAAA;EACA,KAAA,UAAA;CACf;;;CAIH,MAAM,QAAQ;EACZ,MAAM,SAAS,KAAK,MAAM,YAAY;GACpC,MAAM,KAAK,YAAY;EACzB,CAAC;CACH;;;CAIA,MAAM,IAAI,KAAa,OAAgD;EACrE,OAAO,MAAM,SAAS,KAAK,MAAM,YAAY;GAC3C,OAAO,MAAM,KAAK,UAAU,KAAK,KAAK;EACxC,CAAC;CACH;;;CAIA,MAAM,IAAI,KAAa,OAAmB,MAAiC;EACzE,MAAM,SAAS,KAAK,MAAM,YAAY;GACpC,MAAM,KAAK,UAAU,KAAK,OAAO,IAAI;EACvC,CAAC;CACH;CAEA,MAAc,cAAc;EAC1B,KAAK,OAAO;EACZ,KAAK,sCAAsB,IAAI,IAAkB;EAEjD,MAAM,sBAAM,IAAI,KAAK;EAIrB,KAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,IAAI,GAAG;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,GAAG;GACxC,KAAK,QAAQ,WAAW,MAAM;GAC9B,KAAK,oBAAoB,IAAI,KAAK,GAAG;EACvC;CACF;CAEA,MAAc,UAAU,KAAa,OAAgD;EAKnF,IAAI,MAAM,KAAK,QAAQ,OAAO,GAAG,GAAG;GAClC,KAAK,oBAAoB,IAAI,qBAAK,IAAI,KAAK,CAAC;GAG5C,IAAI,eAAe,MADE,KAAK,UAAU,GAAG,GACZ,KAAK,GAC9B,OAAO,KAAK,QAAQ,KAAK,GAAG;GAG9B;EACF;CAGF;CAEA,MAAc,UAAU,KAAa,OAAqC,MAAkB;EAC1F,MAAM,KAAK,mBAAmB,KAAK,OAAO,IAAI;EAC9C,MAAM,KAAK,cAAc;CAC3B;;CAGA,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;EACxF,IAAI,MAAM,KAAK,MAAM,SAAS,KAAK,QACjC,MAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,KAAK,EAAE,UAAU,KAAK,QACnD;EAGF,KAAK,oBAAoB,IAAI,qBAAK,IAAI,KAAK,CAAC;EAE5C,MAAM,SAAS,MAAM,KAAK,UAAU,GAAG;EACvC,KAAK,QAAQ,WAAW,MAAM;EAE9B,MAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,IAAI;EAE9C,MAAM,YAAY,SAAS,QAAQ,KAAK;EACxC,KAAK,QAAQ,WAAW,SAAS;EAEjC,MAAM,KAAK,OAAO,IAAI,KAAK,SAAS;CACtC;;CAGA,MAAM,gBAA+B;EACnC,MAAM,SAAS,WAAW,KAAK,mBAAmB;EAClD,OAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,QAAQ,IAAI,MAAM,QAAQ,CAAC;EAE1E,OAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,IAAI;GAC9B,IAAI,CAAC,YACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,GAAG;GACvC,KAAK,QAAQ,WAAW,MAAM;GAC9B,KAAK,MAAM,GAAG;EAChB;CACF;;CAGA,MAAc,UAAU,KAAa;EACnC,IAAI;GACF,OAAO,MAAM,KAAK,OAAO,IAAI,GAAG;EAClC,SAAS,GAAY;GACnB,IAAI,aAAa,sBAAsB;IAGrC,MAAM,KAAK,MAAM,GAAG;IACpB,MAAM,KAAK,YAAY;IAEvB,OAAO,MAAM,KAAK,OAAO,IAAI,GAAG;GAClC;GAEA,MAAM;EACR;CACF;;CAGA,MAAc,MAAM,KAAa;EAC/B,MAAM,KAAK,QAAQ,OAAO,GAAG;EAC7B,MAAM,KAAK,OAAO,OAAO,GAAG;EAC5B,KAAK,oBAAoB,OAAO,GAAG;CACrC;CAEA,MAAM,UAAyB;EAC7B,MAAM,KAAK,KAAK,aAAa;CAC/B;CAEA,OAAO,OAAO,gBAA+B;EAC3C,MAAM,KAAK,QAAQ;CACrB;AACF;;AAGA,eAAe,SAAY,MAA2B,IAAkC;CACtF,IAAI;EACF,MAAM,KAAK,aAAa;EACxB,OAAO,MAAM,GAAG;CAClB,UAAU;EACR,KAAK,QAAQ;CACf;AACF"}

package/dist/drivers/download_blob/sparse_cache/file.cjs

@@ -1,6 +1,6 @@
 const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
 let node_fs_promises = require("node:fs/promises");
-node_fs_promises = require_runtime.__toESM(node_fs_promises);
+node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 //#region src/drivers/download_blob/sparse_cache/file.ts
 /** Creates a sparse file for all systems
@@ -35,6 +35,7 @@ async function writeToSparseFile(logger, platform, path, data, from) {
 	await fileHandle.close();
 }
 //#endregion
+exports.createSparseFile = createSparseFile;
 exports.writeToSparseFile = writeToSparseFile;
 
 //# sourceMappingURL=file.cjs.map

package/dist/drivers/download_blob/sparse_cache/file.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"file.cjs","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;AAChG,KAAI;AAEF,SADsB,MAAMA,iBAAG,KAAK,MAAM,IAAI,EAC1B,OAAO;AAE3B,QAAM,sBAAsB,MAAM,SAAS;UACpC,OAAgB;AACvB,SAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,QAAQ;;;;;;;;;AAUjF,eAAe,sBAAsB,MAAc,UAA2B;AAC5E,KAAI,aAAa,QAEf,QAAA,GAAA,2BAAA,YAAiB,UAAU;EAAC;EAAU;EAAW;EAAK,EAAE,EAAE,OAAO,QAAQ,CAAC;;;AAK9E,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;AACA,KAAI,CAAE,OAAA,GAAA,2BAAA,YAAiB,KAAK,CAC1B,OAAM,iBAAiB,QAAQ,MAAM,SAAS;CAGhD,MAAM,aAAa,MAAMA,iBAAG,KAAK,MAAM,KAAK;AAC5C,OAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,KAAK;AAClD,OAAM,WAAW,OAAO"}
+{"version":3,"file":"file.cjs","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;CAChG,IAAI;EAEF,OAAM,MADsBA,iBAAG,KAAK,MAAM,GAAG,EAAA,CACzB,MAAM;EAE1B,MAAM,sBAAsB,MAAM,QAAQ;CAC5C,SAAS,OAAgB;EACvB,OAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,OAAO;CAC9E;AACF;;;;;;;AAQA,eAAe,sBAAsB,MAAc,UAA2B;CAC5E,IAAI,aAAa,SAEf,OAAA,GAAA,2BAAA,WAAA,CAAiB,UAAU;EAAC;EAAU;EAAW;CAAI,GAAG,EAAE,OAAO,OAAO,CAAC;AAE7E;;AAGA,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;CACA,IAAI,CAAE,OAAA,GAAA,2BAAA,WAAA,CAAiB,IAAI,GACzB,MAAM,iBAAiB,QAAQ,MAAM,QAAQ;CAG/C,MAAM,aAAa,MAAMA,iBAAG,KAAK,MAAM,IAAI;CAC3C,MAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,IAAI;CACjD,MAAM,WAAW,MAAM;AACzB"}

package/dist/drivers/download_blob/sparse_cache/file.js

@@ -33,6 +33,6 @@ async function writeToSparseFile(logger, platform, path, data, from) {
 	await fileHandle.close();
 }
 //#endregion
-export { writeToSparseFile };
+export { createSparseFile, writeToSparseFile };
 
 //# sourceMappingURL=file.js.map

package/dist/drivers/download_blob/sparse_cache/file.js.map

@@ -1 +1 @@
-{"version":3,"file":"file.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;AAChG,KAAI;AAEF,SADsB,MAAM,GAAG,KAAK,MAAM,IAAI,EAC1B,OAAO;AAE3B,QAAM,sBAAsB,MAAM,SAAS;UACpC,OAAgB;AACvB,SAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,QAAQ;;;;;;;;;AAUjF,eAAe,sBAAsB,MAAc,UAA2B;AAC5E,KAAI,aAAa,QAEf,OAAM,WAAW,UAAU;EAAC;EAAU;EAAW;EAAK,EAAE,EAAE,OAAO,QAAQ,CAAC;;;AAK9E,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;AACA,KAAI,CAAE,MAAM,WAAW,KAAK,CAC1B,OAAM,iBAAiB,QAAQ,MAAM,SAAS;CAGhD,MAAM,aAAa,MAAM,GAAG,KAAK,MAAM,KAAK;AAC5C,OAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,KAAK;AAClD,OAAM,WAAW,OAAO"}
+{"version":3,"file":"file.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;CAChG,IAAI;EAEF,OAAM,MADsB,GAAG,KAAK,MAAM,GAAG,EAAA,CACzB,MAAM;EAE1B,MAAM,sBAAsB,MAAM,QAAQ;CAC5C,SAAS,OAAgB;EACvB,OAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,OAAO;CAC9E;AACF;;;;;;;AAQA,eAAe,sBAAsB,MAAc,UAA2B;CAC5E,IAAI,aAAa,SAEf,MAAM,WAAW,UAAU;EAAC;EAAU;EAAW;CAAI,GAAG,EAAE,OAAO,OAAO,CAAC;AAE7E;;AAGA,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;CACA,IAAI,CAAE,MAAM,WAAW,IAAI,GACzB,MAAM,iBAAiB,QAAQ,MAAM,QAAQ;CAG/C,MAAM,aAAa,MAAM,GAAG,KAAK,MAAM,IAAI;CAC3C,MAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,IAAI;CACjD,MAAM,WAAW,MAAM;AACzB"}

package/dist/drivers/download_blob/sparse_cache/ranges.cjs

@@ -1,7 +1,7 @@
 const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
 const require_cache = require("./cache.cjs");
 let node_fs_promises = require("node:fs/promises");
-node_fs_promises = require_runtime.__toESM(node_fs_promises);
+node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let zod = require("zod");
@@ -67,7 +67,9 @@ function addRange(s, range) {
 //#endregion
 exports.addRange = addRange;
 exports.doesRangeExist = doesRangeExist;
+exports.normalizeRanges = normalizeRanges;
 exports.rangesFileName = rangesFileName;
+exports.rangesFilePostfix = rangesFilePostfix;
 exports.rangesSize = rangesSize;
 exports.readRangesFile = readRangesFile;
 exports.writeRangesFile = writeRangesFile;

package/dist/drivers/download_blob/sparse_cache/ranges.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"ranges.cjs","names":["z","RangeBytes","fs","CorruptedRangesError"],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, \"utf8\");\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n"],"mappings":";;;;;;;;;;;AASA,MAAM,SAASA,IAAAA,EAAE,OAAO,EACtB,QAAQA,IAAAA,EAAE,MAAMC,gCAAAA,WAAW,EAC5B,CAAC;AAIF,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;AACpD,QAAO,QAAQ;;AAGjB,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,EAAE,EAAE;AACnC,KAAI;EACF,MAAM,OAAO,MAAMC,iBAAG,SAAS,MAAM,OAAO;AAC5C,WAAS,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC;UAChC,GAAY;AACnB,MAAI,aAAa,eAAe,aAAaF,IAAAA,EAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;AAC/D,UAAO,MAAM,IAAI;AACjB,SAAM,IAAIG,cAAAA,qBAAqB,IAAI;;AAGrC,MAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,UACpD,OAAM;;AAMV,iBAAgB,OAAO;AAEvB,QAAO;;;AAIT,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;AACpF,QAAA,GAAA,2BAAA,sBAA2B,QAAQ,MAAM,OAAO,aAAqB;AACnE,QAAMD,iBAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;GAC7E;;;AAIJ,SAAgB,gBAAgB,GAAW;AACzC,GAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK;AAExC,MAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,IACvC,KAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,MAAM;AAC1C,cAAY,GAAG,EAAE;AACjB;;;AAKN,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,KAAK;CAC7D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,GAAG,GAAG;AAEvD,GAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;EAAI,CAAC;;AAGrC,SAAgB,WAAW,GAAW;AACpC,QAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;;AAGxE,SAAgB,eAAe,WAAmB,OAA4B;AAC5E,MAAK,MAAM,KAAK,UAAU,OACxB,KAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,GACxC,QAAO;AAIX,QAAO;;AAGT,SAAgB,SAAS,GAAW,OAAmB;AACrD,GAAE,OAAO,KAAK,MAAM;AACpB,iBAAgB,EAAE;AAElB,QAAO"}
+{"version":3,"file":"ranges.cjs","names":["z","RangeBytes","fs","CorruptedRangesError"],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, \"utf8\");\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n"],"mappings":";;;;;;;;;;;AASA,MAAM,SAASA,IAAAA,EAAE,OAAO,EACtB,QAAQA,IAAAA,EAAE,MAAMC,gCAAAA,UAAU,EAC5B,CAAC;AAID,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;CACpD,OAAO,QAAQ;AACjB;AAEA,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,CAAC,EAAE;CAClC,IAAI;EACF,MAAM,OAAO,MAAMC,iBAAG,SAAS,MAAM,MAAM;EAC3C,SAAS,OAAO,MAAM,KAAK,MAAM,IAAI,CAAC;CACxC,SAAS,GAAY;EACnB,IAAI,aAAa,eAAe,aAAaF,IAAAA,EAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;GAC/D,OAAO,MAAM,GAAG;GAChB,MAAM,IAAIG,cAAAA,qBAAqB,GAAG;EACpC;EAEA,IAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,WACpD,MAAM;CAIV;CAEA,gBAAgB,MAAM;CAEtB,OAAO;AACT;;AAGA,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;CACpF,OAAA,GAAA,2BAAA,qBAAA,CAA2B,QAAQ,MAAM,OAAO,aAAqB;EACnE,MAAMD,iBAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,EAAE,MAAM,KAAK,CAAC;CAC9E,CAAC;AACH;;AAGA,SAAgB,gBAAgB,GAAW;CACzC,EAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI;CAEvC,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,KACvC,IAAI,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,MAAM;EAC1C,YAAY,GAAG,CAAC;EAChB;CACF;AAEJ;AAEA,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,IAAI;CAC5D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE;CAEtD,EAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;CAAG,CAAC;AACpC;AAEA,SAAgB,WAAW,GAAW;CACpC,OAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,CAAC;AACvE;AAEA,SAAgB,eAAe,WAAmB,OAA4B;CAC5E,KAAK,MAAM,KAAK,UAAU,QACxB,IAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,IACxC,OAAO;CAIX,OAAO;AACT;AAEA,SAAgB,SAAS,GAAW,OAAmB;CACrD,EAAE,OAAO,KAAK,KAAK;CACnB,gBAAgB,CAAC;CAEjB,OAAO;AACT"}

package/dist/drivers/download_blob/sparse_cache/ranges.js

@@ -63,6 +63,6 @@ function addRange(s, range) {
 	return s;
 }
 //#endregion
-export { addRange, doesRangeExist, rangesFileName, rangesSize, readRangesFile, writeRangesFile };
+export { addRange, doesRangeExist, normalizeRanges, rangesFileName, rangesFilePostfix, rangesSize, readRangesFile, writeRangesFile };
 
 //# sourceMappingURL=ranges.js.map

package/dist/drivers/download_blob/sparse_cache/ranges.js.map

@@ -1 +1 @@
-{"version":3,"file":"ranges.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, \"utf8\");\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n"],"mappings":";;;;;;;;;AASA,MAAM,SAAS,EAAE,OAAO,EACtB,QAAQ,EAAE,MAAM,WAAW,EAC5B,CAAC;AAIF,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;AACpD,QAAO,QAAQ;;AAGjB,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,EAAE,EAAE;AACnC,KAAI;EACF,MAAM,OAAO,MAAM,GAAG,SAAS,MAAM,OAAO;AAC5C,WAAS,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC;UAChC,GAAY;AACnB,MAAI,aAAa,eAAe,aAAa,EAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;AAC/D,UAAO,MAAM,IAAI;AACjB,SAAM,IAAI,qBAAqB,IAAI;;AAGrC,MAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,UACpD,OAAM;;AAMV,iBAAgB,OAAO;AAEvB,QAAO;;;AAIT,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;AACpF,OAAM,qBAAqB,QAAQ,MAAM,OAAO,aAAqB;AACnE,QAAM,GAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;GAC7E;;;AAIJ,SAAgB,gBAAgB,GAAW;AACzC,GAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK;AAExC,MAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,IACvC,KAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,MAAM;AAC1C,cAAY,GAAG,EAAE;AACjB;;;AAKN,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,KAAK;CAC7D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,GAAG,GAAG;AAEvD,GAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;EAAI,CAAC;;AAGrC,SAAgB,WAAW,GAAW;AACpC,QAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;;AAGxE,SAAgB,eAAe,WAAmB,OAA4B;AAC5E,MAAK,MAAM,KAAK,UAAU,OACxB,KAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,GACxC,QAAO;AAIX,QAAO;;AAGT,SAAgB,SAAS,GAAW,OAAmB;AACrD,GAAE,OAAO,KAAK,MAAM;AACpB,iBAAgB,EAAE;AAElB,QAAO"}
+{"version":3,"file":"ranges.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, \"utf8\");\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n"],"mappings":";;;;;;;;;AASA,MAAM,SAAS,EAAE,OAAO,EACtB,QAAQ,EAAE,MAAM,UAAU,EAC5B,CAAC;AAID,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;CACpD,OAAO,QAAQ;AACjB;AAEA,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,CAAC,EAAE;CAClC,IAAI;EACF,MAAM,OAAO,MAAM,GAAG,SAAS,MAAM,MAAM;EAC3C,SAAS,OAAO,MAAM,KAAK,MAAM,IAAI,CAAC;CACxC,SAAS,GAAY;EACnB,IAAI,aAAa,eAAe,aAAa,EAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;GAC/D,OAAO,MAAM,GAAG;GAChB,MAAM,IAAI,qBAAqB,GAAG;EACpC;EAEA,IAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,WACpD,MAAM;CAIV;CAEA,gBAAgB,MAAM;CAEtB,OAAO;AACT;;AAGA,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;CACpF,MAAM,qBAAqB,QAAQ,MAAM,OAAO,aAAqB;EACnE,MAAM,GAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,EAAE,MAAM,KAAK,CAAC;CAC9E,CAAC;AACH;;AAGA,SAAgB,gBAAgB,GAAW;CACzC,EAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI;CAEvC,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,KACvC,IAAI,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,MAAM;EAC1C,YAAY,GAAG,CAAC;EAChB;CACF;AAEJ;AAEA,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,IAAI;CAC5D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE;CAEtD,EAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;CAAG,CAAC;AACpC;AAEA,SAAgB,WAAW,GAAW;CACpC,OAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,CAAC;AACvE;AAEA,SAAgB,eAAe,WAAmB,OAA4B;CAC5E,KAAK,MAAM,KAAK,UAAU,QACxB,IAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,IACxC,OAAO;CAIX,OAAO;AACT;AAEA,SAAgB,SAAS,GAAW,OAAmB;CACrD,EAAE,OAAO,KAAK,KAAK;CACnB,gBAAgB,CAAC;CAEjB,OAAO;AACT"}

package/dist/drivers/download_blob_url/driver.cjs

@@ -8,7 +8,7 @@ const require_driver_id = require("./driver_id.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_path = require("node:path");
-node_path = require_runtime.__toESM(node_path);
+node_path = require_runtime.__toESM(node_path, 1);
 let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
@@ -17,6 +17,11 @@ let node_crypto = require("node:crypto");
 /** Downloads .tar, .tar.gz or zip archives,
 * extracts them into saveDir and gets a url for it. */
 var DownloadBlobToURLDriver = class {
+	logger;
+	signer;
+	clientDownload;
+	saveDir;
+	opts;
 	idToDownload = /* @__PURE__ */ new Map();
 	downloadQueue;
 	/** Writes and removes files to a hard drive and holds a counter for every

package/dist/drivers/download_blob_url/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","getPathForFolderURL","Computable","makeDownloadableBlobSnapshot","newId","nonRecoverableError","rmRFDir","DownloadAndUnarchiveTask","path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,OAAA,GAAA,gCAAA,aAAgB,IAAI,CAClB,QAAOC,YAAAA,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,SAAA,GAAA,wBAAA,eAAgD,IAAI,GACtDC,iBAAAA,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAIC,kBAAAA,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAID,kBAAAA,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAME,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAIC,aAAAA,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAIH,kBAAAA,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,aAAA,GAAA,0BAAA,oBAA+B,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAOA,kBAAAA,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,GAAG;AAC9C,SAAOI,UAAK,KAAK,KAAK,SAAS,GAAG,OAAO,SAAS,CAAC,GAAG,SAAS"}
+{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","getPathForFolderURL","Computable","makeDownloadableBlobSnapshot","newId","nonRecoverableError","rmRFDir","DownloadAndUnarchiveTask","path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAS3C;CACA;CACA;CACA;CACA;CAZnB,+BAA0D,IAAI,IAAI;CAClE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;CACxB,GACA;EARiB,KAAA,SAAA;EACA,KAAA,SAAA;EACA,KAAA,iBAAA;EACA,KAAA,UAAA;EACA,KAAA,OAAA;EAKjB,KAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;EACV,CAAC;EACD,KAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,kBAAkB;CAC1D;CAEA,OAAmB;EACjB,OAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,KAAK;GACzC,cAAc,MAAM,KAAK,KAAK,aAAa,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;EAC7F;CACF;;;;CAKA,yBAAyB,KAAwB;EAC/C,KAAA,GAAA,gCAAA,YAAA,CAAgB,GAAG,GACjB,OAAOC,YAAAA,oBAAoB,KAAK,QAAQ,KAAK,KAAK,OAAO;EAG3D,MAAM,IAAI,MAAM,6BAA6B,IAAI,YAAY;CAC/D;CAaA,wBACE,KACA,QACA,KAC2D;EAE3D,IAAI,QAAQ,KAAA,GACV,OAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,CAAC,CAAC;EAE5E,MAAM,SAAA,GAAA,wBAAA,cAAA,CAAgD,GAAG,IACrDC,iBAAAA,6BAA6B,KAAK,GAAG,IACrC;EAEJ,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAE5B,IAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,QAAQ,CAAC;EAEnE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,QAAQ;EACrF,IAAI,QAAQ,QAAQ,KAAA,GAClB,IAAI,aACF,4EAA4E,QAC9E;EAEF,IAAI,QAAQ,UAAU,KAAA,GAAW,MAAM,QAAQ;EAE/C,OAAO,QAAQ;CACjB;CAEA,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAIC,kBAAAA,MAAM,MAAM,IAAI,MAAM,CAAC;EAE1D,IAAI,QAAQ,KAAA,GAAW;GACrB,KAAK,OAAO,GAAG,QAAQ;GACvB,OAAO,KAAK,OAAO;EACrB;EAEA,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,QAAQ;EAC1D,KAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,QAAQ;GAClD,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,CAAC;EAC1D,CAAC;EAED,OAAO,QAAQ,OAAO;CACxB;;CAGA,MAAM,YAAY,MAAgC,UAAkB;EAClE,MAAM,KAAK,SAAS;EAEpB,IAAI,KAAK,OAAO,CAAC,EAAE,OAAO,KAAA,GAAW,KAAK,MAAM,SAAS,MAAM,QAAQ;CACzE;;;CAIA,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAID,kBAAAA,MAAM,IAAI,MAAM,CAAC;EACpD,IAAI,QAAQ,KAAA,GAAW;EAEvB,IAAI,KAAK,MAAM,WAAW,KAAK,IAAI,GAAG;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ;GAE1D,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;IACrD,MAAME,aAAAA,QAAQ,KAAK,IAAI;IACvB,KAAK,MAAM,YAAY,IAAI;IAE3B,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,qCAAA,GAAA,0BAAA,wBAAA,CACE,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,GAClF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,wBACnD;CAEN;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,MAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;GAC/D,MAAMA,aAAAA,QAAQ,KAAK,IAAI;GACvB,KAAK,MAAM,YAAY,IAAI;GAE3B,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,yCACnD;EACF,CAAC,CACH;CACF;CAEA,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAIC,aAAAA,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,MAAM,GACjC,OACA,QACA,KAAK,cACP;EACA,OAAO,OAAO,GAAG,QAAQ;EACzB,KAAK,aAAa,IAAIH,kBAAAA,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM;EAErD,OAAO;CACT;CAEA,WAAmB,MAAgC,QAAgB;EACjE,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,aAAA,GAAA,0BAAA,mBAAA,CAA+B,KAAK,MAAM,EAAE,EAAE,YAAY,QAAQ;EAC1F,KAAK,aAAa,OAAOA,kBAAAA,MAAM,KAAK,MAAM,IAAI,KAAK,MAAM,CAAC;CAC5D;CAEA,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,cAAA,GAAA,0BAAA,sBAAA,CAAmC,EAAE;EAC7C,OAAOI,UAAK,KAAK,KAAK,SAAS,GAAG,OAAO,QAAQ,EAAE,GAAG,QAAQ;CAChE;AACF"}

package/dist/drivers/download_blob_url/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"mappings":";;;;;;;;;;KA8BY,0BAAA;EACV,kBAAA;EACA,oBAAA;AAAA;;;cAKW,uBAAA,YAAmC,eAAA;EAAA,iBAS3B,MAAA;EAAA,iBACA,MAAA;EAAA,iBACA,cAAA;EAAA,iBACA,OAAA;EAAA,iBACA,IAAA;EAAA,QAZX,YAAA;EAAA,QACA,aAAA;EAgDD;;EAAA,QA5CC,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EACR,cAAA,EAAgB,cAAA,EAChB,OAAA,UACA,IAAA,GAAM,0BAAA;EAelB,IAAA,CAAA;EA4BA;;;EAfP,wBAAA,CAAyB,GAAA,EAAK,SAAA;EAQ9B,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,EACR,GAAA,EAAK,aAAA,GACJ,SAAA;EAEH,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,GACP,UAAA,CAAW,SAAA;EAAA,QA8BN,4BAAA;EA+Bc;EARhB,WAAA,CAAY,IAAA,EAAM,wBAAA,EAA0B,QAAA,WAAgB,OAAA;EAQgB;;EAA5E,WAAA,CAAY,EAAA,EAAI,gBAAA,EAAkB,MAAA,EAAQ,aAAA,EAAe,QAAA,WAAmB,OAAA;EAvHrB;EAsJvD,UAAA,CAAA,GAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAqBA,UAAA;EAAA,QAMA,WAAA;AAAA"}
+{"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"mappings":";;;;;;;;;;KA8BY,0BAAA;EACV,kBAAA;EACA,oBAAoB;AAAA;;AAAA;cAKT,uBAAA,YAAmC,eAAA;EAAA,iBAS3B,MAAA;EAAA,iBACA,MAAA;EAAA,iBACA,cAAA;EAAA,iBACA,OAAA;EAAA,iBACA,IAAA;EAAA,QAZX,YAAA;EAAA,QACA,aAAA;EAgDD;;EAAA,QA5CC,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EACR,cAAA,EAAgB,cAAA,EAChB,OAAA,UACA,IAAA,GAAM,0BAAA;EAelB,IAAA;EA4BA;;;EAfP,wBAAA,CAAyB,GAAA,EAAK,SAAA;EAQ9B,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,EACR,GAAA,EAAK,aAAA,GACJ,SAAA;EAEH,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,GACP,UAAA,CAAW,SAAA;EAAA,QA8BN,4BAAA;EA+Bc;EARhB,WAAA,CAAY,IAAA,EAAM,wBAAA,EAA0B,QAAA,WAAgB,OAAA;EAQgB;;EAA5E,WAAA,CAAY,EAAA,EAAI,gBAAA,EAAkB,MAAA,EAAQ,aAAA,EAAe,QAAA,WAAmB,OAAA;EAvHrB;EAsJvD,UAAA,IAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAqBA,UAAA;EAAA,QAMA,WAAA;AAAA"}

package/dist/drivers/download_blob_url/driver.js

@@ -15,6 +15,11 @@ import { randomUUID } from "node:crypto";
 /** Downloads .tar, .tar.gz or zip archives,
 * extracts them into saveDir and gets a url for it. */
 var DownloadBlobToURLDriver = class {
+	logger;
+	signer;
+	clientDownload;
+	saveDir;
+	opts;
 	idToDownload = /* @__PURE__ */ new Map();
 	downloadQueue;
 	/** Writes and removes files to a hard drive and holds a counter for every

package/dist/drivers/download_blob_url/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,MAAI,YAAY,IAAI,CAClB,QAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,QAAkC,cAAc,IAAI,GACtD,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,WAAW,YAAY;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,aAAa,sBAAsB,GAAG;AAC9C,SAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,SAAS,CAAC,GAAG,SAAS"}
+{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAS3C;CACA;CACA;CACA;CACA;CAZnB,+BAA0D,IAAI,IAAI;CAClE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;CACxB,GACA;EARiB,KAAA,SAAA;EACA,KAAA,SAAA;EACA,KAAA,iBAAA;EACA,KAAA,UAAA;EACA,KAAA,OAAA;EAKjB,KAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;EACV,CAAC;EACD,KAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,kBAAkB;CAC1D;CAEA,OAAmB;EACjB,OAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,KAAK;GACzC,cAAc,MAAM,KAAK,KAAK,aAAa,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;EAC7F;CACF;;;;CAKA,yBAAyB,KAAwB;EAC/C,IAAI,YAAY,GAAG,GACjB,OAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,OAAO;EAG3D,MAAM,IAAI,MAAM,6BAA6B,IAAI,YAAY;CAC/D;CAaA,wBACE,KACA,QACA,KAC2D;EAE3D,IAAI,QAAQ,KAAA,GACV,OAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,CAAC,CAAC;EAE5E,MAAM,QAAkC,cAAc,GAAG,IACrD,6BAA6B,KAAK,GAAG,IACrC;EAEJ,MAAM,WAAW,WAAW;EAE5B,IAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,QAAQ,CAAC;EAEnE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,QAAQ;EACrF,IAAI,QAAQ,QAAQ,KAAA,GAClB,IAAI,aACF,4EAA4E,QAC9E;EAEF,IAAI,QAAQ,UAAU,KAAA,GAAW,MAAM,QAAQ;EAE/C,OAAO,QAAQ;CACjB;CAEA,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,MAAM,CAAC;EAE1D,IAAI,QAAQ,KAAA,GAAW;GACrB,KAAK,OAAO,GAAG,QAAQ;GACvB,OAAO,KAAK,OAAO;EACrB;EAEA,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,QAAQ;EAC1D,KAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,QAAQ;GAClD,4BAA4B,MAAM,CAAC,oBAAoB,CAAC;EAC1D,CAAC;EAED,OAAO,QAAQ,OAAO;CACxB;;CAGA,MAAM,YAAY,MAAgC,UAAkB;EAClE,MAAM,KAAK,SAAS;EAEpB,IAAI,KAAK,OAAO,CAAC,EAAE,OAAO,KAAA,GAAW,KAAK,MAAM,SAAS,MAAM,QAAQ;CACzE;;;CAIA,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,MAAM,CAAC;EACpD,IAAI,QAAQ,KAAA,GAAW;EAEvB,IAAI,KAAK,MAAM,WAAW,KAAK,IAAI,GAAG;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ;GAE1D,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;IACrD,MAAM,QAAQ,KAAK,IAAI;IACvB,KAAK,MAAM,YAAY,IAAI;IAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,GAClF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,wBACnD;CAEN;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,MAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;GAC/D,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK,MAAM,YAAY,IAAI;GAE3B,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,yCACnD;EACF,CAAC,CACH;CACF;CAEA,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,MAAM,GACjC,OACA,QACA,KAAK,cACP;EACA,OAAO,OAAO,GAAG,QAAQ;EACzB,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM;EAErD,OAAO;CACT;CAEA,WAAmB,MAAgC,QAAgB;EACjE,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,EAAE,EAAE,YAAY,QAAQ;EAC1F,KAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,MAAM,CAAC;CAC5D;CAEA,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,aAAa,sBAAsB,EAAE;EAC7C,OAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,QAAQ,EAAE,GAAG,QAAQ;CAChE;AACF"}

package/dist/drivers/download_blob_url/driver_id.cjs

@@ -1,4 +1,3 @@
-require("../../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 //#region src/drivers/download_blob_url/driver_id.ts
 function newId(id, format) {

package/dist/drivers/download_blob_url/driver_id.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,GAAG;AAC9C,QAAO,MAAM,OAAO,SAAS,CAAC,GAAG"}
+{"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,sBAAA,CAAmC,EAAE;CAC7C,OAAO,MAAM,OAAO,QAAQ,EAAE,GAAG;AACnC"}

package/dist/drivers/download_blob_url/driver_id.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver_id.js","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,aAAa,sBAAsB,GAAG;AAC9C,QAAO,MAAM,OAAO,SAAS,CAAC,GAAG"}
+{"version":3,"file":"driver_id.js","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,aAAa,sBAAsB,EAAE;CAC7C,OAAO,MAAM,OAAO,QAAQ,EAAE,GAAG;AACnC"}

package/dist/drivers/download_blob_url/snapshot.cjs

@@ -1,4 +1,3 @@
-require("../../_virtual/_rolldown/runtime.cjs");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 //#region src/drivers/download_blob_url/snapshot.ts
 /** We need only resource type for this driver. */

package/dist/drivers/download_blob_url/snapshot.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;;AAeA,MAAa,4BAAA,GAAA,wBAAA,UAAoC,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,SAAA,GAAA,wBAAA,uBAAA,GAAA,wBAAA,eAN2B,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,IAAA,GAAA,wBAAA,uBACd,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}
+{"version":3,"file":"snapshot.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;AAeA,MAAa,4BAAA,GAAA,wBAAA,SAAA,CAAoC,CAAC,CAAC;AAGnD,SAAgB,6BACd,iBACA,KAC0B;CAO1B,QAAA,GAAA,wBAAA,qBAAA,EAAA,GAAA,wBAAA,cAAA,CAN2B,eAAe,IACtC,IAAI,SAAS,eAAe,CAAC,CAAC,KAAK,KAAA,GAAA,wBAAA,sBAAA,CACb,eAAe,IACnC,gBAAgB,KAAK,IACrB,iBAE4B,wBAAwB;AAC5D"}

package/dist/drivers/download_blob_url/snapshot.d.ts

@@ -1,10 +1,9 @@
 import { ComputableCtx } from "@milaboratories/computable";
-import * as _$_milaboratories_pl_tree0 from "@milaboratories/pl-tree";
 import { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 
 //#region src/drivers/download_blob_url/snapshot.d.ts
 /** We need only resource type for this driver. */
-declare const DownloadableBlobSnapshot: _$_milaboratories_pl_tree0.ResourceSnapshotSchema<undefined, undefined, undefined>;
+declare const DownloadableBlobSnapshot: import("@milaboratories/pl-tree").ResourceSnapshotSchema<undefined, undefined, undefined>;
 type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;
 declare function makeDownloadableBlobSnapshot(entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor, ctx: ComputableCtx): DownloadableBlobSnapshot;
 //#endregion

package/dist/drivers/download_blob_url/snapshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"mappings":";;;;;;cAea,wBAAA,EAAuC,0BAAA,CAAf,sBAAA;AAAA,KACzB,wBAAA,GAA2B,aAAA,QAAqB,wBAAA;AAAA,iBAE5C,4BAAA,CACd,eAAA,EAAiB,WAAA,GAAc,kBAAA,GAAqB,mBAAA,EACpD,GAAA,EAAK,aAAA,GACJ,wBAAA"}
+{"version":3,"file":"snapshot.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"mappings":";;;;;cAea,wBAAA,oCAAwB,sBAAA;AAAA,KACzB,wBAAA,GAA2B,aAAa,QAAQ,wBAAA;AAAA,iBAE5C,4BAAA,CACd,eAAA,EAAiB,WAAA,GAAc,kBAAA,GAAqB,mBAAA,EACpD,GAAA,EAAK,aAAA,GACJ,wBAAA"}

package/dist/drivers/download_blob_url/snapshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.js","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;AAeA,MAAa,2BAA2B,SAAS,EAAE,CAAC;AAGpD,SAAgB,6BACd,iBACA,KAC0B;AAO1B,QAAO,qBANM,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB,iBAE4B,yBAAyB"}
+{"version":3,"file":"snapshot.js","names":[],"sources":["../../../src/drivers/download_blob_url/snapshot.ts"],"sourcesContent":["import type { ComputableCtx } from \"@milaboratories/computable\";\nimport type {\n  InferSnapshot,\n  PlTreeEntry,\n  PlTreeEntryAccessor,\n  PlTreeNodeAccessor,\n} from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n  rsSchema,\n} from \"@milaboratories/pl-tree\";\n\n/** We need only resource type for this driver. */\nexport const DownloadableBlobSnapshot = rsSchema({});\nexport type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;\n\nexport function makeDownloadableBlobSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): DownloadableBlobSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  return makeResourceSnapshot(node, DownloadableBlobSnapshot);\n}\n"],"mappings":";;;AAeA,MAAa,2BAA2B,SAAS,CAAC,CAAC;AAGnD,SAAgB,6BACd,iBACA,KAC0B;CAO1B,OAAO,qBANM,cAAc,eAAe,IACtC,IAAI,SAAS,eAAe,CAAC,CAAC,KAAK,IACnC,sBAAsB,eAAe,IACnC,gBAAgB,KAAK,IACrB,iBAE4B,wBAAwB;AAC5D"}