@milaboratories/pl-drivers 1.9.0 → 1.10.0

package/dist/drivers/download_blob/download_blob_task.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"download_blob_task.cjs","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} finished`);\n    } catch (e: any) {\n      this.logger.error(`download blob ${stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["ChangeSource","CallersCounter","resourceIdToString","stringifyWithResourceId","fsp","ensureDirExists","path","fileExists","createPathAtomically","Writable","NetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAIA,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAIC,wBAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QAC5F;QAAE,OAAO,CAAM,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiBC,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;AAC5H,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqBD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAExF,gBAAA,MAAME,cAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,+BAAA,EAAkC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CAAC;AACzE,YAAA,MAAM,IAAI,GAAG,MAAMH,cAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAMI,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAGC,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;;;"}

package/dist/drivers/download_blob/download_blob_task.d.ts

@@ -51,4 +51,3 @@ type DownloadState = {
     done?: boolean;
 };
 export {};
-//# sourceMappingURL=download_blob_task.d.ts.map

package/dist/drivers/download_blob/download_blob_task.js

@@ -0,0 +1,146 @@
+import { ChangeSource } from '@milaboratories/computable';
+import { CallersCounter, ensureDirExists, fileExists, createPathAtomically } from '@milaboratories/ts-helpers';
+import fs__default from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+import { Writable } from 'node:stream';
+import { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download.js';
+import { NetworkError400 } from '../../helpers/download.js';
+import { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';
+
+/** Downloads a blob and holds callers and watchers for the blob. */
+class DownloadBlobTask {
+    logger;
+    clientDownload;
+    rInfo;
+    handle;
+    path;
+    change = new ChangeSource();
+    signalCtl = new AbortController();
+    counter = new CallersCounter();
+    error;
+    done = false;
+    size = 0;
+    state = {};
+    constructor(logger, clientDownload, rInfo, handle, path) {
+        this.logger = logger;
+        this.clientDownload = clientDownload;
+        this.rInfo = rInfo;
+        this.handle = handle;
+        this.path = path;
+    }
+    /** Returns a simple object that describes this task for debugging purposes. */
+    info() {
+        return {
+            rInfo: this.rInfo,
+            fPath: this.path,
+            done: this.done,
+            error: this.error,
+            state: this.state,
+        };
+    }
+    attach(w, callerId) {
+        this.counter.inc(callerId);
+        if (!this.done)
+            this.change.attachWatcher(w);
+    }
+    async download() {
+        try {
+            const size = await this.ensureDownloaded();
+            this.setDone(size);
+            this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} finished`);
+        }
+        catch (e) {
+            this.logger.error(`download blob ${stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);
+            if (nonRecoverableError(e)) {
+                this.setError(e);
+                this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} failed`);
+                // Just in case we were half-way extracting an archive.
+                await fsp.rm(this.path, { force: true });
+            }
+            throw e;
+        }
+    }
+    async ensureDownloaded() {
+        this.signalCtl.signal.throwIfAborted();
+        this.state = {};
+        this.state.filePath = this.path;
+        await ensureDirExists(path.dirname(this.state.filePath));
+        this.signalCtl.signal.throwIfAborted();
+        this.state.dirExists = true;
+        const alreadyExists = await fileExists(this.state.filePath);
+        this.signalCtl.signal.throwIfAborted();
+        if (alreadyExists) {
+            this.state.fileExists = true;
+            this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);
+            const stat = await fsp.stat(this.state.filePath);
+            this.signalCtl.signal.throwIfAborted();
+            this.state.fileSize = stat.size;
+            return this.state.fileSize;
+        }
+        const fileSize = await this.clientDownload.withBlobContent(this.rInfo, {}, { signal: this.signalCtl.signal }, async (content, size) => {
+            this.state.fileSize = size;
+            this.state.downloaded = true;
+            await createPathAtomically(this.logger, this.state.filePath, async (fPath) => {
+                const f = Writable.toWeb(fs__default.createWriteStream(fPath, { flags: 'wx' }));
+                await content.pipeTo(f, { signal: this.signalCtl.signal });
+                this.state.tempWritten = true;
+            });
+            this.state.done = true;
+            return size;
+        });
+        return fileSize;
+    }
+    abort(reason) {
+        this.signalCtl.abort(new DownloadAborted(reason));
+    }
+    getBlob() {
+        if (!this.done)
+            return { done: false };
+        return {
+            done: this.done,
+            result: getDownloadedBlobResponse(this.handle, this.size, this.error),
+        };
+    }
+    setDone(sizeBytes) {
+        this.done = true;
+        this.size = sizeBytes;
+    }
+    setError(e) {
+        this.done = true;
+        this.error = e;
+    }
+}
+function nonRecoverableError(e) {
+    return (e instanceof DownloadAborted
+        || e instanceof NetworkError400
+        || e instanceof UnknownStorageError
+        || e instanceof WrongLocalFileUrl
+        // file that we downloads from was moved or deleted.
+        || e?.code == 'ENOENT'
+        // A resource was deleted.
+        || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED')));
+}
+/** The downloading task was aborted by a signal.
+ * It may happen when the computable is done, for example. */
+class DownloadAborted extends Error {
+    name = 'DownloadAborted';
+}
+function getDownloadedBlobResponse(handle, size, error) {
+    if (error) {
+        return { ok: false, error };
+    }
+    if (!handle) {
+        return { ok: false, error: new Error('No file or handle provided') };
+    }
+    return {
+        ok: true,
+        value: {
+            handle,
+            size,
+        },
+    };
+}
+
+export { DownloadBlobTask, getDownloadedBlobResponse, nonRecoverableError };
+//# sourceMappingURL=download_blob_task.js.map

package/dist/drivers/download_blob/download_blob_task.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"download_blob_task.js","sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';\nimport type { ResourceSnapshot } from '@milaboratories/pl-tree';\nimport type {\n  ValueOrError,\n  MiLogger,\n} from '@milaboratories/ts-helpers';\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from '@milaboratories/ts-helpers';\nimport fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Writable } from 'node:stream';\nimport type { ClientDownload } from '../../clients/download';\nimport { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';\nimport { NetworkError400 } from '../../helpers/download';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} finished`);\n    } catch (e: any) {\n      this.logger.error(`download blob ${stringifyWithResourceId(this.rInfo)} failed: ${e}, state: ${JSON.stringify(this.state)}`);\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob download for ${resourceIdToString(this.rInfo.id)} failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(`a blob was already downloaded: ${this.state.filePath}`);\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: 'wx' }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      }\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n      done: true;\n      result: ValueOrError<LocalBlobHandleAndSize>;\n    } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted\n    || e instanceof NetworkError400\n    || e instanceof UnknownStorageError\n    || e instanceof WrongLocalFileUrl\n    // file that we downloads from was moved or deleted.\n    || e?.code == 'ENOENT'\n    // A resource was deleted.\n    || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = 'DownloadAborted';\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error('No file or handle provided') };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n}\n"],"names":["fs"],"mappings":";;;;;;;;;;AAuBA;MACa,gBAAgB,CAAA;AAUR,IAAA,MAAA;AACA,IAAA,cAAA;AACR,IAAA,KAAA;AACQ,IAAA,MAAA;AACR,IAAA,IAAA;AAbF,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClC,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AACtC,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACb,IAAI,GAAG,CAAC;IACP,KAAK,GAAkB,EAAE;IAEjC,WAAA,CACmB,MAAgB,EAChB,cAA8B,EACtC,KAAuB,EACf,MAAuB,EAC/B,IAAY,EAAA;QAJJ,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,cAAc,GAAd,cAAc;QACtB,IAAA,CAAA,KAAK,GAAL,KAAK;QACG,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;IACZ;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEO,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACxC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEO,IAAA,MAAM,QAAQ,GAAA;AACnB,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;AAC1C,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;QAC5F;QAAE,OAAO,CAAM,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,CAAE,CAAC;AAC5H,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAExF,gBAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,GAAA;AAC5B,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AAEtC,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAC/B,QAAA,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AACxD,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;AACtC,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI;QAE3B,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAC3D,QAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QACtC,IAAI,aAAa,EAAE;AACjB,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAC5B,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,+BAAA,EAAkC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAA,CAAE,CAAC;AACzE,YAAA,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;AAChD,YAAA,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI;AAE/B,YAAA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ;QAC5B;AAEA,QAAA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACxD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EACjC,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI;AAC1B,YAAA,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI;AAE5B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,QAAS,EAAE,OAAO,KAAa,KAAI;AACpF,gBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACA,WAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,gBAAA,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;AAC1D,gBAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;AAC/B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI;AACtB,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,QAAQ;IACjB;AAEO,IAAA,KAAK,CAAC,MAAc,EAAA;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IACnD;IAEO,OAAO,GAAA;QAMZ,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;QAEtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,YAAA,MAAM,EAAE,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;SACtE;IACH;AAEQ,IAAA,OAAO,CAAC,SAAiB,EAAA;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,SAAS;IACvB;AAEQ,IAAA,QAAQ,CAAC,CAAU,EAAA;AACzB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,KAAK,GAAG,CAAC;IAChB;AACD;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;YAEV,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;AAE/E;AAEA;AAC6D;AAC7D,MAAM,eAAgB,SAAQ,KAAK,CAAA;IACjC,IAAI,GAAG,iBAAiB;AACzB;SAEe,yBAAyB,CACvC,MAAmC,EACnC,IAAY,EACZ,KAAe,EAAA;IAEf,IAAI,KAAK,EAAE;AACT,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC7B;IAEA,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE;IACtE;IAEA,OAAO;AACL,QAAA,EAAE,EAAE,IAAI;AACR,QAAA,KAAK,EAAE;YACL,MAAM;YACN,IAAI;AACL,SAAA;KACF;AACH;;;;"}

package/dist/drivers/download_blob/sparse_cache/cache.cjs

@@ -0,0 +1,202 @@
+'use strict';
+
+var tsHelpers = require('@milaboratories/ts-helpers');
+var fs = require('node:fs');
+var path = require('node:path');
+var ranges = require('./ranges.cjs');
+var file = require('./file.cjs');
+var helpers = require('@milaboratories/helpers');
+
+/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */
+class CorruptedRangesError extends Error {
+    name = 'CorruptedRangesError';
+}
+/** Stores ranges in a directory as JSON files (the default implementation). */
+class SparseCacheFsRanges {
+    logger;
+    cacheDir;
+    constructor(logger, cacheDir) {
+        this.logger = logger;
+        this.cacheDir = cacheDir;
+    }
+    fPath(key) {
+        return path.join(this.cacheDir, ranges.rangesFileName(key));
+    }
+    async get(key) {
+        return await ranges.readRangesFile(this.logger, this.fPath(key));
+    }
+    async set(key, ranges$1) {
+        return await ranges.writeRangesFile(this.logger, this.fPath(key), ranges$1);
+    }
+    async delete(key) {
+        await fs.promises.rm(this.fPath(key));
+    }
+}
+/** Stores sparse files in a directory (the default implementation). */
+class SparseCacheFsFile {
+    logger;
+    cacheDir;
+    suffix = '.sparse.bin';
+    constructor(logger, cacheDir) {
+        this.logger = logger;
+        this.cacheDir = cacheDir;
+    }
+    async all() {
+        await tsHelpers.ensureDirExists(this.cacheDir);
+        const files = await fs.promises.readdir(this.cacheDir);
+        return files.filter((f) => f.endsWith(this.suffix));
+    }
+    async exists(key) {
+        return await tsHelpers.fileExists(this.path(key));
+    }
+    path(key) {
+        return path.join(this.cacheDir, key + this.suffix);
+    }
+    async write(key, data, from) {
+        await tsHelpers.ensureDirExists(this.cacheDir);
+        await file.writeToSparseFile(this.logger, process.platform, this.path(key), data, from);
+    }
+    async delete(key) {
+        await fs.promises.rm(this.path(key));
+    }
+}
+/** LRU cache for ranges of sparse files. */
+class SparseCache {
+    logger;
+    maxSize;
+    ranges;
+    storage;
+    /** Fields are public for tests. */
+    /** The lock to make sure cache requests are done one by one. */
+    lock = new helpers.functions.AwaitLock();
+    keyToLastAccessTime = new Map();
+    size = 0;
+    constructor(logger, 
+    /** The hard limit in bytes. */
+    maxSize, ranges, storage) {
+        this.logger = logger;
+        this.maxSize = maxSize;
+        this.ranges = ranges;
+        this.storage = storage;
+    }
+    /** Resets a cache's size by rereading everything we already store.
+     * Safe for concurrent use. */
+    async reset() {
+        await withLock(this.lock, async () => {
+            await this.resetUnsafe();
+        });
+    }
+    /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.
+     * Safe for concurrent use. */
+    async get(key, range) {
+        return await withLock(this.lock, async () => {
+            return await this.getUnsafe(key, range);
+        });
+    }
+    /** Sets data to the cache's file and clear the cache if it's needed.
+     * Safe for concurrent use. */
+    async set(key, range, data) {
+        await withLock(this.lock, async () => {
+            await this.setUnsafe(key, range, data);
+        });
+    }
+    async resetUnsafe() {
+        this.size = 0;
+        this.keyToLastAccessTime = new Map();
+        const now = new Date();
+        // In rmKey method we first deletes the key from a storage and only then from ranges,
+        // so if something goes wrong between 2 operations,
+        // on reset the logic will be correct.
+        for (const key of await this.storage.all()) {
+            const ranges$1 = await this.ranges.get(key);
+            this.size += ranges.rangesSize(ranges$1);
+            this.keyToLastAccessTime.set(key, now);
+        }
+    }
+    async getUnsafe(key, range) {
+        // It first checks the storage, and then the ranges.
+        // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,
+        // so if we don't have a key in storage but have it in ranges, the logic here is correct.
+        // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.
+        if (await this.storage.exists(key)) {
+            this.keyToLastAccessTime.set(key, new Date());
+            const ranges$1 = await this.getRanges(key);
+            if (ranges.doesRangeExist(ranges$1, range)) {
+                return this.storage.path(key);
+            }
+            return undefined;
+        }
+        return undefined;
+    }
+    async setUnsafe(key, range, data) {
+        await this.setWithoutEviction(key, range, data);
+        await this.ensureEvicted();
+    }
+    /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */
+    async setWithoutEviction(key, range, data) {
+        if (range.to - range.from !== data.length) {
+            throw new Error(`SparseCache.set: trying to set ${key} with wrong range length: `
+                + `range: ${JSON.stringify(range)}, data: ${data.length}`);
+        }
+        this.keyToLastAccessTime.set(key, new Date());
+        const ranges$1 = await this.getRanges(key);
+        this.size -= ranges.rangesSize(ranges$1);
+        await this.storage.write(key, data, range.from);
+        const newRanges = ranges.addRange(ranges$1, range);
+        this.size += ranges.rangesSize(newRanges);
+        await this.ranges.set(key, newRanges);
+    }
+    /** Ensures the size is less than hard limit by deleting the oldest keys. */
+    async ensureEvicted() {
+        const byTime = tsHelpers.mapEntries(this.keyToLastAccessTime);
+        byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());
+        while (this.size > this.maxSize) {
+            const keyAndDate = byTime.pop(); // removes the oldest
+            if (!keyAndDate) {
+                break;
+            }
+            const [key, _] = keyAndDate;
+            const ranges$1 = await this.getRanges(key);
+            this.size -= ranges.rangesSize(ranges$1);
+            this.rmKey(key);
+        }
+    }
+    /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */
+    async getRanges(key) {
+        try {
+            return await this.ranges.get(key);
+        }
+        catch (e) {
+            if (e instanceof CorruptedRangesError) {
+                // We need to reset a state of the cache and update current size,
+                // it's the only way to calculate the real size when one of the ranges were corrupted.
+                await this.rmKey(key);
+                await this.resetUnsafe();
+                return await this.ranges.get(key);
+            }
+            throw e;
+        }
+    }
+    /** Removes a key the state of the cache. The size should be updated. */
+    async rmKey(key) {
+        await this.storage.delete(key);
+        await this.ranges.delete(key);
+        this.keyToLastAccessTime.delete(key);
+    }
+}
+/** Acquires the lock and executes a callback. */
+async function withLock(lock, cb) {
+    try {
+        await lock.acquireAsync();
+        return await cb();
+    }
+    finally {
+        lock.release();
+    }
+}
+
+exports.CorruptedRangesError = CorruptedRangesError;
+exports.SparseCache = SparseCache;
+exports.SparseCacheFsFile = SparseCacheFsFile;
+exports.SparseCacheFsRanges = SparseCacheFsRanges;
+//# sourceMappingURL=cache.cjs.map

package/dist/drivers/download_blob/sparse_cache/cache.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.cjs","sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from '@milaboratories/pl-model-common';\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from '@milaboratories/ts-helpers';\nimport { promises as fs } from 'node:fs';\nimport path from 'node:path';\nimport { addRange, doesRangeExist, Ranges, rangesFileName, rangesFilePostfix, rangesSize, readRangesFile, writeRangesFile } from './ranges';\nimport { writeToSparseFile } from './file';\nimport { functions } from '@milaboratories/helpers';\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = 'CorruptedRangesError';\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = '.sparse.bin';\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock()\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    })\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    })\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number; }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: `\n        + `range: ${JSON.stringify(range)}, data: ${data.length}`\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"names":["rangesFileName","readRangesFile","ranges","writeRangesFile","fs","ensureDirExists","fileExists","writeToSparseFile","functions","rangesSize","doesRangeExist","addRange","mapEntries"],"mappings":";;;;;;;;;AAQA;AACM,MAAO,oBAAqB,SAAQ,KAAK,CAAA;IAC7C,IAAI,GAAG,sBAAsB;AAC9B;AASD;MACa,mBAAmB,CAAA;AAEX,IAAA,MAAA;AACA,IAAA,QAAA;IAFnB,WAAA,CACmB,MAAgB,EAChB,QAAgB,EAAA;QADhB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,QAAQ,GAAR,QAAQ;IACxB;AAEK,IAAA,KAAK,CAAC,GAAW,EAAA;AACvB,QAAA,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAEA,qBAAc,CAAC,GAAG,CAAC,CAAC;IACtD;IAEA,MAAM,GAAG,CAAC,GAAW,EAAA;AACnB,QAAA,OAAO,MAAMC,qBAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3D;AAEA,IAAA,MAAM,GAAG,CAAC,GAAW,EAAEC,QAAc,EAAA;AACnC,QAAA,OAAO,MAAMC,sBAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAED,QAAM,CAAC;IACpE;IAEA,MAAM,MAAM,CAAC,GAAW,EAAA;QACtB,MAAME,WAAE,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B;AACD;AAWD;MACa,iBAAiB,CAAA;AAIT,IAAA,MAAA;AACA,IAAA,QAAA;IAJF,MAAM,GAAG,aAAa;IAEvC,WAAA,CACmB,MAAgB,EAChB,QAAgB,EAAA;QADhB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,QAAQ,GAAR,QAAQ;IACxB;AAEH,IAAA,MAAM,GAAG,GAAA;AACP,QAAA,MAAMC,yBAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;QACpC,MAAM,KAAK,GAAG,MAAMD,WAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;AAC7C,QAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrD;IAEA,MAAM,MAAM,CAAC,GAAW,EAAA;QACtB,OAAO,MAAME,oBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC;AAEA,IAAA,IAAI,CAAC,GAAW,EAAA;AACd,QAAA,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;IACpD;AAEA,IAAA,MAAM,KAAK,CAAC,GAAW,EAAE,IAAgB,EAAE,IAAY,EAAA;AACrD,QAAA,MAAMD,yBAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;QACpC,MAAME,sBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC;IACpF;IAEA,MAAM,MAAM,CAAC,GAAW,EAAA;QACtB,MAAMH,WAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B;AACD;AAED;MACa,WAAW,CAAA;AAUJ,IAAA,MAAA;AAEA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;;;AAVV,IAAA,IAAI,GAAG,IAAII,iBAAS,CAAC,SAAS,EAAE;AAEjC,IAAA,mBAAmB,GAAG,IAAI,GAAG,EAAgB;IAC7C,IAAI,GAAG,CAAC;AAEf,IAAA,WAAA,CACkB,MAAgB;;IAEhB,OAAe,EACf,MAAyB,EACzB,OAA0B,EAAA;QAJ1B,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACtB;AAEH;AAC8B;AAC9B,IAAA,MAAM,KAAK,GAAA;QACT,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,YAAW;AACnC,YAAA,MAAM,IAAI,CAAC,WAAW,EAAE;AAC1B,QAAA,CAAC,CAAC;IACJ;AAEA;AAC8B;AAC9B,IAAA,MAAM,GAAG,CAAC,GAAW,EAAE,KAAiB,EAAA;QACtC,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,YAAW;YAC1C,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC;AACzC,QAAA,CAAC,CAAC;IACJ;AAEA;AAC8B;AAC9B,IAAA,MAAM,GAAG,CAAC,GAAW,EAAE,KAAiB,EAAE,IAAgB,EAAA;QACxD,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,YAAW;YACnC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC;AACxC,QAAA,CAAC,CAAC;IACJ;AAEQ,IAAA,MAAM,WAAW,GAAA;AACvB,QAAA,IAAI,CAAC,IAAI,GAAG,CAAC;AACb,QAAA,IAAI,CAAC,mBAAmB,GAAG,IAAI,GAAG,EAAgB;AAElD,QAAA,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE;;;;QAItB,KAAK,MAAM,GAAG,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE;YAC1C,MAAMN,QAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;AACzC,YAAA,IAAI,CAAC,IAAI,IAAIO,iBAAU,CAACP,QAAM,CAAC;YAC/B,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC;QACxC;IACF;AAEQ,IAAA,MAAM,SAAS,CAAC,GAAW,EAAE,KAAiB,EAAA;;;;;QAKpD,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAClC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC;YAE7C,MAAMA,QAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxC,YAAA,IAAIQ,qBAAc,CAACR,QAAM,EAAE,KAAK,CAAC,EAAE;gBACjC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;YAC/B;AAEA,YAAA,OAAO,SAAS;QAClB;AAEA,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,MAAM,SAAS,CAAC,GAAW,EAAE,KAAoC,EAAE,IAAgB,EAAA;QACzF,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC;AAC/C,QAAA,MAAM,IAAI,CAAC,aAAa,EAAE;IAC5B;;AAGA,IAAA,MAAM,kBAAkB,CAAC,GAAW,EAAE,KAAiB,EAAE,IAAgB,EAAA;AACvE,QAAA,IAAI,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE;AACzC,YAAA,MAAM,IAAI,KAAK,CACb,CAAA,+BAAA,EAAkC,GAAG,CAAA,0BAAA;AACnC,kBAAA,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA,QAAA,EAAW,IAAI,CAAC,MAAM,CAAA,CAAE,CAC1D;QACH;QAEA,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC;QAE7C,MAAMA,QAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxC,QAAA,IAAI,CAAC,IAAI,IAAIO,iBAAU,CAACP,QAAM,CAAC;AAE/B,QAAA,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QAE/C,MAAM,SAAS,GAAGS,eAAQ,CAACT,QAAM,EAAE,KAAK,CAAC;AACzC,QAAA,IAAI,CAAC,IAAI,IAAIO,iBAAU,CAAC,SAAS,CAAC;QAElC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC;IACvC;;AAGA,IAAA,MAAM,aAAa,GAAA;QACjB,MAAM,MAAM,GAAGG,oBAAU,CAAC,IAAI,CAAC,mBAAmB,CAAC;AACnD,QAAA,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAE3E,OAAO,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE;YAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,UAAU,EAAE;gBACf;YACF;AACA,YAAA,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,UAAU;YAE3B,MAAMV,QAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxC,YAAA,IAAI,CAAC,IAAI,IAAIO,iBAAU,CAACP,QAAM,CAAC;AAC/B,YAAA,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;QACjB;IACF;;IAGQ,MAAM,SAAS,CAAC,GAAW,EAAA;AACjC,QAAA,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;QACnC;QAAE,OAAO,CAAU,EAAE;AACnB,YAAA,IAAI,CAAC,YAAY,oBAAoB,EAAE;;;AAGrC,gBAAA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;AACrB,gBAAA,MAAM,IAAI,CAAC,WAAW,EAAE;gBAExB,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YACnC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,KAAK,CAAC,GAAW,EAAA;QAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;QAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;AAC7B,QAAA,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC;IACtC;AACD;AAED;AACA,eAAe,QAAQ,CAAI,IAAyB,EAAE,EAAoB,EAAA;AACxE,IAAA,IAAI;AACF,QAAA,MAAM,IAAI,CAAC,YAAY,EAAE;QACzB,OAAO,MAAM,EAAE,EAAE;IACnB;YAAU;QACR,IAAI,CAAC,OAAO,EAAE;IAChB;AACF;;;;;;;"}

package/dist/drivers/download_blob/sparse_cache/cache.d.ts

@@ -77,4 +77,3 @@ export declare class SparseCache {
     /** Removes a key the state of the cache. The size should be updated. */
     private rmKey;
 }
-//# sourceMappingURL=cache.d.ts.map

package/dist/drivers/download_blob/sparse_cache/cache.js

@@ -0,0 +1,197 @@
+import { ensureDirExists, fileExists, mapEntries } from '@milaboratories/ts-helpers';
+import { promises } from 'node:fs';
+import path__default from 'node:path';
+import { rangesFileName, readRangesFile, writeRangesFile, rangesSize, doesRangeExist, addRange } from './ranges.js';
+import { writeToSparseFile } from './file.js';
+import { functions } from '@milaboratories/helpers';
+
+/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */
+class CorruptedRangesError extends Error {
+    name = 'CorruptedRangesError';
+}
+/** Stores ranges in a directory as JSON files (the default implementation). */
+class SparseCacheFsRanges {
+    logger;
+    cacheDir;
+    constructor(logger, cacheDir) {
+        this.logger = logger;
+        this.cacheDir = cacheDir;
+    }
+    fPath(key) {
+        return path__default.join(this.cacheDir, rangesFileName(key));
+    }
+    async get(key) {
+        return await readRangesFile(this.logger, this.fPath(key));
+    }
+    async set(key, ranges) {
+        return await writeRangesFile(this.logger, this.fPath(key), ranges);
+    }
+    async delete(key) {
+        await promises.rm(this.fPath(key));
+    }
+}
+/** Stores sparse files in a directory (the default implementation). */
+class SparseCacheFsFile {
+    logger;
+    cacheDir;
+    suffix = '.sparse.bin';
+    constructor(logger, cacheDir) {
+        this.logger = logger;
+        this.cacheDir = cacheDir;
+    }
+    async all() {
+        await ensureDirExists(this.cacheDir);
+        const files = await promises.readdir(this.cacheDir);
+        return files.filter((f) => f.endsWith(this.suffix));
+    }
+    async exists(key) {
+        return await fileExists(this.path(key));
+    }
+    path(key) {
+        return path__default.join(this.cacheDir, key + this.suffix);
+    }
+    async write(key, data, from) {
+        await ensureDirExists(this.cacheDir);
+        await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);
+    }
+    async delete(key) {
+        await promises.rm(this.path(key));
+    }
+}
+/** LRU cache for ranges of sparse files. */
+class SparseCache {
+    logger;
+    maxSize;
+    ranges;
+    storage;
+    /** Fields are public for tests. */
+    /** The lock to make sure cache requests are done one by one. */
+    lock = new functions.AwaitLock();
+    keyToLastAccessTime = new Map();
+    size = 0;
+    constructor(logger, 
+    /** The hard limit in bytes. */
+    maxSize, ranges, storage) {
+        this.logger = logger;
+        this.maxSize = maxSize;
+        this.ranges = ranges;
+        this.storage = storage;
+    }
+    /** Resets a cache's size by rereading everything we already store.
+     * Safe for concurrent use. */
+    async reset() {
+        await withLock(this.lock, async () => {
+            await this.resetUnsafe();
+        });
+    }
+    /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.
+     * Safe for concurrent use. */
+    async get(key, range) {
+        return await withLock(this.lock, async () => {
+            return await this.getUnsafe(key, range);
+        });
+    }
+    /** Sets data to the cache's file and clear the cache if it's needed.
+     * Safe for concurrent use. */
+    async set(key, range, data) {
+        await withLock(this.lock, async () => {
+            await this.setUnsafe(key, range, data);
+        });
+    }
+    async resetUnsafe() {
+        this.size = 0;
+        this.keyToLastAccessTime = new Map();
+        const now = new Date();
+        // In rmKey method we first deletes the key from a storage and only then from ranges,
+        // so if something goes wrong between 2 operations,
+        // on reset the logic will be correct.
+        for (const key of await this.storage.all()) {
+            const ranges = await this.ranges.get(key);
+            this.size += rangesSize(ranges);
+            this.keyToLastAccessTime.set(key, now);
+        }
+    }
+    async getUnsafe(key, range) {
+        // It first checks the storage, and then the ranges.
+        // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,
+        // so if we don't have a key in storage but have it in ranges, the logic here is correct.
+        // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.
+        if (await this.storage.exists(key)) {
+            this.keyToLastAccessTime.set(key, new Date());
+            const ranges = await this.getRanges(key);
+            if (doesRangeExist(ranges, range)) {
+                return this.storage.path(key);
+            }
+            return undefined;
+        }
+        return undefined;
+    }
+    async setUnsafe(key, range, data) {
+        await this.setWithoutEviction(key, range, data);
+        await this.ensureEvicted();
+    }
+    /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */
+    async setWithoutEviction(key, range, data) {
+        if (range.to - range.from !== data.length) {
+            throw new Error(`SparseCache.set: trying to set ${key} with wrong range length: `
+                + `range: ${JSON.stringify(range)}, data: ${data.length}`);
+        }
+        this.keyToLastAccessTime.set(key, new Date());
+        const ranges = await this.getRanges(key);
+        this.size -= rangesSize(ranges);
+        await this.storage.write(key, data, range.from);
+        const newRanges = addRange(ranges, range);
+        this.size += rangesSize(newRanges);
+        await this.ranges.set(key, newRanges);
+    }
+    /** Ensures the size is less than hard limit by deleting the oldest keys. */
+    async ensureEvicted() {
+        const byTime = mapEntries(this.keyToLastAccessTime);
+        byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());
+        while (this.size > this.maxSize) {
+            const keyAndDate = byTime.pop(); // removes the oldest
+            if (!keyAndDate) {
+                break;
+            }
+            const [key, _] = keyAndDate;
+            const ranges = await this.getRanges(key);
+            this.size -= rangesSize(ranges);
+            this.rmKey(key);
+        }
+    }
+    /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */
+    async getRanges(key) {
+        try {
+            return await this.ranges.get(key);
+        }
+        catch (e) {
+            if (e instanceof CorruptedRangesError) {
+                // We need to reset a state of the cache and update current size,
+                // it's the only way to calculate the real size when one of the ranges were corrupted.
+                await this.rmKey(key);
+                await this.resetUnsafe();
+                return await this.ranges.get(key);
+            }
+            throw e;
+        }
+    }
+    /** Removes a key the state of the cache. The size should be updated. */
+    async rmKey(key) {
+        await this.storage.delete(key);
+        await this.ranges.delete(key);
+        this.keyToLastAccessTime.delete(key);
+    }
+}
+/** Acquires the lock and executes a callback. */
+async function withLock(lock, cb) {
+    try {
+        await lock.acquireAsync();
+        return await cb();
+    }
+    finally {
+        lock.release();
+    }
+}
+
+export { CorruptedRangesError, SparseCache, SparseCacheFsFile, SparseCacheFsRanges };
+//# sourceMappingURL=cache.js.map