@milaboratories/pl-drivers 1.12.8 → 1.12.10

package/dist/drivers/download_blob/download_blob.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob.js","names":["path","fs"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n  isNotFoundError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n  AnyLogHandle,\n  BlobDriver,\n  ContentHandler,\n  GetContentOptions,\n  LocalBlobHandle,\n  LocalBlobHandleAndSize,\n  ReadyLogHandle,\n  RemoteBlobHandle,\n  RemoteBlobHandleAndSize,\n  StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  makeResourceSnapshot,\n  treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n  isLocalBlobHandle,\n  newLocalHandle,\n  parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n  isRemoteBlobHandle,\n  newRemoteHandle,\n  parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n  /**\n   * A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /**\n   * A hard limit of the amount of sparse cache, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one.\n   *\n   * The sparse cache is used to store ranges of blobs.\n   * */\n  rangesCacheMaxSizeBytes: number;\n\n  /**\n   * Max number of concurrent downloads while calculating computable states\n   * derived from this driver\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. */\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n  /** Represents a unique key to the path of a blob as a map. */\n  private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadBlobTask>;\n  private rangesCache: SparseCache;\n\n  /** Downloads files and writes them to the local dir. */\n  private downloadQueue: TaskProcessor;\n\n  private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n  private idToLastLines: Map<string, LastLinesGetter> = new Map();\n  private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n  private readonly saveDir: string;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    private readonly clientLogs: ClientLogs,\n    saveDir: string,\n    private readonly rangesCacheDir: string,\n    private readonly signer: Signer,\n    private readonly ops: DownloadDriverOps,\n  ) {\n    this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n    const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n    const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n    this.rangesCache = new SparseCache(\n      this.logger,\n      this.ops.rangesCacheMaxSizeBytes,\n      fsRanges,\n      fsStorage,\n    );\n\n    this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n    this.saveDir = path.resolve(saveDir);\n  }\n\n  static async init(\n    logger: MiLogger,\n    clientDownload: ClientDownload,\n    clientLogs: ClientLogs,\n    saveDir: string,\n    rangesCacheDir: string,\n    signer: Signer,\n    ops: DownloadDriverOps,\n  ): Promise<DownloadDriver> {\n    const driver = new DownloadDriver(\n      logger,\n      clientDownload,\n      clientLogs,\n      saveDir,\n      rangesCacheDir,\n      signer,\n      ops,\n    );\n    await driver.rangesCache.reset();\n\n    return driver;\n  }\n\n  /** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx: ComputableCtx,\n  ): LocalBlobHandleAndSize | undefined;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n  ): ComputableStableDefined<LocalBlobHandleAndSize>;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {\n    if (ctx === undefined) {\n      return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n    }\n\n    const rInfo = treeEntryToResourceInfo(res, ctx);\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n    const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n    if (result == undefined) {\n      ctx.markUnstable(\"download blob is still undefined\");\n    }\n\n    return result;\n  }\n\n  private getDownloadedBlobNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    callerId: string,\n  ): LocalBlobHandleAndSize | undefined {\n    validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n    // We don't need to request files with wider limits,\n    // PFrame's engine does it disk-optimally by itself.\n\n    const task = this.getOrSetNewTask(rInfo, callerId);\n    task.attach(w, callerId);\n\n    const result = task.getBlob();\n    if (!result.done) {\n      return undefined;\n    }\n    if (result.result.ok) {\n      return result.result.value;\n    }\n    throw result.result.error;\n  }\n\n  private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n    const key = blobKey(rInfo.id);\n\n    const inMemoryTask = this.keyToDownload.get(key);\n    if (inMemoryTask) {\n      return inMemoryTask;\n    }\n\n    // schedule the blob downloading, then it'll be added to the cache.\n    const fPath = path.resolve(this.saveDir, key);\n\n    const newTask = new DownloadBlobTask(\n      this.logger,\n      this.clientDownload,\n      rInfo,\n      newLocalHandle(fPath, this.signer),\n      fPath,\n    );\n    this.keyToDownload.set(key, newTask);\n\n    this.downloadQueue.push({\n      fn: () => this.downloadBlob(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask;\n  }\n\n  private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n    await task.download();\n    const blob = task.getBlob();\n    if (blob.done && blob.result.ok) {\n      this.cache.addCache(task, callerId);\n    }\n  }\n\n  /** Gets on demand blob. */\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: undefined,\n    fromBytes?: number,\n    toBytes?: number,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx: ComputableCtx,\n    fromBytes?: number,\n    toBytes?: number,\n  ): RemoteBlobHandleAndSize;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): ComputableStableDefined<RemoteBlobHandleAndSize> | RemoteBlobHandleAndSize | undefined {\n    if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n    const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n      ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n      : res;\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n    // note that the watcher is not needed,\n    // the handler never changes.\n    const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n    return result;\n  }\n\n  private getOnDemandBlobNoCtx(\n    info: OnDemandBlobResourceSnapshot,\n    callerId: string,\n  ): RemoteBlobHandleAndSize {\n    validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n    let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n    if (blob === undefined) {\n      blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n      this.keyToOnDemand.set(blobKey(info.id), blob);\n    }\n\n    blob.attach(callerId);\n\n    return blob.getHandle();\n  }\n\n  /** Gets a path from a handle. */\n  public getLocalPath(handle: LocalBlobHandle): string {\n    const { path } = parseLocalHandle(handle, this.signer);\n    return path;\n  }\n\n  /** Gets a content of a blob by a handle. */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options?: GetContentOptions,\n  ): Promise<Uint8Array>;\n  /** @deprecated Use {@link getContent} with {@link GetContentOptions} instead */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    range?: RangeBytes,\n  ): Promise<Uint8Array>;\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    optionsOrRange?: GetContentOptions | RangeBytes,\n  ): Promise<Uint8Array> {\n    let options: GetContentOptions = {};\n    if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n      if (\"range\" in optionsOrRange) {\n        options = optionsOrRange;\n      } else {\n        const range = optionsOrRange as RangeBytes;\n        validateRangeBytes(range, `getContent`);\n        options = { range };\n      }\n    }\n\n    const request = () =>\n      this.withContent(handle, {\n        ...options,\n        handler: async (content) => {\n          const chunks: Uint8Array[] = [];\n          for await (const chunk of content) {\n            options.signal?.throwIfAborted();\n            chunks.push(chunk);\n          }\n          return Buffer.concat(chunks);\n        },\n      });\n\n    try {\n      return await request();\n    } catch (error) {\n      if (isOffByOneError(error)) {\n        return await request();\n      }\n      throw error;\n    }\n  }\n\n  /** Gets a content stream of a blob by a handle and calls handler with it. */\n  public async withContent<T>(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options: GetContentOptions & {\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    const { range, signal, handler } = options;\n\n    if (isLocalBlobHandle(handle)) {\n      return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n    }\n\n    if (isRemoteBlobHandle(handle)) {\n      const result = parseRemoteHandle(handle, this.signer);\n\n      const key = blobKey(result.info.id);\n      const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n      signal?.throwIfAborted();\n\n      if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n      return await this.clientDownload.withBlobContent(\n        result.info,\n        { signal },\n        options,\n        async (content, size) => {\n          const [handlerStream, cacheStream] = content.tee();\n\n          const handlerPromise = handler(handlerStream, size);\n          const _cachePromise = buffer(cacheStream)\n            .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n            .catch(() => {\n              // Ignore cache errors - they shouldn't affect the main handler result\n              // This prevents unhandled promise rejections when the stream fails\n            });\n\n          return await handlerPromise;\n        },\n      );\n    }\n\n    throw new Error(\"Malformed remote handle\");\n  }\n\n  /**\n   * Creates computable that will return blob content once it is downloaded.\n   * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n   */\n  public getComputableContent(\n    res: ResourceInfo | PlTreeEntry,\n    range?: RangeBytes,\n  ): ComputableStableDefined<Uint8Array> {\n    if (range) {\n      validateRangeBytes(range, `getComputableContent`);\n    }\n\n    return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n      postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n    }).withStableType();\n  }\n\n  /** Returns all logs and schedules a job that reads remain logs.\n   * Notifies when a new portion of the log appeared. */\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n    if (result == undefined)\n      ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, lines);\n      this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n  ): Computable<string | undefined>;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx,\n  ): string | undefined;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getProgressLogNoCtx(\n      ctx.watcher,\n      r as ResourceSnapshot,\n      patternToSearch,\n      callerId,\n    );\n    if (result === undefined)\n      ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n      this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns an Id of a smart object, that can read logs directly from\n   * the platform. */\n  public getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<AnyLogHandle>;\n  public getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n  public getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<AnyLogHandle> | AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    return this.getLogHandleNoCtx(r as ResourceSnapshot);\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n    validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n    return newLogHandle(false, rInfo);\n  }\n\n  public async lastLines(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number, // if 0n, then start from the end.\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.lastLines(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  public async readText(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.readText(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n    const task = this.keyToDownload.get(blobKey(rInfo.id));\n    if (task == undefined) {\n      return;\n    }\n\n    if (this.cache.existsFile(blobKey(rInfo.id))) {\n      const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n      await Promise.all(\n        toDelete.map(async (cachedFile) => {\n          await fsp.rm(cachedFile.path);\n\n          this.cache.removeCache(cachedFile);\n\n          this.removeTask(\n            mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n            `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted) {\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n      }\n    }\n  }\n\n  private removeTask(task: DownloadBlobTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n    this.keyToDownload.delete(pathToKey(task.path));\n    this.idToLastLines.delete(blobKey(task.rInfo.id));\n    this.idToProgressLog.delete(blobKey(task.rInfo.id));\n  }\n\n  private async releaseOnDemandBlob(blobId: ResourceId, callerId: string) {\n    const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n    if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    this.keyToDownload.forEach((task, key) => {\n      this.keyToDownload.delete(key);\n      task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n    });\n  }\n\n  async dispose(): Promise<void> {\n    await this.rangesCache.dispose();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Keeps a counter to the on demand handle. */\nclass OnDemandBlobHolder {\n  private readonly counter = new CallersCounter();\n\n  constructor(\n    private readonly size: number,\n    private readonly handle: RemoteBlobHandle,\n  ) {}\n\n  public getHandle(): RemoteBlobHandleAndSize {\n    return { handle: this.handle, size: this.size };\n  }\n\n  public attach(callerId: string) {\n    this.counter.inc(callerId);\n  }\n\n  public release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n}\n\nclass LastLinesGetter {\n  private updater: Updater;\n  private log: string | undefined;\n  private readonly change: ChangeSource = new ChangeSource();\n  private error: any | undefined = undefined;\n\n  constructor(\n    private readonly path: string,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {\n    this.updater = new Updater(async () => this.update());\n  }\n\n  getOrSchedule(w: Watcher): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    this.change.attachWatcher(w);\n\n    this.updater.schedule();\n\n    return {\n      log: this.log,\n      error: this.error,\n    };\n  }\n\n  async update(): Promise<void> {\n    try {\n      const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n      if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n      this.log = newLogs;\n    } catch (e: any) {\n      if (isNotFoundError(e)) {\n        // No resource\n        this.log = \"\";\n        this.error = e;\n        this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n        return;\n      }\n\n      throw e;\n    }\n  }\n}\n\n/** Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n  fPath: string,\n  nLines: number,\n  patternToSearch?: string,\n): Promise<string> {\n  let inStream: fs.ReadStream | undefined;\n  let rl: readline.Interface | undefined;\n\n  try {\n    inStream = fs.createReadStream(fPath);\n    rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n    const lines = new Denque();\n\n    for await (const line of rl) {\n      if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n      lines.push(line);\n      if (lines.length > nLines) {\n        lines.shift();\n      }\n    }\n\n    // last EOL is for keeping backward compat with platforma implementation.\n    return lines.toArray().join(os.EOL) + os.EOL;\n  } finally {\n    // Cleanup resources in finally block to ensure they're always cleaned up\n    try {\n      if (rl) {\n        rl.close();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error closing readline interface:\", cleanupError);\n    }\n\n    try {\n      if (inStream && !inStream.destroyed) {\n        inStream.destroy();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error destroying read stream:\", cleanupError);\n    }\n  }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith(\"Blob/\")) {\n    let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n    if (rType.name == \"Blob\")\n      message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n    throw new WrongResourceTypeError(message);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,AAAQ,gCAA+C,IAAI,KAAK;;;CAIhE,AAAQ;CACR,AAAQ;;CAGR,AAAQ;CAER,AAAQ,gCAAiD,IAAI,KAAK;CAElE,AAAQ,gCAA8C,IAAI,KAAK;CAC/D,AAAQ,kCAAgD,IAAI,KAAK;CAEjE,AAAiB;CAEjB,YACE,AAAiB,QACjB,AAAiB,gBACjB,AAAiB,YACjB,SACA,AAAiB,gBACjB,AAAiB,QACjB,AAAiB,KACjB;EAPiB;EACA;EACA;EAEA;EACA;EACA;AAEjB,OAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAI,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAI,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAI,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUA,OAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,AAAO,kBACL,KACA,KACqF;AACrF,MAAI,QAAQ,OACV,QAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,QAAQ,wBAAwB,KAAK,IAAI;EAE/C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,OACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,AAAQ,uBACN,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,AAAQ,gBAAgB,OAAyB,UAAoC;EACnF,MAAM,MAAM,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQA,OAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAI,iBAClB,KAAK,QACL,KAAK,gBACL,OACA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,AAAO,gBACL,KACA,KACwF;AACxF,MAAI,QAAQ,OAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,QAAsC,cAAc,IAAI,GAC1D,qBAAqB,KAAK,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,AAAQ,qBACN,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,QAAW;AACtB,UAAO,IAAI,mBAAmB,QAAQ,KAAK,EAAE,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,AAAO,aAAa,QAAiC;EACnD,MAAM,EAAE,SAAS,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,sBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;EAIvB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAI,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAGY;EACZ,MAAM,EAAE,OAAO,QAAQ,YAAY;AAEnC,MAAI,kBAAkB,OAAO,CAC3B,QAAO,MAAM,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAI,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAAS,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAM,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AACvF,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAM,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;IACvB,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,WAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,AAAO,qBACL,KACA,OACqC;AACrC,MAAI,MACF,oBAAmB,OAAO,uBAAuB;AAGnD,SAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,QACtE,CAAC,CAAC,gBAAgB;;CAcrB,AAAO,YACL,KACA,OACA,KACqD;AACrD,MAAI,OAAO,OAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,OACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,AAAQ,iBACN,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,OAAW,QAAO;EAE9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,QAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,AAAO,eACL,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,OACT,QAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,OACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,AAAQ,oBACN,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,OAAW,QAAO;EAC9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,QAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,AAAO,aACL,KACA,KACyC;AACzC,MAAI,OAAO,OAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,IAAI,wBAAwB,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,AAAQ,kBAAkB,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAO,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,OACV;AAGF,MAAI,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAM,IAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,WACH,OAAO,KAAK,eAAe,UAAU,WAAW,KAAK,CAAC,EACtD,YAAY,wBAAwB,WAAW,CAAC,oCACrB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,AAAQ,WAAW,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAO,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAAoB,UAAkB;AAEtE,MADgB,KAAK,cAAc,IAAI,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAO,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,AAAiB,UAAU,IAAI,gBAAgB;CAE/C,YACE,AAAiB,MACjB,AAAiB,QACjB;EAFiB;EACA;;CAGnB,AAAO,YAAqC;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,AAAO,OAAO,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,AAAO,QAAQ,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB,AAAQ;CACR,AAAQ;CACR,AAAiB,SAAuB,IAAI,cAAc;CAC1D,AAAQ,QAAyB;CAEjC,YACE,AAAiB,MACjB,AAAiB,OACjB,AAAiB,iBACjB;EAHiB;EACA;EACA;AAEjB,OAAK,UAAU,IAAI,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,OAAI,gBAAgB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAWC,KAAG,iBAAiB,MAAM;AACrC,OAAK,SAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAI,QAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,UAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAK,GAAG,IAAI,GAAG,GAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAI,uBAAuB,QAAQ"}
+{"version":3,"file":"download_blob.js","names":["path","fs"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n  isNotFoundError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n  AnyLogHandle,\n  BlobDriver,\n  ContentHandler,\n  GetContentOptions,\n  LocalBlobHandle,\n  LocalBlobHandleAndSize,\n  ReadyLogHandle,\n  RemoteBlobHandle,\n  RemoteBlobHandleAndSize,\n  StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  makeResourceSnapshot,\n  treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n  isLocalBlobHandle,\n  newLocalHandle,\n  parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n  isRemoteBlobHandle,\n  newRemoteHandle,\n  parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n  /**\n   * A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /**\n   * A hard limit of the amount of sparse cache, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one.\n   *\n   * The sparse cache is used to store ranges of blobs.\n   * */\n  rangesCacheMaxSizeBytes: number;\n\n  /**\n   * Max number of concurrent downloads while calculating computable states\n   * derived from this driver\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. */\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n  /** Represents a unique key to the path of a blob as a map. */\n  private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadBlobTask>;\n  private rangesCache: SparseCache;\n\n  /** Downloads files and writes them to the local dir. */\n  private downloadQueue: TaskProcessor;\n\n  private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n  private idToLastLines: Map<string, LastLinesGetter> = new Map();\n  private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n  private readonly saveDir: string;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    private readonly clientLogs: ClientLogs,\n    saveDir: string,\n    private readonly rangesCacheDir: string,\n    private readonly signer: Signer,\n    private readonly ops: DownloadDriverOps,\n  ) {\n    this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n    const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n    const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n    this.rangesCache = new SparseCache(\n      this.logger,\n      this.ops.rangesCacheMaxSizeBytes,\n      fsRanges,\n      fsStorage,\n    );\n\n    this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n    this.saveDir = path.resolve(saveDir);\n  }\n\n  static async init(\n    logger: MiLogger,\n    clientDownload: ClientDownload,\n    clientLogs: ClientLogs,\n    saveDir: string,\n    rangesCacheDir: string,\n    signer: Signer,\n    ops: DownloadDriverOps,\n  ): Promise<DownloadDriver> {\n    const driver = new DownloadDriver(\n      logger,\n      clientDownload,\n      clientLogs,\n      saveDir,\n      rangesCacheDir,\n      signer,\n      ops,\n    );\n    await driver.rangesCache.reset();\n\n    return driver;\n  }\n\n  /** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx: ComputableCtx,\n  ): LocalBlobHandleAndSize | undefined;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n  ): ComputableStableDefined<LocalBlobHandleAndSize>;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {\n    if (ctx === undefined) {\n      return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n    }\n\n    const rInfo = treeEntryToResourceInfo(res, ctx);\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n    const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n    if (result == undefined) {\n      ctx.markUnstable(\"download blob is still undefined\");\n    }\n\n    return result;\n  }\n\n  private getDownloadedBlobNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    callerId: string,\n  ): LocalBlobHandleAndSize | undefined {\n    validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n    // We don't need to request files with wider limits,\n    // PFrame's engine does it disk-optimally by itself.\n\n    const task = this.getOrSetNewTask(rInfo, callerId);\n    task.attach(w, callerId);\n\n    const result = task.getBlob();\n    if (!result.done) {\n      return undefined;\n    }\n    if (result.result.ok) {\n      return result.result.value;\n    }\n    throw result.result.error;\n  }\n\n  private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n    const key = blobKey(rInfo.id);\n\n    const inMemoryTask = this.keyToDownload.get(key);\n    if (inMemoryTask) {\n      return inMemoryTask;\n    }\n\n    // schedule the blob downloading, then it'll be added to the cache.\n    const fPath = path.resolve(this.saveDir, key);\n\n    const newTask = new DownloadBlobTask(\n      this.logger,\n      this.clientDownload,\n      rInfo,\n      newLocalHandle(fPath, this.signer),\n      fPath,\n    );\n    this.keyToDownload.set(key, newTask);\n\n    this.downloadQueue.push({\n      fn: () => this.downloadBlob(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask;\n  }\n\n  private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n    await task.download();\n    const blob = task.getBlob();\n    if (blob.done && blob.result.ok) {\n      this.cache.addCache(task, callerId);\n    }\n  }\n\n  /** Gets on demand blob. */\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: undefined,\n    fromBytes?: number,\n    toBytes?: number,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx: ComputableCtx,\n    fromBytes?: number,\n    toBytes?: number,\n  ): RemoteBlobHandleAndSize;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): ComputableStableDefined<RemoteBlobHandleAndSize> | RemoteBlobHandleAndSize | undefined {\n    if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n    const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n      ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n      : res;\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n    // note that the watcher is not needed,\n    // the handler never changes.\n    const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n    return result;\n  }\n\n  private getOnDemandBlobNoCtx(\n    info: OnDemandBlobResourceSnapshot,\n    callerId: string,\n  ): RemoteBlobHandleAndSize {\n    validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n    let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n    if (blob === undefined) {\n      blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n      this.keyToOnDemand.set(blobKey(info.id), blob);\n    }\n\n    blob.attach(callerId);\n\n    return blob.getHandle();\n  }\n\n  /** Gets a path from a handle. */\n  public getLocalPath(handle: LocalBlobHandle): string {\n    const { path } = parseLocalHandle(handle, this.signer);\n    return path;\n  }\n\n  /** Gets a content of a blob by a handle. */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options?: GetContentOptions,\n  ): Promise<Uint8Array>;\n  /** @deprecated Use {@link getContent} with {@link GetContentOptions} instead */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    range?: RangeBytes,\n  ): Promise<Uint8Array>;\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    optionsOrRange?: GetContentOptions | RangeBytes,\n  ): Promise<Uint8Array> {\n    let options: GetContentOptions = {};\n    if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n      if (\"range\" in optionsOrRange) {\n        options = optionsOrRange;\n      } else {\n        const range = optionsOrRange as RangeBytes;\n        validateRangeBytes(range, `getContent`);\n        options = { range };\n      }\n    }\n\n    const request = () =>\n      this.withContent(handle, {\n        ...options,\n        handler: async (content) => {\n          const chunks: Uint8Array[] = [];\n          for await (const chunk of content) {\n            options.signal?.throwIfAborted();\n            chunks.push(chunk);\n          }\n          return Buffer.concat(chunks);\n        },\n      });\n\n    try {\n      return await request();\n    } catch (error) {\n      if (isOffByOneError(error)) {\n        return await request();\n      }\n      throw error;\n    }\n  }\n\n  /** Gets a content stream of a blob by a handle and calls handler with it. */\n  public async withContent<T>(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options: GetContentOptions & {\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    const { range, signal, handler } = options;\n\n    if (isLocalBlobHandle(handle)) {\n      return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n    }\n\n    if (isRemoteBlobHandle(handle)) {\n      const result = parseRemoteHandle(handle, this.signer);\n\n      const key = blobKey(result.info.id);\n      const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n      signal?.throwIfAborted();\n\n      if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n      return await this.clientDownload.withBlobContent(\n        result.info,\n        { signal },\n        options,\n        async (content, size) => {\n          const [handlerStream, cacheStream] = content.tee();\n\n          const handlerPromise = handler(handlerStream, size);\n          const _cachePromise = buffer(cacheStream)\n            .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n            .catch(() => {\n              // Ignore cache errors - they shouldn't affect the main handler result\n              // This prevents unhandled promise rejections when the stream fails\n            });\n\n          return await handlerPromise;\n        },\n      );\n    }\n\n    throw new Error(\"Malformed remote handle\");\n  }\n\n  /**\n   * Creates computable that will return blob content once it is downloaded.\n   * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n   */\n  public getComputableContent(\n    res: ResourceInfo | PlTreeEntry,\n    range?: RangeBytes,\n  ): ComputableStableDefined<Uint8Array> {\n    if (range) {\n      validateRangeBytes(range, `getComputableContent`);\n    }\n\n    return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n      postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n    }).withStableType();\n  }\n\n  /** Returns all logs and schedules a job that reads remain logs.\n   * Notifies when a new portion of the log appeared. */\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n    if (result == undefined)\n      ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, lines);\n      this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n  ): Computable<string | undefined>;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx,\n  ): string | undefined;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getProgressLogNoCtx(\n      ctx.watcher,\n      r as ResourceSnapshot,\n      patternToSearch,\n      callerId,\n    );\n    if (result === undefined)\n      ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n      this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns an Id of a smart object, that can read logs directly from\n   * the platform. */\n  public getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<AnyLogHandle>;\n  public getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n  public getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<AnyLogHandle> | AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    return this.getLogHandleNoCtx(r as ResourceSnapshot);\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n    validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n    return newLogHandle(false, rInfo);\n  }\n\n  public async lastLines(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number, // if 0n, then start from the end.\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.lastLines(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  public async readText(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.readText(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n    const task = this.keyToDownload.get(blobKey(rInfo.id));\n    if (task == undefined) {\n      return;\n    }\n\n    if (this.cache.existsFile(blobKey(rInfo.id))) {\n      const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n      await Promise.all(\n        toDelete.map(async (cachedFile) => {\n          await fsp.rm(cachedFile.path);\n\n          this.cache.removeCache(cachedFile);\n\n          this.removeTask(\n            mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n            `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted) {\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n      }\n    }\n  }\n\n  private removeTask(task: DownloadBlobTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n    this.keyToDownload.delete(pathToKey(task.path));\n    this.idToLastLines.delete(blobKey(task.rInfo.id));\n    this.idToProgressLog.delete(blobKey(task.rInfo.id));\n  }\n\n  private async releaseOnDemandBlob(blobId: ResourceId, callerId: string) {\n    const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n    if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    this.keyToDownload.forEach((task, key) => {\n      this.keyToDownload.delete(key);\n      task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n    });\n  }\n\n  async dispose(): Promise<void> {\n    await this.rangesCache.dispose();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Keeps a counter to the on demand handle. */\nclass OnDemandBlobHolder {\n  private readonly counter = new CallersCounter();\n\n  constructor(\n    private readonly size: number,\n    private readonly handle: RemoteBlobHandle,\n  ) {}\n\n  public getHandle(): RemoteBlobHandleAndSize {\n    return { handle: this.handle, size: this.size };\n  }\n\n  public attach(callerId: string) {\n    this.counter.inc(callerId);\n  }\n\n  public release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n}\n\nclass LastLinesGetter {\n  private updater: Updater;\n  private log: string | undefined;\n  private readonly change: ChangeSource = new ChangeSource();\n  private error: any | undefined = undefined;\n\n  constructor(\n    private readonly path: string,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {\n    this.updater = new Updater(async () => this.update());\n  }\n\n  getOrSchedule(w: Watcher): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    this.change.attachWatcher(w);\n\n    this.updater.schedule();\n\n    return {\n      log: this.log,\n      error: this.error,\n    };\n  }\n\n  async update(): Promise<void> {\n    try {\n      const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n      if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n      this.log = newLogs;\n    } catch (e: any) {\n      if (isNotFoundError(e)) {\n        // No resource\n        this.log = \"\";\n        this.error = e;\n        this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n        return;\n      }\n\n      throw e;\n    }\n  }\n}\n\n/** Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n  fPath: string,\n  nLines: number,\n  patternToSearch?: string,\n): Promise<string> {\n  let inStream: fs.ReadStream | undefined;\n  let rl: readline.Interface | undefined;\n\n  try {\n    inStream = fs.createReadStream(fPath);\n    rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n    const lines = new Denque();\n\n    for await (const line of rl) {\n      if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n      lines.push(line);\n      if (lines.length > nLines) {\n        lines.shift();\n      }\n    }\n\n    // last EOL is for keeping backward compat with platforma implementation.\n    return lines.toArray().join(os.EOL) + os.EOL;\n  } finally {\n    // Cleanup resources in finally block to ensure they're always cleaned up\n    try {\n      if (rl) {\n        rl.close();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error closing readline interface:\", cleanupError);\n    }\n\n    try {\n      if (inStream && !inStream.destroyed) {\n        inStream.destroy();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error destroying read stream:\", cleanupError);\n    }\n  }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith(\"Blob/\")) {\n    let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n    if (rType.name == \"Blob\")\n      message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n    throw new WrongResourceTypeError(message);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,gCAAuD,IAAI,KAAK;;;CAIhE;CACA;;CAGA;CAEA,gCAAyD,IAAI,KAAK;CAElE,gCAAsD,IAAI,KAAK;CAC/D,kCAAwD,IAAI,KAAK;CAEjE;CAEA,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;AAPiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,aAAA;AAEA,OAAA,iBAAA;AACA,OAAA,SAAA;AACA,OAAA,MAAA;AAEjB,OAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAI,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAI,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAI,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUA,OAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,kBACE,KACA,KACqF;AACrF,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,QAAQ,wBAAwB,KAAK,IAAI;EAE/C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,uBACE,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAM,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQA,OAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAI,iBAClB,KAAK,QACL,KAAK,gBACL,OACA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,gBACE,KACA,KACwF;AACxF,MAAI,QAAQ,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,QAAsC,cAAc,IAAI,GAC1D,qBAAqB,KAAK,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,qBACE,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,KAAA,GAAW;AACtB,UAAO,IAAI,mBAAmB,QAAQ,KAAK,EAAE,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAAS,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,sBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;EAIvB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAI,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAGY;EACZ,MAAM,EAAE,OAAO,QAAQ,YAAY;AAEnC,MAAI,kBAAkB,OAAO,CAC3B,QAAO,MAAM,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAI,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAAS,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAM,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AACvF,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAM,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;IACvB,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,WAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,qBACE,KACA,OACqC;AACrC,MAAI,MACF,oBAAmB,OAAO,uBAAuB;AAGnD,SAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAA,GACtE,CAAC,CAAC,gBAAgB;;CAcrB,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAE9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,KAAA,EACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAC9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,aACE,KACA,KACyC;AACzC,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,IAAI,wBAAwB,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,kBAA0B,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAO,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,KAAA,EACV;AAGF,MAAI,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAM,IAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,WACH,OAAO,KAAK,eAAe,UAAU,WAAW,KAAK,CAAC,EACtD,YAAY,wBAAwB,WAAW,CAAC,oCACrB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,WAAmB,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAO,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAAoB,UAAkB;AAEtE,MADgB,KAAK,cAAc,IAAI,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAO,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,UAA2B,IAAI,gBAAgB;CAE/C,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,YAA4C;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,OAAc,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAe,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB;CACA;CACA,SAAwC,IAAI,cAAc;CAC1D,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;AAHiB,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;AAEjB,OAAK,UAAU,IAAI,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,OAAI,gBAAgB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAWC,KAAG,iBAAiB,MAAM;AACrC,OAAK,SAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAI,QAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAK,GAAG,IAAI,GAAG,GAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAI,uBAAuB,QAAQ"}

package/dist/drivers/download_blob/download_blob_task.cjs

@@ -1,6 +1,6 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_download_errors = require('../../helpers/download_errors.cjs');
-const require_download = require('../../clients/download.cjs');
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_download_errors = require("../../helpers/download_errors.cjs");
+const require_download = require("../../clients/download.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let node_fs_promises = require("node:fs/promises");
 node_fs_promises = require_runtime.__toESM(node_fs_promises);
@@ -11,7 +11,6 @@ let node_path = require("node:path");
 node_path = require_runtime.__toESM(node_path);
 let node_stream = require("node:stream");
 let _milaboratories_computable = require("@milaboratories/computable");
-
 //#region src/drivers/download_blob/download_blob_task.ts
 /** Downloads a blob and holds callers and watchers for the blob. */
 var DownloadBlobTask = class {
@@ -131,8 +130,8 @@ function getDownloadedBlobResponse(handle, size, error) {
 		}
 	};
 }
-
 //#endregion
 exports.DownloadBlobTask = DownloadBlobTask;
 exports.nonRecoverableError = nonRecoverableError;
+
 //# sourceMappingURL=download_blob_task.cjs.map

package/dist/drivers/download_blob/download_blob_task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob_task.cjs","names":["ChangeSource","CallersCounter","fsp","path","Writable","fs","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from \"@milaboratories/pl-model-common\";\nimport type { ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport type { ValueOrError, MiLogger } from \"@milaboratories/ts-helpers\";\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from \"@milaboratories/ts-helpers\";\nimport fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Writable } from \"node:stream\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { UnknownStorageError, WrongLocalFileUrl } from \"../../clients/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, ` +\n          `state: ${JSON.stringify(this.state)}, ` +\n          `error: ${JSON.stringify(e)}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, ` +\n          `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: \"wx\" }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      },\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n        done: true;\n        result: ValueOrError<LocalBlobHandleAndSize>;\n      } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" && (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\"))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = \"DownloadAborted\";\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error(\"No file or handle provided\") };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n};\n"],"mappings":";;;;;;;;;;;;;;;;AAqBA,IAAa,mBAAb,MAA8B;CAC5B,AAAS,SAAS,IAAIA,yCAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD,AAAgB,UAAU,IAAIC,2CAAgB;CAC9C,AAAQ;CACR,AAAQ,OAAO;CACf,AAAO,OAAO;CACd,AAAQ,QAAuB,EAAE;CAEjC,YACE,AAAiB,QACjB,AAAiB,gBACjB,AAAS,OACT,AAAiB,QACjB,AAAS,MACT;EALiB;EACA;EACR;EACQ;EACR;;;CAIX,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,OAAO,KAAK;GACZ,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,OAAO,KAAK;GACb;;CAGH,AAAO,OAAO,GAAY,UAAkB;AAC1C,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAa,WAAW;AACtB,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,kBAAkB;AAC1C,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,0DAA2B,KAAK,MAAM,GAAG,CAAC,oBAAoB;WAC/E,GAAQ;AACf,QAAK,OAAO,MACV,+DAAgC,KAAK,MAAM,CAAC,2BAChC,KAAK,UAAU,KAAK,MAAM,CAAC,WAC3B,KAAK,UAAU,EAAE,GAC9B;AACD,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,0DAA2B,KAAK,MAAM,GAAG,CAAC,kBAAkB;AAEpF,UAAMC,iBAAI,GAAG,KAAK,MAAM,EAAE,OAAO,MAAM,CAAC;;AAG1C,SAAM;;;CAIV,MAAc,mBAAmB;AAC/B,OAAK,UAAU,OAAO,gBAAgB;AAEtC,OAAK,QAAQ,EAAE;AACf,OAAK,MAAM,WAAW,KAAK;AAC3B,wDAAsBC,UAAK,QAAQ,KAAK,MAAM,SAAS,CAAC;AACxD,OAAK,UAAU,OAAO,gBAAgB;AACtC,OAAK,MAAM,YAAY;EAEvB,MAAM,gBAAgB,iDAAiB,KAAK,MAAM,SAAS;AAC3D,OAAK,UAAU,OAAO,gBAAgB;AACtC,MAAI,eAAe;AACjB,QAAK,MAAM,aAAa;AACxB,QAAK,OAAO,KACV,+DAAgC,KAAK,MAAM,CAAC,iCACjC,KAAK,MAAM,WACvB;GACD,MAAM,OAAO,MAAMD,iBAAI,KAAK,KAAK,MAAM,SAAS;AAChD,QAAK,UAAU,OAAO,gBAAgB;AACtC,QAAK,MAAM,WAAW,KAAK;AAE3B,UAAO,KAAK,MAAM;;AAsBpB,SAnBiB,MAAM,KAAK,eAAe,gBACzC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,KAAK,UAAU,QAAQ,EACjC,OAAO,SAAS,SAAS;AACvB,QAAK,MAAM,WAAW;AACtB,QAAK,MAAM,aAAa;AAExB,8DAA2B,KAAK,QAAQ,KAAK,MAAM,UAAW,OAAO,UAAkB;IACrF,MAAM,IAAIE,qBAAS,MAAMC,gBAAG,kBAAkB,OAAO,EAAE,OAAO,MAAM,CAAC,CAAC;AACtE,UAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,KAAK,UAAU,QAAQ,CAAC;AAC1D,SAAK,MAAM,cAAc;KACzB;AAEF,QAAK,MAAM,OAAO;AAClB,UAAO;IAEV;;CAKH,AAAO,MAAM,QAAgB;AAC3B,OAAK,UAAU,MAAM,IAAI,gBAAgB,OAAO,CAAC;;CAGnD,AAAO,UAKD;AACJ,MAAI,CAAC,KAAK,KAAM,QAAO,EAAE,MAAM,OAAO;AAEtC,SAAO;GACL,MAAM,KAAK;GACX,QAAQ,0BAA0B,KAAK,QAAQ,KAAK,MAAM,KAAK,MAAM;GACtE;;CAGH,AAAQ,QAAQ,WAAmB;AACjC,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,AAAQ,SAAS,GAAY;AAC3B,OAAK,OAAO;AACZ,OAAK,QAAQ;;;AAIjB,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,mBACbC,kDAA0B,EAAE,IAC5B,aAAaC,wCACb,aAAaC,sCAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eAAe,EAAE,QAAQ,eAAe,EAAE,QAAQ;;;;AAMjE,IAAM,kBAAN,cAA8B,MAAM;CAClC,OAAO;;AAGT,SAAgB,0BACd,QACA,MACA,OACsC;AACtC,KAAI,MACF,QAAO;EAAE,IAAI;EAAO;EAAO;AAG7B,KAAI,CAAC,OACH,QAAO;EAAE,IAAI;EAAO,uBAAO,IAAI,MAAM,6BAA6B;EAAE;AAGtE,QAAO;EACL,IAAI;EACJ,OAAO;GACL;GACA;GACD;EACF"}
+{"version":3,"file":"download_blob_task.cjs","names":["ChangeSource","CallersCounter","fsp","path","Writable","fs","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from \"@milaboratories/pl-model-common\";\nimport type { ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport type { ValueOrError, MiLogger } from \"@milaboratories/ts-helpers\";\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from \"@milaboratories/ts-helpers\";\nimport fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Writable } from \"node:stream\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { UnknownStorageError, WrongLocalFileUrl } from \"../../clients/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, ` +\n          `state: ${JSON.stringify(this.state)}, ` +\n          `error: ${JSON.stringify(e)}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, ` +\n          `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: \"wx\" }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      },\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n        done: true;\n        result: ValueOrError<LocalBlobHandleAndSize>;\n      } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" && (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\"))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = \"DownloadAborted\";\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error(\"No file or handle provided\") };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n};\n"],"mappings":";;;;;;;;;;;;;;;AAqBA,IAAa,mBAAb,MAA8B;CAC5B,SAAkB,IAAIA,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD,UAA0B,IAAIC,2BAAAA,gBAAgB;CAC9C;CACA,OAAe;CACf,OAAc;CACd,QAA+B,EAAE;CAEjC,YACE,QACA,gBACA,OACA,QACA,MACA;AALiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACR,OAAA,QAAA;AACQ,OAAA,SAAA;AACR,OAAA,OAAA;;;CAIX,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,OAAO,KAAK;GACZ,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,OAAO,KAAK;GACb;;CAGH,OAAc,GAAY,UAAkB;AAC1C,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAa,WAAW;AACtB,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,kBAAkB;AAC1C,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,SAAA,GAAA,0BAAA,oBAA2B,KAAK,MAAM,GAAG,CAAC,oBAAoB;WAC/E,GAAQ;AACf,QAAK,OAAO,MACV,SAAA,GAAA,0BAAA,yBAAgC,KAAK,MAAM,CAAC,2BAChC,KAAK,UAAU,KAAK,MAAM,CAAC,WAC3B,KAAK,UAAU,EAAE,GAC9B;AACD,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,SAAA,GAAA,0BAAA,oBAA2B,KAAK,MAAM,GAAG,CAAC,kBAAkB;AAEpF,UAAMC,iBAAI,GAAG,KAAK,MAAM,EAAE,OAAO,MAAM,CAAC;;AAG1C,SAAM;;;CAIV,MAAc,mBAAmB;AAC/B,OAAK,UAAU,OAAO,gBAAgB;AAEtC,OAAK,QAAQ,EAAE;AACf,OAAK,MAAM,WAAW,KAAK;AAC3B,SAAA,GAAA,2BAAA,iBAAsBC,UAAK,QAAQ,KAAK,MAAM,SAAS,CAAC;AACxD,OAAK,UAAU,OAAO,gBAAgB;AACtC,OAAK,MAAM,YAAY;EAEvB,MAAM,gBAAgB,OAAA,GAAA,2BAAA,YAAiB,KAAK,MAAM,SAAS;AAC3D,OAAK,UAAU,OAAO,gBAAgB;AACtC,MAAI,eAAe;AACjB,QAAK,MAAM,aAAa;AACxB,QAAK,OAAO,KACV,SAAA,GAAA,0BAAA,yBAAgC,KAAK,MAAM,CAAC,iCACjC,KAAK,MAAM,WACvB;GACD,MAAM,OAAO,MAAMD,iBAAI,KAAK,KAAK,MAAM,SAAS;AAChD,QAAK,UAAU,OAAO,gBAAgB;AACtC,QAAK,MAAM,WAAW,KAAK;AAE3B,UAAO,KAAK,MAAM;;AAsBpB,SAnBiB,MAAM,KAAK,eAAe,gBACzC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,KAAK,UAAU,QAAQ,EACjC,OAAO,SAAS,SAAS;AACvB,QAAK,MAAM,WAAW;AACtB,QAAK,MAAM,aAAa;AAExB,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,UAAW,OAAO,UAAkB;IACrF,MAAM,IAAIE,YAAAA,SAAS,MAAMC,QAAAA,QAAG,kBAAkB,OAAO,EAAE,OAAO,MAAM,CAAC,CAAC;AACtE,UAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,KAAK,UAAU,QAAQ,CAAC;AAC1D,SAAK,MAAM,cAAc;KACzB;AAEF,QAAK,MAAM,OAAO;AAClB,UAAO;IAEV;;CAKH,MAAa,QAAgB;AAC3B,OAAK,UAAU,MAAM,IAAI,gBAAgB,OAAO,CAAC;;CAGnD,UAKM;AACJ,MAAI,CAAC,KAAK,KAAM,QAAO,EAAE,MAAM,OAAO;AAEtC,SAAO;GACL,MAAM,KAAK;GACX,QAAQ,0BAA0B,KAAK,QAAQ,KAAK,MAAM,KAAK,MAAM;GACtE;;CAGH,QAAgB,WAAmB;AACjC,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAY;AAC3B,OAAK,OAAO;AACZ,OAAK,QAAQ;;;AAIjB,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,mBACbC,wBAAAA,0BAA0B,EAAE,IAC5B,aAAaC,iBAAAA,uBACb,aAAaC,iBAAAA,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eAAe,EAAE,QAAQ,eAAe,EAAE,QAAQ;;;;AAMjE,IAAM,kBAAN,cAA8B,MAAM;CAClC,OAAO;;AAGT,SAAgB,0BACd,QACA,MACA,OACsC;AACtC,KAAI,MACF,QAAO;EAAE,IAAI;EAAO;EAAO;AAG7B,KAAI,CAAC,OACH,QAAO;EAAE,IAAI;EAAO,uBAAO,IAAI,MAAM,6BAA6B;EAAE;AAGtE,QAAO;EACL,IAAI;EACJ,OAAO;GACL;GACA;GACD;EACF"}

package/dist/drivers/download_blob/download_blob_task.js

@@ -7,7 +7,6 @@ import fs from "node:fs";
 import * as path$1 from "node:path";
 import { Writable } from "node:stream";
 import { ChangeSource } from "@milaboratories/computable";
-
 //#region src/drivers/download_blob/download_blob_task.ts
 /** Downloads a blob and holds callers and watchers for the blob. */
 var DownloadBlobTask = class {
@@ -127,7 +126,7 @@ function getDownloadedBlobResponse(handle, size, error) {
 		}
 	};
 }
-
 //#endregion
 export { DownloadBlobTask, nonRecoverableError };
+
 //# sourceMappingURL=download_blob_task.js.map

package/dist/drivers/download_blob/download_blob_task.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob_task.js","names":["path"],"sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from \"@milaboratories/pl-model-common\";\nimport type { ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport type { ValueOrError, MiLogger } from \"@milaboratories/ts-helpers\";\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from \"@milaboratories/ts-helpers\";\nimport fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Writable } from \"node:stream\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { UnknownStorageError, WrongLocalFileUrl } from \"../../clients/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, ` +\n          `state: ${JSON.stringify(this.state)}, ` +\n          `error: ${JSON.stringify(e)}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, ` +\n          `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: \"wx\" }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      },\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n        done: true;\n        result: ValueOrError<LocalBlobHandleAndSize>;\n      } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" && (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\"))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = \"DownloadAborted\";\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error(\"No file or handle provided\") };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n};\n"],"mappings":";;;;;;;;;;;;AAqBA,IAAa,mBAAb,MAA8B;CAC5B,AAAS,SAAS,IAAI,cAAc;CACpC,AAAiB,YAAY,IAAI,iBAAiB;CAClD,AAAgB,UAAU,IAAI,gBAAgB;CAC9C,AAAQ;CACR,AAAQ,OAAO;CACf,AAAO,OAAO;CACd,AAAQ,QAAuB,EAAE;CAEjC,YACE,AAAiB,QACjB,AAAiB,gBACjB,AAAS,OACT,AAAiB,QACjB,AAAS,MACT;EALiB;EACA;EACR;EACQ;EACR;;;CAIX,AAAO,OAAO;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,OAAO,KAAK;GACZ,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,OAAO,KAAK;GACb;;CAGH,AAAO,OAAO,GAAY,UAAkB;AAC1C,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAa,WAAW;AACtB,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,kBAAkB;AAC1C,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,oBAAoB;WAC/E,GAAQ;AACf,QAAK,OAAO,MACV,QAAQ,wBAAwB,KAAK,MAAM,CAAC,2BAChC,KAAK,UAAU,KAAK,MAAM,CAAC,WAC3B,KAAK,UAAU,EAAE,GAC9B;AACD,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,kBAAkB;AAEpF,UAAM,IAAI,GAAG,KAAK,MAAM,EAAE,OAAO,MAAM,CAAC;;AAG1C,SAAM;;;CAIV,MAAc,mBAAmB;AAC/B,OAAK,UAAU,OAAO,gBAAgB;AAEtC,OAAK,QAAQ,EAAE;AACf,OAAK,MAAM,WAAW,KAAK;AAC3B,QAAM,gBAAgBA,OAAK,QAAQ,KAAK,MAAM,SAAS,CAAC;AACxD,OAAK,UAAU,OAAO,gBAAgB;AACtC,OAAK,MAAM,YAAY;EAEvB,MAAM,gBAAgB,MAAM,WAAW,KAAK,MAAM,SAAS;AAC3D,OAAK,UAAU,OAAO,gBAAgB;AACtC,MAAI,eAAe;AACjB,QAAK,MAAM,aAAa;AACxB,QAAK,OAAO,KACV,QAAQ,wBAAwB,KAAK,MAAM,CAAC,iCACjC,KAAK,MAAM,WACvB;GACD,MAAM,OAAO,MAAM,IAAI,KAAK,KAAK,MAAM,SAAS;AAChD,QAAK,UAAU,OAAO,gBAAgB;AACtC,QAAK,MAAM,WAAW,KAAK;AAE3B,UAAO,KAAK,MAAM;;AAsBpB,SAnBiB,MAAM,KAAK,eAAe,gBACzC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,KAAK,UAAU,QAAQ,EACjC,OAAO,SAAS,SAAS;AACvB,QAAK,MAAM,WAAW;AACtB,QAAK,MAAM,aAAa;AAExB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,UAAW,OAAO,UAAkB;IACrF,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,OAAO,EAAE,OAAO,MAAM,CAAC,CAAC;AACtE,UAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,KAAK,UAAU,QAAQ,CAAC;AAC1D,SAAK,MAAM,cAAc;KACzB;AAEF,QAAK,MAAM,OAAO;AAClB,UAAO;IAEV;;CAKH,AAAO,MAAM,QAAgB;AAC3B,OAAK,UAAU,MAAM,IAAI,gBAAgB,OAAO,CAAC;;CAGnD,AAAO,UAKD;AACJ,MAAI,CAAC,KAAK,KAAM,QAAO,EAAE,MAAM,OAAO;AAEtC,SAAO;GACL,MAAM,KAAK;GACX,QAAQ,0BAA0B,KAAK,QAAQ,KAAK,MAAM,KAAK,MAAM;GACtE;;CAGH,AAAQ,QAAQ,WAAmB;AACjC,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,AAAQ,SAAS,GAAY;AAC3B,OAAK,OAAO;AACZ,OAAK,QAAQ;;;AAIjB,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,mBACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eAAe,EAAE,QAAQ,eAAe,EAAE,QAAQ;;;;AAMjE,IAAM,kBAAN,cAA8B,MAAM;CAClC,OAAO;;AAGT,SAAgB,0BACd,QACA,MACA,OACsC;AACtC,KAAI,MACF,QAAO;EAAE,IAAI;EAAO;EAAO;AAG7B,KAAI,CAAC,OACH,QAAO;EAAE,IAAI;EAAO,uBAAO,IAAI,MAAM,6BAA6B;EAAE;AAGtE,QAAO;EACL,IAAI;EACJ,OAAO;GACL;GACA;GACD;EACF"}
+{"version":3,"file":"download_blob_task.js","names":["path"],"sources":["../../../src/drivers/download_blob/download_blob_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { LocalBlobHandle, LocalBlobHandleAndSize } from \"@milaboratories/pl-model-common\";\nimport type { ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport type { ValueOrError, MiLogger } from \"@milaboratories/ts-helpers\";\nimport {\n  ensureDirExists,\n  fileExists,\n  createPathAtomically,\n  CallersCounter,\n} from \"@milaboratories/ts-helpers\";\nimport fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Writable } from \"node:stream\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { UnknownStorageError, WrongLocalFileUrl } from \"../../clients/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\n\n/** Downloads a blob and holds callers and watchers for the blob. */\nexport class DownloadBlobTask {\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  public readonly counter = new CallersCounter();\n  private error: unknown | undefined;\n  private done = false;\n  public size = 0;\n  private state: DownloadState = {};\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    readonly rInfo: ResourceSnapshot,\n    private readonly handle: LocalBlobHandle,\n    readonly path: string,\n  ) {}\n\n  /** Returns a simple object that describes this task for debugging purposes. */\n  public info() {\n    return {\n      rInfo: this.rInfo,\n      fPath: this.path,\n      done: this.done,\n      error: this.error,\n      state: this.state,\n    };\n  }\n\n  public attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  public async download() {\n    try {\n      const size = await this.ensureDownloaded();\n      this.setDone(size);\n      this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download finished`);\n    } catch (e: any) {\n      this.logger.error(\n        `blob ${stringifyWithResourceId(this.rInfo)} download failed, ` +\n          `state: ${JSON.stringify(this.state)}, ` +\n          `error: ${JSON.stringify(e)}`,\n      );\n      if (nonRecoverableError(e)) {\n        this.setError(e);\n        this.change.markChanged(`blob ${resourceIdToString(this.rInfo.id)} download failed`);\n        // Just in case we were half-way extracting an archive.\n        await fsp.rm(this.path, { force: true });\n      }\n\n      throw e;\n    }\n  }\n\n  private async ensureDownloaded() {\n    this.signalCtl.signal.throwIfAborted();\n\n    this.state = {};\n    this.state.filePath = this.path;\n    await ensureDirExists(path.dirname(this.state.filePath));\n    this.signalCtl.signal.throwIfAborted();\n    this.state.dirExists = true;\n\n    const alreadyExists = await fileExists(this.state.filePath);\n    this.signalCtl.signal.throwIfAborted();\n    if (alreadyExists) {\n      this.state.fileExists = true;\n      this.logger.info(\n        `blob ${stringifyWithResourceId(this.rInfo)} was already downloaded, ` +\n          `path: ${this.state.filePath}`,\n      );\n      const stat = await fsp.stat(this.state.filePath);\n      this.signalCtl.signal.throwIfAborted();\n      this.state.fileSize = stat.size;\n\n      return this.state.fileSize;\n    }\n\n    const fileSize = await this.clientDownload.withBlobContent(\n      this.rInfo,\n      {},\n      { signal: this.signalCtl.signal },\n      async (content, size) => {\n        this.state.fileSize = size;\n        this.state.downloaded = true;\n\n        await createPathAtomically(this.logger, this.state.filePath!, async (fPath: string) => {\n          const f = Writable.toWeb(fs.createWriteStream(fPath, { flags: \"wx\" }));\n          await content.pipeTo(f, { signal: this.signalCtl.signal });\n          this.state.tempWritten = true;\n        });\n\n        this.state.done = true;\n        return size;\n      },\n    );\n\n    return fileSize;\n  }\n\n  public abort(reason: string) {\n    this.signalCtl.abort(new DownloadAborted(reason));\n  }\n\n  public getBlob():\n    | { done: false }\n    | {\n        done: true;\n        result: ValueOrError<LocalBlobHandleAndSize>;\n      } {\n    if (!this.done) return { done: false };\n\n    return {\n      done: this.done,\n      result: getDownloadedBlobResponse(this.handle, this.size, this.error),\n    };\n  }\n\n  private setDone(sizeBytes: number) {\n    this.done = true;\n    this.size = sizeBytes;\n  }\n\n  private setError(e: unknown) {\n    this.done = true;\n    this.error = e;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof DownloadAborted ||\n    isDownloadNetworkError400(e) ||\n    e instanceof UnknownStorageError ||\n    e instanceof WrongLocalFileUrl ||\n    // file that we downloads from was moved or deleted.\n    e?.code == \"ENOENT\" ||\n    // A resource was deleted.\n    (e.name == \"RpcError\" && (e.code == \"NOT_FOUND\" || e.code == \"ABORTED\"))\n  );\n}\n\n/** The downloading task was aborted by a signal.\n * It may happen when the computable is done, for example. */\nclass DownloadAborted extends Error {\n  name = \"DownloadAborted\";\n}\n\nexport function getDownloadedBlobResponse(\n  handle: LocalBlobHandle | undefined,\n  size: number,\n  error?: unknown,\n): ValueOrError<LocalBlobHandleAndSize> {\n  if (error) {\n    return { ok: false, error };\n  }\n\n  if (!handle) {\n    return { ok: false, error: new Error(\"No file or handle provided\") };\n  }\n\n  return {\n    ok: true,\n    value: {\n      handle,\n      size,\n    },\n  };\n}\n\ntype DownloadState = {\n  filePath?: string;\n  dirExists?: boolean;\n  fileExists?: boolean;\n  fileSize?: number;\n  downloaded?: boolean;\n  tempWritten?: boolean;\n  done?: boolean;\n};\n"],"mappings":";;;;;;;;;;;AAqBA,IAAa,mBAAb,MAA8B;CAC5B,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD,UAA0B,IAAI,gBAAgB;CAC9C;CACA,OAAe;CACf,OAAc;CACd,QAA+B,EAAE;CAEjC,YACE,QACA,gBACA,OACA,QACA,MACA;AALiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACR,OAAA,QAAA;AACQ,OAAA,SAAA;AACR,OAAA,OAAA;;;CAIX,OAAc;AACZ,SAAO;GACL,OAAO,KAAK;GACZ,OAAO,KAAK;GACZ,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,OAAO,KAAK;GACb;;CAGH,OAAc,GAAY,UAAkB;AAC1C,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAa,WAAW;AACtB,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,kBAAkB;AAC1C,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,oBAAoB;WAC/E,GAAQ;AACf,QAAK,OAAO,MACV,QAAQ,wBAAwB,KAAK,MAAM,CAAC,2BAChC,KAAK,UAAU,KAAK,MAAM,CAAC,WAC3B,KAAK,UAAU,EAAE,GAC9B;AACD,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,kBAAkB;AAEpF,UAAM,IAAI,GAAG,KAAK,MAAM,EAAE,OAAO,MAAM,CAAC;;AAG1C,SAAM;;;CAIV,MAAc,mBAAmB;AAC/B,OAAK,UAAU,OAAO,gBAAgB;AAEtC,OAAK,QAAQ,EAAE;AACf,OAAK,MAAM,WAAW,KAAK;AAC3B,QAAM,gBAAgBA,OAAK,QAAQ,KAAK,MAAM,SAAS,CAAC;AACxD,OAAK,UAAU,OAAO,gBAAgB;AACtC,OAAK,MAAM,YAAY;EAEvB,MAAM,gBAAgB,MAAM,WAAW,KAAK,MAAM,SAAS;AAC3D,OAAK,UAAU,OAAO,gBAAgB;AACtC,MAAI,eAAe;AACjB,QAAK,MAAM,aAAa;AACxB,QAAK,OAAO,KACV,QAAQ,wBAAwB,KAAK,MAAM,CAAC,iCACjC,KAAK,MAAM,WACvB;GACD,MAAM,OAAO,MAAM,IAAI,KAAK,KAAK,MAAM,SAAS;AAChD,QAAK,UAAU,OAAO,gBAAgB;AACtC,QAAK,MAAM,WAAW,KAAK;AAE3B,UAAO,KAAK,MAAM;;AAsBpB,SAnBiB,MAAM,KAAK,eAAe,gBACzC,KAAK,OACL,EAAE,EACF,EAAE,QAAQ,KAAK,UAAU,QAAQ,EACjC,OAAO,SAAS,SAAS;AACvB,QAAK,MAAM,WAAW;AACtB,QAAK,MAAM,aAAa;AAExB,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,UAAW,OAAO,UAAkB;IACrF,MAAM,IAAI,SAAS,MAAM,GAAG,kBAAkB,OAAO,EAAE,OAAO,MAAM,CAAC,CAAC;AACtE,UAAM,QAAQ,OAAO,GAAG,EAAE,QAAQ,KAAK,UAAU,QAAQ,CAAC;AAC1D,SAAK,MAAM,cAAc;KACzB;AAEF,QAAK,MAAM,OAAO;AAClB,UAAO;IAEV;;CAKH,MAAa,QAAgB;AAC3B,OAAK,UAAU,MAAM,IAAI,gBAAgB,OAAO,CAAC;;CAGnD,UAKM;AACJ,MAAI,CAAC,KAAK,KAAM,QAAO,EAAE,MAAM,OAAO;AAEtC,SAAO;GACL,MAAM,KAAK;GACX,QAAQ,0BAA0B,KAAK,QAAQ,KAAK,MAAM,KAAK,MAAM;GACtE;;CAGH,QAAgB,WAAmB;AACjC,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAY;AAC3B,OAAK,OAAO;AACZ,OAAK,QAAQ;;;AAIjB,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,mBACb,0BAA0B,EAAE,IAC5B,aAAa,uBACb,aAAa,qBAEb,GAAG,QAAQ,YAEV,EAAE,QAAQ,eAAe,EAAE,QAAQ,eAAe,EAAE,QAAQ;;;;AAMjE,IAAM,kBAAN,cAA8B,MAAM;CAClC,OAAO;;AAGT,SAAgB,0BACd,QACA,MACA,OACsC;AACtC,KAAI,MACF,QAAO;EAAE,IAAI;EAAO;EAAO;AAG7B,KAAI,CAAC,OACH,QAAO;EAAE,IAAI;EAAO,uBAAO,IAAI,MAAM,6BAA6B;EAAE;AAGtE,QAAO;EACL,IAAI;EACJ,OAAO;GACL;GACA;GACD;EACF"}

package/dist/drivers/download_blob/sparse_cache/cache.cjs

@@ -1,12 +1,11 @@
-const require_runtime = require('../../../_virtual/_rolldown/runtime.cjs');
-const require_ranges = require('./ranges.cjs');
-const require_file = require('./file.cjs');
+const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
+const require_ranges = require("./ranges.cjs");
+const require_file = require("./file.cjs");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_helpers = require("@milaboratories/helpers");
 let node_fs = require("node:fs");
 let node_path = require("node:path");
 node_path = require_runtime.__toESM(node_path);
-
 //#region src/drivers/download_blob/sparse_cache/cache.ts
 /** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */
 var CorruptedRangesError = class extends Error {
@@ -170,10 +169,10 @@ async function withLock(lock, cb) {
 		lock.release();
 	}
 }
-
 //#endregion
 exports.CorruptedRangesError = CorruptedRangesError;
 exports.SparseCache = SparseCache;
 exports.SparseCacheFsFile = SparseCacheFsFile;
 exports.SparseCacheFsRanges = SparseCacheFsRanges;
+
 //# sourceMappingURL=cache.cjs.map

package/dist/drivers/download_blob/sparse_cache/cache.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"cache.cjs","names":["path","rangesFileName","readRangesFile","writeRangesFile","fs","writeToSparseFile","functions","rangesSize","doesRangeExist","addRange"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;;;AAWT,IAAa,sBAAb,MAA8D;CAC5D,YACE,AAAiB,QACjB,AAAiB,UACjB;EAFiB;EACA;;CAGnB,AAAQ,MAAM,KAAqB;AACjC,SAAOA,kBAAK,KAAK,KAAK,UAAUC,8BAAe,IAAI,CAAC;;CAGtD,MAAM,IAAI,KAA8B;AACtC,SAAO,MAAMC,8BAAe,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;;CAG3D,MAAM,IAAI,KAAa,QAAgB;AACrC,SAAO,MAAMC,+BAAgB,KAAK,QAAQ,KAAK,MAAM,IAAI,EAAE,OAAO;;CAGpE,MAAM,OAAO,KAAa;AACxB,QAAMC,iBAAG,GAAG,KAAK,MAAM,IAAI,CAAC;;;;AAchC,IAAa,oBAAb,MAA4D;CAC1D,AAAiB,SAAS;CAE1B,YACE,AAAiB,QACjB,AAAiB,UACjB;EAFiB;EACA;;CAGnB,MAAM,MAAyB;AAC7B,wDAAsB,KAAK,SAAS;AAEpC,UADc,MAAMA,iBAAG,QAAQ,KAAK,SAAS,EAChC,QAAQ,MAAM,EAAE,SAAS,KAAK,OAAO,CAAC;;CAGrD,MAAM,OAAO,KAA+B;AAC1C,SAAO,iDAAiB,KAAK,KAAK,IAAI,CAAC;;CAGzC,KAAK,KAAqB;AACxB,SAAOJ,kBAAK,KAAK,KAAK,UAAU,MAAM,KAAK,OAAO;;CAGpD,MAAM,MAAM,KAAa,MAAkB,MAA6B;AACtE,wDAAsB,KAAK,SAAS;AACpC,QAAMK,+BAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,IAAI,EAAE,MAAM,KAAK;;CAGpF,MAAM,OAAO,KAA4B;AACvC,QAAMD,iBAAG,GAAG,KAAK,KAAK,IAAI,CAAC;;;;AAK/B,IAAa,cAAb,MAAoD;;;CAIlD,AAAQ,OAAO,IAAIE,kCAAU,WAAW;CAExC,AAAO,sCAAsB,IAAI,KAAmB;CACpD,AAAO,OAAO;CAEd,YACE,AAAgB,QAEhB,AAAgB,SAChB,AAAgB,QAChB,AAAgB,SAChB;EALgB;EAEA;EACA;EACA;;;;CAKlB,MAAM,QAAQ;AACZ,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,aAAa;IACxB;;;;CAKJ,MAAM,IAAI,KAAa,OAAgD;AACrE,SAAO,MAAM,SAAS,KAAK,MAAM,YAAY;AAC3C,UAAO,MAAM,KAAK,UAAU,KAAK,MAAM;IACvC;;;;CAKJ,MAAM,IAAI,KAAa,OAAmB,MAAiC;AACzE,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,UAAU,KAAK,OAAO,KAAK;IACtC;;CAGJ,MAAc,cAAc;AAC1B,OAAK,OAAO;AACZ,OAAK,sCAAsB,IAAI,KAAmB;EAElD,MAAM,sBAAM,IAAI,MAAM;AAItB,OAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,EAAE;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,IAAI;AACzC,QAAK,QAAQC,0BAAW,OAAO;AAC/B,QAAK,oBAAoB,IAAI,KAAK,IAAI;;;CAI1C,MAAc,UAAU,KAAa,OAAgD;AAKnF,MAAI,MAAM,KAAK,QAAQ,OAAO,IAAI,EAAE;AAClC,QAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;AAG7C,OAAIC,8BADW,MAAM,KAAK,UAAU,IAAI,EACb,MAAM,CAC/B,QAAO,KAAK,QAAQ,KAAK,IAAI;AAG/B;;;CAMJ,MAAc,UAAU,KAAa,OAAqC,MAAkB;AAC1F,QAAM,KAAK,mBAAmB,KAAK,OAAO,KAAK;AAC/C,QAAM,KAAK,eAAe;;;CAI5B,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;AACxF,MAAI,MAAM,KAAK,MAAM,SAAS,KAAK,OACjC,OAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,MAAM,CAAC,UAAU,KAAK,SAClD;AAGH,OAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;EAE7C,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,OAAK,QAAQD,0BAAW,OAAO;AAE/B,QAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,KAAK;EAE/C,MAAM,YAAYE,wBAAS,QAAQ,MAAM;AACzC,OAAK,QAAQF,0BAAW,UAAU;AAElC,QAAM,KAAK,OAAO,IAAI,KAAK,UAAU;;;CAIvC,MAAM,gBAA+B;EACnC,MAAM,oDAAoB,KAAK,oBAAoB;AACnD,SAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;AAE3E,SAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,KAAK;AAC/B,OAAI,CAAC,WACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,QAAK,QAAQA,0BAAW,OAAO;AAC/B,QAAK,MAAM,IAAI;;;;CAKnB,MAAc,UAAU,KAAa;AACnC,MAAI;AACF,UAAO,MAAM,KAAK,OAAO,IAAI,IAAI;WAC1B,GAAY;AACnB,OAAI,aAAa,sBAAsB;AAGrC,UAAM,KAAK,MAAM,IAAI;AACrB,UAAM,KAAK,aAAa;AAExB,WAAO,MAAM,KAAK,OAAO,IAAI,IAAI;;AAGnC,SAAM;;;;CAKV,MAAc,MAAM,KAAa;AAC/B,QAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,QAAM,KAAK,OAAO,OAAO,IAAI;AAC7B,OAAK,oBAAoB,OAAO,IAAI;;CAGtC,MAAM,UAAyB;AAC7B,QAAM,KAAK,KAAK,cAAc;;CAGhC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,eAAe,SAAY,MAA2B,IAAkC;AACtF,KAAI;AACF,QAAM,KAAK,cAAc;AACzB,SAAO,MAAM,IAAI;WACT;AACR,OAAK,SAAS"}
+{"version":3,"file":"cache.cjs","names":["path","rangesFileName","readRangesFile","writeRangesFile","fs","writeToSparseFile","functions","rangesSize","doesRangeExist","addRange"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;;;AAWT,IAAa,sBAAb,MAA8D;CAC5D,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAc,KAAqB;AACjC,SAAOA,UAAAA,QAAK,KAAK,KAAK,UAAUC,eAAAA,eAAe,IAAI,CAAC;;CAGtD,MAAM,IAAI,KAA8B;AACtC,SAAO,MAAMC,eAAAA,eAAe,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;;CAG3D,MAAM,IAAI,KAAa,QAAgB;AACrC,SAAO,MAAMC,eAAAA,gBAAgB,KAAK,QAAQ,KAAK,MAAM,IAAI,EAAE,OAAO;;CAGpE,MAAM,OAAO,KAAa;AACxB,QAAMC,QAAAA,SAAG,GAAG,KAAK,MAAM,IAAI,CAAC;;;;AAchC,IAAa,oBAAb,MAA4D;CAC1D,SAA0B;CAE1B,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAM,MAAyB;AAC7B,SAAA,GAAA,2BAAA,iBAAsB,KAAK,SAAS;AAEpC,UADc,MAAMA,QAAAA,SAAG,QAAQ,KAAK,SAAS,EAChC,QAAQ,MAAM,EAAE,SAAS,KAAK,OAAO,CAAC;;CAGrD,MAAM,OAAO,KAA+B;AAC1C,SAAO,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK,IAAI,CAAC;;CAGzC,KAAK,KAAqB;AACxB,SAAOJ,UAAAA,QAAK,KAAK,KAAK,UAAU,MAAM,KAAK,OAAO;;CAGpD,MAAM,MAAM,KAAa,MAAkB,MAA6B;AACtE,SAAA,GAAA,2BAAA,iBAAsB,KAAK,SAAS;AACpC,QAAMK,aAAAA,kBAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,IAAI,EAAE,MAAM,KAAK;;CAGpF,MAAM,OAAO,KAA4B;AACvC,QAAMD,QAAAA,SAAG,GAAG,KAAK,KAAK,IAAI,CAAC;;;;AAK/B,IAAa,cAAb,MAAoD;;;CAIlD,OAAe,IAAIE,wBAAAA,UAAU,WAAW;CAExC,sCAA6B,IAAI,KAAmB;CACpD,OAAc;CAEd,YACE,QAEA,SACA,QACA,SACA;AALgB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;;;CAKlB,MAAM,QAAQ;AACZ,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,aAAa;IACxB;;;;CAKJ,MAAM,IAAI,KAAa,OAAgD;AACrE,SAAO,MAAM,SAAS,KAAK,MAAM,YAAY;AAC3C,UAAO,MAAM,KAAK,UAAU,KAAK,MAAM;IACvC;;;;CAKJ,MAAM,IAAI,KAAa,OAAmB,MAAiC;AACzE,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,UAAU,KAAK,OAAO,KAAK;IACtC;;CAGJ,MAAc,cAAc;AAC1B,OAAK,OAAO;AACZ,OAAK,sCAAsB,IAAI,KAAmB;EAElD,MAAM,sBAAM,IAAI,MAAM;AAItB,OAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,EAAE;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,IAAI;AACzC,QAAK,QAAQC,eAAAA,WAAW,OAAO;AAC/B,QAAK,oBAAoB,IAAI,KAAK,IAAI;;;CAI1C,MAAc,UAAU,KAAa,OAAgD;AAKnF,MAAI,MAAM,KAAK,QAAQ,OAAO,IAAI,EAAE;AAClC,QAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;AAG7C,OAAIC,eAAAA,eADW,MAAM,KAAK,UAAU,IAAI,EACb,MAAM,CAC/B,QAAO,KAAK,QAAQ,KAAK,IAAI;AAG/B;;;CAMJ,MAAc,UAAU,KAAa,OAAqC,MAAkB;AAC1F,QAAM,KAAK,mBAAmB,KAAK,OAAO,KAAK;AAC/C,QAAM,KAAK,eAAe;;;CAI5B,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;AACxF,MAAI,MAAM,KAAK,MAAM,SAAS,KAAK,OACjC,OAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,MAAM,CAAC,UAAU,KAAK,SAClD;AAGH,OAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;EAE7C,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,OAAK,QAAQD,eAAAA,WAAW,OAAO;AAE/B,QAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,KAAK;EAE/C,MAAM,YAAYE,eAAAA,SAAS,QAAQ,MAAM;AACzC,OAAK,QAAQF,eAAAA,WAAW,UAAU;AAElC,QAAM,KAAK,OAAO,IAAI,KAAK,UAAU;;;CAIvC,MAAM,gBAA+B;EACnC,MAAM,UAAA,GAAA,2BAAA,YAAoB,KAAK,oBAAoB;AACnD,SAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;AAE3E,SAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,KAAK;AAC/B,OAAI,CAAC,WACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,QAAK,QAAQA,eAAAA,WAAW,OAAO;AAC/B,QAAK,MAAM,IAAI;;;;CAKnB,MAAc,UAAU,KAAa;AACnC,MAAI;AACF,UAAO,MAAM,KAAK,OAAO,IAAI,IAAI;WAC1B,GAAY;AACnB,OAAI,aAAa,sBAAsB;AAGrC,UAAM,KAAK,MAAM,IAAI;AACrB,UAAM,KAAK,aAAa;AAExB,WAAO,MAAM,KAAK,OAAO,IAAI,IAAI;;AAGnC,SAAM;;;;CAKV,MAAc,MAAM,KAAa;AAC/B,QAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,QAAM,KAAK,OAAO,OAAO,IAAI;AAC7B,OAAK,oBAAoB,OAAO,IAAI;;CAGtC,MAAM,UAAyB;AAC7B,QAAM,KAAK,KAAK,cAAc;;CAGhC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,eAAe,SAAY,MAA2B,IAAkC;AACtF,KAAI;AACF,QAAM,KAAK,cAAc;AACzB,SAAO,MAAM,IAAI;WACT;AACR,OAAK,SAAS"}

package/dist/drivers/download_blob/sparse_cache/cache.js

@@ -4,7 +4,6 @@ import { ensureDirExists, fileExists, mapEntries } from "@milaboratories/ts-help
 import { functions } from "@milaboratories/helpers";
 import { promises } from "node:fs";
 import path from "node:path";
-
 //#region src/drivers/download_blob/sparse_cache/cache.ts
 /** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */
 var CorruptedRangesError = class extends Error {
@@ -168,7 +167,7 @@ async function withLock(lock, cb) {
 		lock.release();
 	}
 }
-
 //#endregion
 export { CorruptedRangesError, SparseCache, SparseCacheFsFile, SparseCacheFsRanges };
+
 //# sourceMappingURL=cache.js.map

package/dist/drivers/download_blob/sparse_cache/cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache.js","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;;;AAWT,IAAa,sBAAb,MAA8D;CAC5D,YACE,AAAiB,QACjB,AAAiB,UACjB;EAFiB;EACA;;CAGnB,AAAQ,MAAM,KAAqB;AACjC,SAAO,KAAK,KAAK,KAAK,UAAU,eAAe,IAAI,CAAC;;CAGtD,MAAM,IAAI,KAA8B;AACtC,SAAO,MAAM,eAAe,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;;CAG3D,MAAM,IAAI,KAAa,QAAgB;AACrC,SAAO,MAAM,gBAAgB,KAAK,QAAQ,KAAK,MAAM,IAAI,EAAE,OAAO;;CAGpE,MAAM,OAAO,KAAa;AACxB,QAAMA,SAAG,GAAG,KAAK,MAAM,IAAI,CAAC;;;;AAchC,IAAa,oBAAb,MAA4D;CAC1D,AAAiB,SAAS;CAE1B,YACE,AAAiB,QACjB,AAAiB,UACjB;EAFiB;EACA;;CAGnB,MAAM,MAAyB;AAC7B,QAAM,gBAAgB,KAAK,SAAS;AAEpC,UADc,MAAMA,SAAG,QAAQ,KAAK,SAAS,EAChC,QAAQ,MAAM,EAAE,SAAS,KAAK,OAAO,CAAC;;CAGrD,MAAM,OAAO,KAA+B;AAC1C,SAAO,MAAM,WAAW,KAAK,KAAK,IAAI,CAAC;;CAGzC,KAAK,KAAqB;AACxB,SAAO,KAAK,KAAK,KAAK,UAAU,MAAM,KAAK,OAAO;;CAGpD,MAAM,MAAM,KAAa,MAAkB,MAA6B;AACtE,QAAM,gBAAgB,KAAK,SAAS;AACpC,QAAM,kBAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,IAAI,EAAE,MAAM,KAAK;;CAGpF,MAAM,OAAO,KAA4B;AACvC,QAAMA,SAAG,GAAG,KAAK,KAAK,IAAI,CAAC;;;;AAK/B,IAAa,cAAb,MAAoD;;;CAIlD,AAAQ,OAAO,IAAI,UAAU,WAAW;CAExC,AAAO,sCAAsB,IAAI,KAAmB;CACpD,AAAO,OAAO;CAEd,YACE,AAAgB,QAEhB,AAAgB,SAChB,AAAgB,QAChB,AAAgB,SAChB;EALgB;EAEA;EACA;EACA;;;;CAKlB,MAAM,QAAQ;AACZ,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,aAAa;IACxB;;;;CAKJ,MAAM,IAAI,KAAa,OAAgD;AACrE,SAAO,MAAM,SAAS,KAAK,MAAM,YAAY;AAC3C,UAAO,MAAM,KAAK,UAAU,KAAK,MAAM;IACvC;;;;CAKJ,MAAM,IAAI,KAAa,OAAmB,MAAiC;AACzE,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,UAAU,KAAK,OAAO,KAAK;IACtC;;CAGJ,MAAc,cAAc;AAC1B,OAAK,OAAO;AACZ,OAAK,sCAAsB,IAAI,KAAmB;EAElD,MAAM,sBAAM,IAAI,MAAM;AAItB,OAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,EAAE;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,IAAI;AACzC,QAAK,QAAQ,WAAW,OAAO;AAC/B,QAAK,oBAAoB,IAAI,KAAK,IAAI;;;CAI1C,MAAc,UAAU,KAAa,OAAgD;AAKnF,MAAI,MAAM,KAAK,QAAQ,OAAO,IAAI,EAAE;AAClC,QAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;AAG7C,OAAI,eADW,MAAM,KAAK,UAAU,IAAI,EACb,MAAM,CAC/B,QAAO,KAAK,QAAQ,KAAK,IAAI;AAG/B;;;CAMJ,MAAc,UAAU,KAAa,OAAqC,MAAkB;AAC1F,QAAM,KAAK,mBAAmB,KAAK,OAAO,KAAK;AAC/C,QAAM,KAAK,eAAe;;;CAI5B,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;AACxF,MAAI,MAAM,KAAK,MAAM,SAAS,KAAK,OACjC,OAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,MAAM,CAAC,UAAU,KAAK,SAClD;AAGH,OAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;EAE7C,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,OAAK,QAAQ,WAAW,OAAO;AAE/B,QAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,KAAK;EAE/C,MAAM,YAAY,SAAS,QAAQ,MAAM;AACzC,OAAK,QAAQ,WAAW,UAAU;AAElC,QAAM,KAAK,OAAO,IAAI,KAAK,UAAU;;;CAIvC,MAAM,gBAA+B;EACnC,MAAM,SAAS,WAAW,KAAK,oBAAoB;AACnD,SAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;AAE3E,SAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,KAAK;AAC/B,OAAI,CAAC,WACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,QAAK,QAAQ,WAAW,OAAO;AAC/B,QAAK,MAAM,IAAI;;;;CAKnB,MAAc,UAAU,KAAa;AACnC,MAAI;AACF,UAAO,MAAM,KAAK,OAAO,IAAI,IAAI;WAC1B,GAAY;AACnB,OAAI,aAAa,sBAAsB;AAGrC,UAAM,KAAK,MAAM,IAAI;AACrB,UAAM,KAAK,aAAa;AAExB,WAAO,MAAM,KAAK,OAAO,IAAI,IAAI;;AAGnC,SAAM;;;;CAKV,MAAc,MAAM,KAAa;AAC/B,QAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,QAAM,KAAK,OAAO,OAAO,IAAI;AAC7B,OAAK,oBAAoB,OAAO,IAAI;;CAGtC,MAAM,UAAyB;AAC7B,QAAM,KAAK,KAAK,cAAc;;CAGhC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,eAAe,SAAY,MAA2B,IAAkC;AACtF,KAAI;AACF,QAAM,KAAK,cAAc;AACzB,SAAO,MAAM,IAAI;WACT;AACR,OAAK,SAAS"}
+{"version":3,"file":"cache.js","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/cache.ts"],"sourcesContent":["import { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { ensureDirExists, fileExists, mapEntries, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { promises as fs } from \"node:fs\";\nimport path from \"node:path\";\nimport {\n  addRange,\n  doesRangeExist,\n  Ranges,\n  rangesFileName,\n  rangesSize,\n  readRangesFile,\n  writeRangesFile,\n} from \"./ranges\";\nimport { writeToSparseFile } from \"./file\";\nimport { functions } from \"@milaboratories/helpers\";\n\n/** The implementer of SparseCacheRanges could throw it if ranges were corrupted. */\nexport class CorruptedRangesError extends Error {\n  name = \"CorruptedRangesError\";\n}\n\n/** Extracted ranges methods to be able to store ranges somewhere else (e.g. in memory for tests). */\nexport interface SparseCacheRanges {\n  get(key: string): Promise<Ranges>;\n  set(key: string, ranges: Ranges): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores ranges in a directory as JSON files (the default implementation). */\nexport class SparseCacheFsRanges implements SparseCacheRanges {\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  private fPath(key: string): string {\n    return path.join(this.cacheDir, rangesFileName(key));\n  }\n\n  async get(key: string): Promise<Ranges> {\n    return await readRangesFile(this.logger, this.fPath(key));\n  }\n\n  async set(key: string, ranges: Ranges) {\n    return await writeRangesFile(this.logger, this.fPath(key), ranges);\n  }\n\n  async delete(key: string) {\n    await fs.rm(this.fPath(key));\n  }\n}\n\n/** Extracted interface for storing sparse files. */\nexport interface SparseFileStorage {\n  all(): Promise<string[]>;\n  exists(key: string): Promise<boolean>;\n  path(key: string): string;\n  write(key: string, data: Uint8Array, from: number): Promise<void>;\n  delete(key: string): Promise<void>;\n}\n\n/** Stores sparse files in a directory (the default implementation). */\nexport class SparseCacheFsFile implements SparseFileStorage {\n  private readonly suffix = \".sparse.bin\";\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly cacheDir: string,\n  ) {}\n\n  async all(): Promise<string[]> {\n    await ensureDirExists(this.cacheDir);\n    const files = await fs.readdir(this.cacheDir);\n    return files.filter((f) => f.endsWith(this.suffix));\n  }\n\n  async exists(key: string): Promise<boolean> {\n    return await fileExists(this.path(key));\n  }\n\n  path(key: string): string {\n    return path.join(this.cacheDir, key + this.suffix);\n  }\n\n  async write(key: string, data: Uint8Array, from: number): Promise<void> {\n    await ensureDirExists(this.cacheDir);\n    await writeToSparseFile(this.logger, process.platform, this.path(key), data, from);\n  }\n\n  async delete(key: string): Promise<void> {\n    await fs.rm(this.path(key));\n  }\n}\n\n/** LRU cache for ranges of sparse files. */\nexport class SparseCache implements AsyncDisposable {\n  /** Fields are public for tests. */\n\n  /** The lock to make sure cache requests are done one by one. */\n  private lock = new functions.AwaitLock();\n\n  public keyToLastAccessTime = new Map<string, Date>();\n  public size = 0;\n\n  constructor(\n    public readonly logger: MiLogger,\n    /** The hard limit in bytes. */\n    public readonly maxSize: number,\n    public readonly ranges: SparseCacheRanges,\n    public readonly storage: SparseFileStorage,\n  ) {}\n\n  /** Resets a cache's size by rereading everything we already store.\n   * Safe for concurrent use. */\n  async reset() {\n    await withLock(this.lock, async () => {\n      await this.resetUnsafe();\n    });\n  }\n\n  /** Returns a path to the key if the range exists in a cache, otherwise returns undefined.\n   * Safe for concurrent use. */\n  async get(key: string, range: RangeBytes): Promise<string | undefined> {\n    return await withLock(this.lock, async () => {\n      return await this.getUnsafe(key, range);\n    });\n  }\n\n  /** Sets data to the cache's file and clear the cache if it's needed.\n   * Safe for concurrent use. */\n  async set(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    await withLock(this.lock, async () => {\n      await this.setUnsafe(key, range, data);\n    });\n  }\n\n  private async resetUnsafe() {\n    this.size = 0;\n    this.keyToLastAccessTime = new Map<string, Date>();\n\n    const now = new Date();\n    // In rmKey method we first deletes the key from a storage and only then from ranges,\n    // so if something goes wrong between 2 operations,\n    // on reset the logic will be correct.\n    for (const key of await this.storage.all()) {\n      const ranges = await this.ranges.get(key);\n      this.size += rangesSize(ranges);\n      this.keyToLastAccessTime.set(key, now);\n    }\n  }\n\n  private async getUnsafe(key: string, range: RangeBytes): Promise<string | undefined> {\n    // It first checks the storage, and then the ranges.\n    // In another method, when we remove a key, it first deletes a key from a storage and then from ranges,\n    // so if we don't have a key in storage but have it in ranges, the logic here is correct.\n    // We probably could reverse the operations here and there, and everywhere we work with both storage and ranges.\n    if (await this.storage.exists(key)) {\n      this.keyToLastAccessTime.set(key, new Date());\n\n      const ranges = await this.getRanges(key);\n      if (doesRangeExist(ranges, range)) {\n        return this.storage.path(key);\n      }\n\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  private async setUnsafe(key: string, range: { from: number; to: number }, data: Uint8Array) {\n    await this.setWithoutEviction(key, range, data);\n    await this.ensureEvicted();\n  }\n\n  /** Sets a key and recalculates a size, but doesn't ensures that the size is less than the hard limit. */\n  async setWithoutEviction(key: string, range: RangeBytes, data: Uint8Array): Promise<void> {\n    if (range.to - range.from !== data.length) {\n      throw new Error(\n        `SparseCache.set: trying to set ${key} with wrong range length: ` +\n          `range: ${JSON.stringify(range)}, data: ${data.length}`,\n      );\n    }\n\n    this.keyToLastAccessTime.set(key, new Date());\n\n    const ranges = await this.getRanges(key);\n    this.size -= rangesSize(ranges);\n\n    await this.storage.write(key, data, range.from);\n\n    const newRanges = addRange(ranges, range);\n    this.size += rangesSize(newRanges);\n\n    await this.ranges.set(key, newRanges);\n  }\n\n  /** Ensures the size is less than hard limit by deleting the oldest keys. */\n  async ensureEvicted(): Promise<void> {\n    const byTime = mapEntries(this.keyToLastAccessTime);\n    byTime.sort(([_, aDate], [__, bDate]) => bDate.getTime() - aDate.getTime());\n\n    while (this.size > this.maxSize) {\n      const keyAndDate = byTime.pop(); // removes the oldest\n      if (!keyAndDate) {\n        break;\n      }\n      const [key, _] = keyAndDate;\n\n      const ranges = await this.getRanges(key);\n      this.size -= rangesSize(ranges);\n      this.rmKey(key);\n    }\n  }\n\n  /** Gets ranges and if they were corrupted, then remove the file from the cache and reset the cache's size. */\n  private async getRanges(key: string) {\n    try {\n      return await this.ranges.get(key);\n    } catch (e: unknown) {\n      if (e instanceof CorruptedRangesError) {\n        // We need to reset a state of the cache and update current size,\n        // it's the only way to calculate the real size when one of the ranges were corrupted.\n        await this.rmKey(key);\n        await this.resetUnsafe();\n\n        return await this.ranges.get(key);\n      }\n\n      throw e;\n    }\n  }\n\n  /** Removes a key the state of the cache. The size should be updated. */\n  private async rmKey(key: string) {\n    await this.storage.delete(key);\n    await this.ranges.delete(key);\n    this.keyToLastAccessTime.delete(key);\n  }\n\n  async dispose(): Promise<void> {\n    await this.lock.acquireAsync();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Acquires the lock and executes a callback. */\nasync function withLock<T>(lock: functions.AwaitLock, cb: () => Promise<T>): Promise<T> {\n  try {\n    await lock.acquireAsync();\n    return await cb();\n  } finally {\n    lock.release();\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;;;AAWT,IAAa,sBAAb,MAA8D;CAC5D,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAc,KAAqB;AACjC,SAAO,KAAK,KAAK,KAAK,UAAU,eAAe,IAAI,CAAC;;CAGtD,MAAM,IAAI,KAA8B;AACtC,SAAO,MAAM,eAAe,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;;CAG3D,MAAM,IAAI,KAAa,QAAgB;AACrC,SAAO,MAAM,gBAAgB,KAAK,QAAQ,KAAK,MAAM,IAAI,EAAE,OAAO;;CAGpE,MAAM,OAAO,KAAa;AACxB,QAAMA,SAAG,GAAG,KAAK,MAAM,IAAI,CAAC;;;;AAchC,IAAa,oBAAb,MAA4D;CAC1D,SAA0B;CAE1B,YACE,QACA,UACA;AAFiB,OAAA,SAAA;AACA,OAAA,WAAA;;CAGnB,MAAM,MAAyB;AAC7B,QAAM,gBAAgB,KAAK,SAAS;AAEpC,UADc,MAAMA,SAAG,QAAQ,KAAK,SAAS,EAChC,QAAQ,MAAM,EAAE,SAAS,KAAK,OAAO,CAAC;;CAGrD,MAAM,OAAO,KAA+B;AAC1C,SAAO,MAAM,WAAW,KAAK,KAAK,IAAI,CAAC;;CAGzC,KAAK,KAAqB;AACxB,SAAO,KAAK,KAAK,KAAK,UAAU,MAAM,KAAK,OAAO;;CAGpD,MAAM,MAAM,KAAa,MAAkB,MAA6B;AACtE,QAAM,gBAAgB,KAAK,SAAS;AACpC,QAAM,kBAAkB,KAAK,QAAQ,QAAQ,UAAU,KAAK,KAAK,IAAI,EAAE,MAAM,KAAK;;CAGpF,MAAM,OAAO,KAA4B;AACvC,QAAMA,SAAG,GAAG,KAAK,KAAK,IAAI,CAAC;;;;AAK/B,IAAa,cAAb,MAAoD;;;CAIlD,OAAe,IAAI,UAAU,WAAW;CAExC,sCAA6B,IAAI,KAAmB;CACpD,OAAc;CAEd,YACE,QAEA,SACA,QACA,SACA;AALgB,OAAA,SAAA;AAEA,OAAA,UAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;;;CAKlB,MAAM,QAAQ;AACZ,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,aAAa;IACxB;;;;CAKJ,MAAM,IAAI,KAAa,OAAgD;AACrE,SAAO,MAAM,SAAS,KAAK,MAAM,YAAY;AAC3C,UAAO,MAAM,KAAK,UAAU,KAAK,MAAM;IACvC;;;;CAKJ,MAAM,IAAI,KAAa,OAAmB,MAAiC;AACzE,QAAM,SAAS,KAAK,MAAM,YAAY;AACpC,SAAM,KAAK,UAAU,KAAK,OAAO,KAAK;IACtC;;CAGJ,MAAc,cAAc;AAC1B,OAAK,OAAO;AACZ,OAAK,sCAAsB,IAAI,KAAmB;EAElD,MAAM,sBAAM,IAAI,MAAM;AAItB,OAAK,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,EAAE;GAC1C,MAAM,SAAS,MAAM,KAAK,OAAO,IAAI,IAAI;AACzC,QAAK,QAAQ,WAAW,OAAO;AAC/B,QAAK,oBAAoB,IAAI,KAAK,IAAI;;;CAI1C,MAAc,UAAU,KAAa,OAAgD;AAKnF,MAAI,MAAM,KAAK,QAAQ,OAAO,IAAI,EAAE;AAClC,QAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;AAG7C,OAAI,eADW,MAAM,KAAK,UAAU,IAAI,EACb,MAAM,CAC/B,QAAO,KAAK,QAAQ,KAAK,IAAI;AAG/B;;;CAMJ,MAAc,UAAU,KAAa,OAAqC,MAAkB;AAC1F,QAAM,KAAK,mBAAmB,KAAK,OAAO,KAAK;AAC/C,QAAM,KAAK,eAAe;;;CAI5B,MAAM,mBAAmB,KAAa,OAAmB,MAAiC;AACxF,MAAI,MAAM,KAAK,MAAM,SAAS,KAAK,OACjC,OAAM,IAAI,MACR,kCAAkC,IAAI,mCAC1B,KAAK,UAAU,MAAM,CAAC,UAAU,KAAK,SAClD;AAGH,OAAK,oBAAoB,IAAI,qBAAK,IAAI,MAAM,CAAC;EAE7C,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,OAAK,QAAQ,WAAW,OAAO;AAE/B,QAAM,KAAK,QAAQ,MAAM,KAAK,MAAM,MAAM,KAAK;EAE/C,MAAM,YAAY,SAAS,QAAQ,MAAM;AACzC,OAAK,QAAQ,WAAW,UAAU;AAElC,QAAM,KAAK,OAAO,IAAI,KAAK,UAAU;;;CAIvC,MAAM,gBAA+B;EACnC,MAAM,SAAS,WAAW,KAAK,oBAAoB;AACnD,SAAO,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,WAAW,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;AAE3E,SAAO,KAAK,OAAO,KAAK,SAAS;GAC/B,MAAM,aAAa,OAAO,KAAK;AAC/B,OAAI,CAAC,WACH;GAEF,MAAM,CAAC,KAAK,KAAK;GAEjB,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;AACxC,QAAK,QAAQ,WAAW,OAAO;AAC/B,QAAK,MAAM,IAAI;;;;CAKnB,MAAc,UAAU,KAAa;AACnC,MAAI;AACF,UAAO,MAAM,KAAK,OAAO,IAAI,IAAI;WAC1B,GAAY;AACnB,OAAI,aAAa,sBAAsB;AAGrC,UAAM,KAAK,MAAM,IAAI;AACrB,UAAM,KAAK,aAAa;AAExB,WAAO,MAAM,KAAK,OAAO,IAAI,IAAI;;AAGnC,SAAM;;;;CAKV,MAAc,MAAM,KAAa;AAC/B,QAAM,KAAK,QAAQ,OAAO,IAAI;AAC9B,QAAM,KAAK,OAAO,OAAO,IAAI;AAC7B,OAAK,oBAAoB,OAAO,IAAI;;CAGtC,MAAM,UAAyB;AAC7B,QAAM,KAAK,KAAK,cAAc;;CAGhC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,eAAe,SAAY,MAA2B,IAAkC;AACtF,KAAI;AACF,QAAM,KAAK,cAAc;AACzB,SAAO,MAAM,IAAI;WACT;AACR,OAAK,SAAS"}

package/dist/drivers/download_blob/sparse_cache/file.cjs

@@ -1,8 +1,7 @@
-const require_runtime = require('../../../_virtual/_rolldown/runtime.cjs');
+const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
 let node_fs_promises = require("node:fs/promises");
 node_fs_promises = require_runtime.__toESM(node_fs_promises);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
-
 //#region src/drivers/download_blob/sparse_cache/file.ts
 /** Creates a sparse file for all systems
 * Table of what supports sparse files:
@@ -35,7 +34,7 @@ async function writeToSparseFile(logger, platform, path, data, from) {
 	await fileHandle.write(data, 0, data.length, from);
 	await fileHandle.close();
 }
-
 //#endregion
 exports.writeToSparseFile = writeToSparseFile;
+
 //# sourceMappingURL=file.cjs.map

package/dist/drivers/download_blob/sparse_cache/file.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"file.cjs","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;AAChG,KAAI;AAEF,SADsB,MAAMA,iBAAG,KAAK,MAAM,IAAI,EAC1B,OAAO;AAE3B,QAAM,sBAAsB,MAAM,SAAS;UACpC,OAAgB;AACvB,SAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,QAAQ;;;;;;;;;AAUjF,eAAe,sBAAsB,MAAc,UAA2B;AAC5E,KAAI,aAAa,QAEf,kDAAiB,UAAU;EAAC;EAAU;EAAW;EAAK,EAAE,EAAE,OAAO,QAAQ,CAAC;;;AAK9E,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;AACA,KAAI,CAAE,iDAAiB,KAAK,CAC1B,OAAM,iBAAiB,QAAQ,MAAM,SAAS;CAGhD,MAAM,aAAa,MAAMA,iBAAG,KAAK,MAAM,KAAK;AAC5C,OAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,KAAK;AAClD,OAAM,WAAW,OAAO"}
+{"version":3,"file":"file.cjs","names":["fs"],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;AAChG,KAAI;AAEF,SADsB,MAAMA,iBAAG,KAAK,MAAM,IAAI,EAC1B,OAAO;AAE3B,QAAM,sBAAsB,MAAM,SAAS;UACpC,OAAgB;AACvB,SAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,QAAQ;;;;;;;;;AAUjF,eAAe,sBAAsB,MAAc,UAA2B;AAC5E,KAAI,aAAa,QAEf,QAAA,GAAA,2BAAA,YAAiB,UAAU;EAAC;EAAU;EAAW;EAAK,EAAE,EAAE,OAAO,QAAQ,CAAC;;;AAK9E,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;AACA,KAAI,CAAE,OAAA,GAAA,2BAAA,YAAiB,KAAK,CAC1B,OAAM,iBAAiB,QAAQ,MAAM,SAAS;CAGhD,MAAM,aAAa,MAAMA,iBAAG,KAAK,MAAM,KAAK;AAC5C,OAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,KAAK;AAClD,OAAM,WAAW,OAAO"}

package/dist/drivers/download_blob/sparse_cache/file.js

@@ -1,6 +1,5 @@
 import * as fs from "node:fs/promises";
 import { fileExists, spawnAsync } from "@milaboratories/ts-helpers";
-
 //#region src/drivers/download_blob/sparse_cache/file.ts
 /** Creates a sparse file for all systems
 * Table of what supports sparse files:
@@ -33,7 +32,7 @@ async function writeToSparseFile(logger, platform, path, data, from) {
 	await fileHandle.write(data, 0, data.length, from);
 	await fileHandle.close();
 }
-
 //#endregion
 export { writeToSparseFile };
+
 //# sourceMappingURL=file.js.map

package/dist/drivers/download_blob/sparse_cache/file.js.map

@@ -1 +1 @@
-{"version":3,"file":"file.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;AAChG,KAAI;AAEF,SADsB,MAAM,GAAG,KAAK,MAAM,IAAI,EAC1B,OAAO;AAE3B,QAAM,sBAAsB,MAAM,SAAS;UACpC,OAAgB;AACvB,SAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,QAAQ;;;;;;;;;AAUjF,eAAe,sBAAsB,MAAc,UAA2B;AAC5E,KAAI,aAAa,QAEf,OAAM,WAAW,UAAU;EAAC;EAAU;EAAW;EAAK,EAAE,EAAE,OAAO,QAAQ,CAAC;;;AAK9E,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;AACA,KAAI,CAAE,MAAM,WAAW,KAAK,CAC1B,OAAM,iBAAiB,QAAQ,MAAM,SAAS;CAGhD,MAAM,aAAa,MAAM,GAAG,KAAK,MAAM,KAAK;AAC5C,OAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,KAAK;AAClD,OAAM,WAAW,OAAO"}
+{"version":3,"file":"file.js","names":[],"sources":["../../../../src/drivers/download_blob/sparse_cache/file.ts"],"sourcesContent":["import { fileExists, MiLogger, spawnAsync } from \"@milaboratories/ts-helpers\";\nimport * as fs from \"node:fs/promises\";\n\n/** Creates a sparse file for all systems\n * Table of what supports sparse files:\n * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */\nexport async function createSparseFile(logger: MiLogger, path: string, platform: NodeJS.Platform) {\n  try {\n    const ensureCreated = await fs.open(path, \"w\");\n    await ensureCreated.close();\n\n    await ensureSparseOnWindows(path, platform);\n  } catch (error: unknown) {\n    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);\n  }\n}\n\n/** Sets a sparse flag on Windows.\n * We could check the file is sparse by running:\n * `fsutil sparse queryflag <path>`\n * and\n * `fsutil sparse queryrange <path>`\n */\nasync function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {\n  if (platform === \"win32\") {\n    // https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-sparse\n    await spawnAsync(\"fsutil\", [\"sparse\", \"setflag\", path], { stdio: \"pipe\" });\n  }\n}\n\n/** Ensures the file is created and writes to it. */\nexport async function writeToSparseFile(\n  logger: MiLogger,\n  platform: NodeJS.Platform,\n  path: string,\n  data: Uint8Array,\n  from: number,\n) {\n  if (!(await fileExists(path))) {\n    await createSparseFile(logger, path, platform);\n  }\n\n  const fileHandle = await fs.open(path, \"r+\");\n  await fileHandle.write(data, 0, data.length, from);\n  await fileHandle.close();\n}\n"],"mappings":";;;;;;AAMA,eAAsB,iBAAiB,QAAkB,MAAc,UAA2B;AAChG,KAAI;AAEF,SADsB,MAAM,GAAG,KAAK,MAAM,IAAI,EAC1B,OAAO;AAE3B,QAAM,sBAAsB,MAAM,SAAS;UACpC,OAAgB;AACvB,SAAO,MAAM,uBAAuB,KAAK,eAAe,SAAS,IAAI,QAAQ;;;;;;;;;AAUjF,eAAe,sBAAsB,MAAc,UAA2B;AAC5E,KAAI,aAAa,QAEf,OAAM,WAAW,UAAU;EAAC;EAAU;EAAW;EAAK,EAAE,EAAE,OAAO,QAAQ,CAAC;;;AAK9E,eAAsB,kBACpB,QACA,UACA,MACA,MACA,MACA;AACA,KAAI,CAAE,MAAM,WAAW,KAAK,CAC1B,OAAM,iBAAiB,QAAQ,MAAM,SAAS;CAGhD,MAAM,aAAa,MAAM,GAAG,KAAK,MAAM,KAAK;AAC5C,OAAM,WAAW,MAAM,MAAM,GAAG,KAAK,QAAQ,KAAK;AAClD,OAAM,WAAW,OAAO"}

package/dist/drivers/download_blob/sparse_cache/ranges.cjs

@@ -1,11 +1,10 @@
-const require_runtime = require('../../../_virtual/_rolldown/runtime.cjs');
-const require_cache = require('./cache.cjs');
+const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
+const require_cache = require("./cache.cjs");
 let node_fs_promises = require("node:fs/promises");
 node_fs_promises = require_runtime.__toESM(node_fs_promises);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let zod = require("zod");
-
 //#region src/drivers/download_blob/sparse_cache/ranges.ts
 /** The content of the ranges file: ranges of bytes.
 * The ranges should be normalized: sorted and no overlaps.
@@ -65,7 +64,6 @@ function addRange(s, range) {
 	normalizeRanges(s);
 	return s;
 }
-
 //#endregion
 exports.addRange = addRange;
 exports.doesRangeExist = doesRangeExist;
@@ -73,4 +71,5 @@ exports.rangesFileName = rangesFileName;
 exports.rangesSize = rangesSize;
 exports.readRangesFile = readRangesFile;
 exports.writeRangesFile = writeRangesFile;
+
 //# sourceMappingURL=ranges.cjs.map

package/dist/drivers/download_blob/sparse_cache/ranges.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"ranges.cjs","names":["z","RangeBytes","fs","CorruptedRangesError"],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, \"utf8\");\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n"],"mappings":";;;;;;;;;;;;AASA,MAAM,SAASA,MAAE,OAAO,EACtB,QAAQA,MAAE,MAAMC,2CAAW,EAC5B,CAAC;AAIF,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;AACpD,QAAO,QAAQ;;AAGjB,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,EAAE,EAAE;AACnC,KAAI;EACF,MAAM,OAAO,MAAMC,iBAAG,SAAS,MAAM,OAAO;AAC5C,WAAS,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC;UAChC,GAAY;AACnB,MAAI,aAAa,eAAe,aAAaF,MAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;AAC/D,UAAO,MAAM,IAAI;AACjB,SAAM,IAAIG,mCAAqB,IAAI;;AAGrC,MAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,UACpD,OAAM;;AAMV,iBAAgB,OAAO;AAEvB,QAAO;;;AAIT,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;AACpF,4DAA2B,QAAQ,MAAM,OAAO,aAAqB;AACnE,QAAMD,iBAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;GAC7E;;;AAIJ,SAAgB,gBAAgB,GAAW;AACzC,GAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK;AAExC,MAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,IACvC,KAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,MAAM;AAC1C,cAAY,GAAG,EAAE;AACjB;;;AAKN,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,KAAK;CAC7D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,GAAG,GAAG;AAEvD,GAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;EAAI,CAAC;;AAGrC,SAAgB,WAAW,GAAW;AACpC,QAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;;AAGxE,SAAgB,eAAe,WAAmB,OAA4B;AAC5E,MAAK,MAAM,KAAK,UAAU,OACxB,KAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,GACxC,QAAO;AAIX,QAAO;;AAGT,SAAgB,SAAS,GAAW,OAAmB;AACrD,GAAE,OAAO,KAAK,MAAM;AACpB,iBAAgB,EAAE;AAElB,QAAO"}
+{"version":3,"file":"ranges.cjs","names":["z","RangeBytes","fs","CorruptedRangesError"],"sources":["../../../../src/drivers/download_blob/sparse_cache/ranges.ts"],"sourcesContent":["import { z } from \"zod\";\nimport * as fs from \"node:fs/promises\";\nimport { RangeBytes } from \"@milaboratories/pl-model-common\";\nimport { createPathAtomically, MiLogger } from \"@milaboratories/ts-helpers\";\nimport { CorruptedRangesError } from \"./cache\";\n\n/** The content of the ranges file: ranges of bytes.\n * The ranges should be normalized: sorted and no overlaps.\n * For that, use `normalizeRanges` function. */\nconst Ranges = z.object({\n  ranges: z.array(RangeBytes),\n});\n\nexport type Ranges = z.infer<typeof Ranges>;\n\nexport const rangesFilePostfix = \".ranges.json\";\n\nexport function rangesFileName(fPath: string): string {\n  return fPath + rangesFilePostfix;\n}\n\nexport async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {\n  let ranges: Ranges = { ranges: [] };\n  try {\n    const file = await fs.readFile(path, \"utf8\");\n    ranges = Ranges.parse(JSON.parse(file));\n  } catch (e: unknown) {\n    if (e instanceof SyntaxError || e instanceof z.ZodError) {\n      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;\n      logger.error(msg);\n      throw new CorruptedRangesError(msg);\n    }\n\n    if (!(e instanceof Error && \"code\" in e && e.code === \"ENOENT\")) {\n      throw e;\n    }\n\n    // If the file does not exist, assume the ranges are empty.\n  }\n\n  normalizeRanges(ranges);\n\n  return ranges;\n}\n\n/** Writes to a temporal file and then renames it atomically. */\nexport async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {\n  await createPathAtomically(logger, path, async (tempPath: string) => {\n    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: \"wx\" });\n  });\n}\n\n/** Sorts and merges overlapping ranges. */\nexport function normalizeRanges(s: Ranges) {\n  s.ranges.sort((a, b) => a.from - b.from);\n\n  for (let i = 0; i < s.ranges.length - 1; i++) {\n    if (s.ranges[i].to >= s.ranges[i + 1].from) {\n      mergeRanges(s, i);\n      i--;\n    }\n  }\n}\n\nfunction mergeRanges(s: Ranges, i: number) {\n  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);\n  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);\n\n  s.ranges.splice(i, 2, { from, to });\n}\n\nexport function rangesSize(s: Ranges) {\n  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);\n}\n\nexport function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {\n  for (const r of allRanges.ranges) {\n    if (r.from <= range.from && range.to <= r.to) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function addRange(s: Ranges, range: RangeBytes) {\n  s.ranges.push(range);\n  normalizeRanges(s);\n\n  return s;\n}\n"],"mappings":";;;;;;;;;;;AASA,MAAM,SAASA,IAAAA,EAAE,OAAO,EACtB,QAAQA,IAAAA,EAAE,MAAMC,gCAAAA,WAAW,EAC5B,CAAC;AAIF,MAAa,oBAAoB;AAEjC,SAAgB,eAAe,OAAuB;AACpD,QAAO,QAAQ;;AAGjB,eAAsB,eAAe,QAAkB,MAA+B;CACpF,IAAI,SAAiB,EAAE,QAAQ,EAAE,EAAE;AACnC,KAAI;EACF,MAAM,OAAO,MAAMC,iBAAG,SAAS,MAAM,OAAO;AAC5C,WAAS,OAAO,MAAM,KAAK,MAAM,KAAK,CAAC;UAChC,GAAY;AACnB,MAAI,aAAa,eAAe,aAAaF,IAAAA,EAAE,UAAU;GACvD,MAAM,MAAM,4BAA4B,KAAK,kBAAkB;AAC/D,UAAO,MAAM,IAAI;AACjB,SAAM,IAAIG,cAAAA,qBAAqB,IAAI;;AAGrC,MAAI,EAAE,aAAa,SAAS,UAAU,KAAK,EAAE,SAAS,UACpD,OAAM;;AAMV,iBAAgB,OAAO;AAEvB,QAAO;;;AAIT,eAAsB,gBAAgB,QAAkB,MAAc,QAAgB;AACpF,QAAA,GAAA,2BAAA,sBAA2B,QAAQ,MAAM,OAAO,aAAqB;AACnE,QAAMD,iBAAG,UAAU,UAAU,KAAK,UAAU,QAAQ,MAAM,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;GAC7E;;;AAIJ,SAAgB,gBAAgB,GAAW;AACzC,GAAE,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK;AAExC,MAAK,IAAI,IAAI,GAAG,IAAI,EAAE,OAAO,SAAS,GAAG,IACvC,KAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,MAAM;AAC1C,cAAY,GAAG,EAAE;AACjB;;;AAKN,SAAS,YAAY,GAAW,GAAW;CACzC,MAAM,OAAO,KAAK,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,GAAG,KAAK;CAC7D,MAAM,KAAK,KAAK,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,GAAG,GAAG;AAEvD,GAAE,OAAO,OAAO,GAAG,GAAG;EAAE;EAAM;EAAI,CAAC;;AAGrC,SAAgB,WAAW,GAAW;AACpC,QAAO,EAAE,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;;AAGxE,SAAgB,eAAe,WAAmB,OAA4B;AAC5E,MAAK,MAAM,KAAK,UAAU,OACxB,KAAI,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,GACxC,QAAO;AAIX,QAAO;;AAGT,SAAgB,SAAS,GAAW,OAAmB;AACrD,GAAE,OAAO,KAAK,MAAM;AACpB,iBAAgB,EAAE;AAElB,QAAO"}