@milaboratories/pl-drivers 1.13.1 → 1.14.0

package/dist/drivers/download_blob/download_blob.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_blob.js","names":["path","fs"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n  isNotFoundError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n  AnyLogHandle,\n  BlobDriver,\n  ContentHandler,\n  GetContentOptions,\n  LocalBlobHandle,\n  LocalBlobHandleAndSize,\n  ReadyLogHandle,\n  RemoteBlobHandle,\n  RemoteBlobHandleAndSize,\n  StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  makeResourceSnapshot,\n  treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n  isLocalBlobHandle,\n  newLocalHandle,\n  parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n  isRemoteBlobHandle,\n  newRemoteHandle,\n  parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n  /**\n   * A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /**\n   * A hard limit of the amount of sparse cache, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one.\n   *\n   * The sparse cache is used to store ranges of blobs.\n   * */\n  rangesCacheMaxSizeBytes: number;\n\n  /**\n   * Max number of concurrent downloads while calculating computable states\n   * derived from this driver\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. */\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n  /** Represents a unique key to the path of a blob as a map. */\n  private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadBlobTask>;\n  private rangesCache: SparseCache;\n\n  /** Downloads files and writes them to the local dir. */\n  private downloadQueue: TaskProcessor;\n\n  private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n  private idToLastLines: Map<string, LastLinesGetter> = new Map();\n  private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n  private readonly saveDir: string;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    private readonly clientLogs: ClientLogs,\n    saveDir: string,\n    private readonly rangesCacheDir: string,\n    private readonly signer: Signer,\n    private readonly ops: DownloadDriverOps,\n  ) {\n    this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n    const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n    const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n    this.rangesCache = new SparseCache(\n      this.logger,\n      this.ops.rangesCacheMaxSizeBytes,\n      fsRanges,\n      fsStorage,\n    );\n\n    this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n    this.saveDir = path.resolve(saveDir);\n  }\n\n  static async init(\n    logger: MiLogger,\n    clientDownload: ClientDownload,\n    clientLogs: ClientLogs,\n    saveDir: string,\n    rangesCacheDir: string,\n    signer: Signer,\n    ops: DownloadDriverOps,\n  ): Promise<DownloadDriver> {\n    const driver = new DownloadDriver(\n      logger,\n      clientDownload,\n      clientLogs,\n      saveDir,\n      rangesCacheDir,\n      signer,\n      ops,\n    );\n    await driver.rangesCache.reset();\n\n    return driver;\n  }\n\n  /** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx: ComputableCtx,\n  ): LocalBlobHandleAndSize | undefined;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n  ): ComputableStableDefined<LocalBlobHandleAndSize>;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {\n    if (ctx === undefined) {\n      return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n    }\n\n    const rInfo = treeEntryToResourceInfo(res, ctx);\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n    const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n    if (result == undefined) {\n      ctx.markUnstable(\"download blob is still undefined\");\n    }\n\n    return result;\n  }\n\n  private getDownloadedBlobNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    callerId: string,\n  ): LocalBlobHandleAndSize | undefined {\n    validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n    // We don't need to request files with wider limits,\n    // PFrame's engine does it disk-optimally by itself.\n\n    const task = this.getOrSetNewTask(rInfo, callerId);\n    task.attach(w, callerId);\n\n    const result = task.getBlob();\n    if (!result.done) {\n      return undefined;\n    }\n    if (result.result.ok) {\n      return result.result.value;\n    }\n    throw result.result.error;\n  }\n\n  private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n    const key = blobKey(rInfo.id);\n\n    const inMemoryTask = this.keyToDownload.get(key);\n    if (inMemoryTask) {\n      return inMemoryTask;\n    }\n\n    // schedule the blob downloading, then it'll be added to the cache.\n    const fPath = path.resolve(this.saveDir, key);\n\n    const newTask = new DownloadBlobTask(\n      this.logger,\n      this.clientDownload,\n      rInfo,\n      newLocalHandle(fPath, this.signer),\n      fPath,\n    );\n    this.keyToDownload.set(key, newTask);\n\n    this.downloadQueue.push({\n      fn: () => this.downloadBlob(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask;\n  }\n\n  private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n    await task.download();\n    const blob = task.getBlob();\n    if (blob.done && blob.result.ok) {\n      this.cache.addCache(task, callerId);\n    }\n  }\n\n  /** Gets on demand blob. */\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: undefined,\n    fromBytes?: number,\n    toBytes?: number,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx: ComputableCtx,\n    fromBytes?: number,\n    toBytes?: number,\n  ): RemoteBlobHandleAndSize;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): ComputableStableDefined<RemoteBlobHandleAndSize> | RemoteBlobHandleAndSize | undefined {\n    if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n    const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n      ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n      : res;\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n    // note that the watcher is not needed,\n    // the handler never changes.\n    const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n    return result;\n  }\n\n  private getOnDemandBlobNoCtx(\n    info: OnDemandBlobResourceSnapshot,\n    callerId: string,\n  ): RemoteBlobHandleAndSize {\n    validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n    let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n    if (blob === undefined) {\n      blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n      this.keyToOnDemand.set(blobKey(info.id), blob);\n    }\n\n    blob.attach(callerId);\n\n    return blob.getHandle();\n  }\n\n  /** Gets a path from a handle. */\n  public getLocalPath(handle: LocalBlobHandle): string {\n    const { path } = parseLocalHandle(handle, this.signer);\n    return path;\n  }\n\n  /** Gets a content of a blob by a handle. */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options?: GetContentOptions,\n  ): Promise<Uint8Array>;\n  /** @deprecated Use {@link getContent} with {@link GetContentOptions} instead */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    range?: RangeBytes,\n  ): Promise<Uint8Array>;\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    optionsOrRange?: GetContentOptions | RangeBytes,\n  ): Promise<Uint8Array> {\n    let options: GetContentOptions = {};\n    if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n      if (\"range\" in optionsOrRange) {\n        options = optionsOrRange;\n      } else {\n        const range = optionsOrRange as RangeBytes;\n        validateRangeBytes(range, `getContent`);\n        options = { range };\n      }\n    }\n\n    const request = () =>\n      this.withContent(handle, {\n        ...options,\n        handler: async (content) => {\n          const chunks: Uint8Array[] = [];\n          for await (const chunk of content) {\n            options.signal?.throwIfAborted();\n            chunks.push(chunk);\n          }\n          return Buffer.concat(chunks);\n        },\n      });\n\n    try {\n      return await request();\n    } catch (error) {\n      if (isOffByOneError(error)) {\n        return await request();\n      }\n      throw error;\n    }\n  }\n\n  /** Gets a content stream of a blob by a handle and calls handler with it. */\n  public async withContent<T>(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options: GetContentOptions & {\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    const { range, signal, handler } = options;\n\n    if (isLocalBlobHandle(handle)) {\n      return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n    }\n\n    if (isRemoteBlobHandle(handle)) {\n      const result = parseRemoteHandle(handle, this.signer);\n\n      const key = blobKey(result.info.id);\n      const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n      signal?.throwIfAborted();\n\n      if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n      return await this.clientDownload.withBlobContent(\n        result.info,\n        { signal },\n        options,\n        async (content, size) => {\n          const [handlerStream, cacheStream] = content.tee();\n\n          const handlerPromise = handler(handlerStream, size);\n          const _cachePromise = buffer(cacheStream)\n            .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n            .catch(() => {\n              // Ignore cache errors - they shouldn't affect the main handler result\n              // This prevents unhandled promise rejections when the stream fails\n            });\n\n          return await handlerPromise;\n        },\n      );\n    }\n\n    throw new Error(\"Malformed remote handle\");\n  }\n\n  /**\n   * Creates computable that will return blob content once it is downloaded.\n   * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n   */\n  public getComputableContent(\n    res: ResourceInfo | PlTreeEntry,\n    range?: RangeBytes,\n  ): ComputableStableDefined<Uint8Array> {\n    if (range) {\n      validateRangeBytes(range, `getComputableContent`);\n    }\n\n    return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n      postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n    }).withStableType();\n  }\n\n  /** Returns all logs and schedules a job that reads remain logs.\n   * Notifies when a new portion of the log appeared. */\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n    if (result == undefined)\n      ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, lines);\n      this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n  ): Computable<string | undefined>;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx,\n  ): string | undefined;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getProgressLogNoCtx(\n      ctx.watcher,\n      r as ResourceSnapshot,\n      patternToSearch,\n      callerId,\n    );\n    if (result === undefined)\n      ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n      this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns an Id of a smart object, that can read logs directly from\n   * the platform. */\n  public getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<AnyLogHandle>;\n  public getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n  public getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<AnyLogHandle> | AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    return this.getLogHandleNoCtx(r as ResourceSnapshot);\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n    validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n    return newLogHandle(false, rInfo);\n  }\n\n  public async lastLines(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number, // if 0n, then start from the end.\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.lastLines(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  public async readText(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.readText(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n    const task = this.keyToDownload.get(blobKey(rInfo.id));\n    if (task == undefined) {\n      return;\n    }\n\n    if (this.cache.existsFile(blobKey(rInfo.id))) {\n      const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n      await Promise.all(\n        toDelete.map(async (cachedFile) => {\n          await fsp.rm(cachedFile.path);\n\n          this.cache.removeCache(cachedFile);\n\n          this.removeTask(\n            mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n            `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted) {\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n      }\n    }\n  }\n\n  private removeTask(task: DownloadBlobTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n    this.keyToDownload.delete(pathToKey(task.path));\n    this.idToLastLines.delete(blobKey(task.rInfo.id));\n    this.idToProgressLog.delete(blobKey(task.rInfo.id));\n  }\n\n  private async releaseOnDemandBlob(blobId: ResourceId, callerId: string) {\n    const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n    if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    this.keyToDownload.forEach((task, key) => {\n      this.keyToDownload.delete(key);\n      task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n    });\n  }\n\n  async dispose(): Promise<void> {\n    await this.rangesCache.dispose();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Keeps a counter to the on demand handle. */\nclass OnDemandBlobHolder {\n  private readonly counter = new CallersCounter();\n\n  constructor(\n    private readonly size: number,\n    private readonly handle: RemoteBlobHandle,\n  ) {}\n\n  public getHandle(): RemoteBlobHandleAndSize {\n    return { handle: this.handle, size: this.size };\n  }\n\n  public attach(callerId: string) {\n    this.counter.inc(callerId);\n  }\n\n  public release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n}\n\nclass LastLinesGetter {\n  private updater: Updater;\n  private log: string | undefined;\n  private readonly change: ChangeSource = new ChangeSource();\n  private error: any | undefined = undefined;\n\n  constructor(\n    private readonly path: string,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {\n    this.updater = new Updater(async () => this.update());\n  }\n\n  getOrSchedule(w: Watcher): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    this.change.attachWatcher(w);\n\n    this.updater.schedule();\n\n    return {\n      log: this.log,\n      error: this.error,\n    };\n  }\n\n  async update(): Promise<void> {\n    try {\n      const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n      if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n      this.log = newLogs;\n    } catch (e: any) {\n      if (isNotFoundError(e)) {\n        // No resource\n        this.log = \"\";\n        this.error = e;\n        this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n        return;\n      }\n\n      throw e;\n    }\n  }\n}\n\n/** Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n  fPath: string,\n  nLines: number,\n  patternToSearch?: string,\n): Promise<string> {\n  let inStream: fs.ReadStream | undefined;\n  let rl: readline.Interface | undefined;\n\n  try {\n    inStream = fs.createReadStream(fPath);\n    rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n    const lines = new Denque();\n\n    for await (const line of rl) {\n      if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n      lines.push(line);\n      if (lines.length > nLines) {\n        lines.shift();\n      }\n    }\n\n    // last EOL is for keeping backward compat with platforma implementation.\n    return lines.toArray().join(os.EOL) + os.EOL;\n  } finally {\n    // Cleanup resources in finally block to ensure they're always cleaned up\n    try {\n      if (rl) {\n        rl.close();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error closing readline interface:\", cleanupError);\n    }\n\n    try {\n      if (inStream && !inStream.destroyed) {\n        inStream.destroy();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error destroying read stream:\", cleanupError);\n    }\n  }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith(\"Blob/\")) {\n    let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n    if (rType.name == \"Blob\")\n      message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n    throw new WrongResourceTypeError(message);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,gCAAuD,IAAI,KAAK;;;CAIhE;CACA;;CAGA;CAEA,gCAAyD,IAAI,KAAK;CAElE,gCAAsD,IAAI,KAAK;CAC/D,kCAAwD,IAAI,KAAK;CAEjE;CAEA,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;AAPiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,aAAA;AAEA,OAAA,iBAAA;AACA,OAAA,SAAA;AACA,OAAA,MAAA;AAEjB,OAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAI,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAI,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAI,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUA,OAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,kBACE,KACA,KACqF;AACrF,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,QAAQ,wBAAwB,KAAK,IAAI;EAE/C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,uBACE,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAM,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQA,OAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAI,iBAClB,KAAK,QACL,KAAK,gBACL,OACA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,gBACE,KACA,KACwF;AACxF,MAAI,QAAQ,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,QAAsC,cAAc,IAAI,GAC1D,qBAAqB,KAAK,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,qBACE,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,KAAA,GAAW;AACtB,UAAO,IAAI,mBAAmB,QAAQ,KAAK,EAAE,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAAS,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,sBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;EAIvB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAI,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAGY;EACZ,MAAM,EAAE,OAAO,QAAQ,YAAY;AAEnC,MAAI,kBAAkB,OAAO,CAC3B,QAAO,MAAM,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAI,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAAS,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAM,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AACvF,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAM,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;IACvB,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,WAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,qBACE,KACA,OACqC;AACrC,MAAI,MACF,oBAAmB,OAAO,uBAAuB;AAGnD,SAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAA,GACtE,CAAC,CAAC,gBAAgB;;CAcrB,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAE9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,KAAA,EACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAC9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,aACE,KACA,KACyC;AACzC,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,IAAI,wBAAwB,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,kBAA0B,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAO,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,KAAA,EACV;AAGF,MAAI,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAM,IAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,WACH,OAAO,KAAK,eAAe,UAAU,WAAW,KAAK,CAAC,EACtD,YAAY,wBAAwB,WAAW,CAAC,oCACrB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,WAAmB,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAO,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAAoB,UAAkB;AAEtE,MADgB,KAAK,cAAc,IAAI,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAO,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,UAA2B,IAAI,gBAAgB;CAE/C,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,YAA4C;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,OAAc,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAe,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB;CACA;CACA,SAAwC,IAAI,cAAc;CAC1D,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;AAHiB,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;AAEjB,OAAK,UAAU,IAAI,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,OAAI,gBAAgB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAWC,KAAG,iBAAiB,MAAM;AACrC,OAAK,SAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAI,QAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAK,GAAG,IAAI,GAAG,GAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAI,uBAAuB,QAAQ"}
+{"version":3,"file":"download_blob.js","names":["path","fs"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n  isNotFoundError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n  AnyLogHandle,\n  BlobDriver,\n  ContentHandler,\n  GetContentOptions,\n  LocalBlobHandle,\n  LocalBlobHandleAndSize,\n  ReadyLogHandle,\n  RemoteBlobHandle,\n  RemoteBlobHandleAndSize,\n  StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  makeResourceSnapshot,\n  treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n  isLocalBlobHandle,\n  newLocalHandle,\n  parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n  isRemoteBlobHandle,\n  newRemoteHandle,\n  parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n  /**\n   * A soft limit of the amount of blob storage, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one\n   * when they become unneeded.\n   * */\n  cacheSoftSizeBytes: number;\n\n  /**\n   * A hard limit of the amount of sparse cache, in bytes.\n   * Once exceeded, the download driver will start deleting blobs one by one.\n   *\n   * The sparse cache is used to store ranges of blobs.\n   * */\n  rangesCacheMaxSizeBytes: number;\n\n  /**\n   * Max number of concurrent downloads while calculating computable states\n   * derived from this driver\n   * */\n  nConcurrentDownloads: number;\n};\n\n/** DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. */\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n  /** Represents a unique key to the path of a blob as a map. */\n  private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadBlobTask>;\n  private rangesCache: SparseCache;\n\n  /** Downloads files and writes them to the local dir. */\n  private downloadQueue: TaskProcessor;\n\n  private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n  private idToLastLines: Map<string, LastLinesGetter> = new Map();\n  private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n  private readonly saveDir: string;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientDownload: ClientDownload,\n    private readonly clientLogs: ClientLogs,\n    saveDir: string,\n    private readonly rangesCacheDir: string,\n    private readonly signer: Signer,\n    private readonly ops: DownloadDriverOps,\n  ) {\n    this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n    const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n    const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n    this.rangesCache = new SparseCache(\n      this.logger,\n      this.ops.rangesCacheMaxSizeBytes,\n      fsRanges,\n      fsStorage,\n    );\n\n    this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n    this.saveDir = path.resolve(saveDir);\n  }\n\n  static async init(\n    logger: MiLogger,\n    clientDownload: ClientDownload,\n    clientLogs: ClientLogs,\n    saveDir: string,\n    rangesCacheDir: string,\n    signer: Signer,\n    ops: DownloadDriverOps,\n  ): Promise<DownloadDriver> {\n    const driver = new DownloadDriver(\n      logger,\n      clientDownload,\n      clientLogs,\n      saveDir,\n      rangesCacheDir,\n      signer,\n      ops,\n    );\n    await driver.rangesCache.reset();\n\n    return driver;\n  }\n\n  /** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx: ComputableCtx,\n  ): LocalBlobHandleAndSize | undefined;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n  ): ComputableStableDefined<LocalBlobHandleAndSize>;\n  public getDownloadedBlob(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {\n    if (ctx === undefined) {\n      return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n    }\n\n    const rInfo = treeEntryToResourceInfo(res, ctx);\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n    const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n    if (result == undefined) {\n      ctx.markUnstable(\"download blob is still undefined\");\n    }\n\n    return result;\n  }\n\n  private getDownloadedBlobNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    callerId: string,\n  ): LocalBlobHandleAndSize | undefined {\n    validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n    // We don't need to request files with wider limits,\n    // PFrame's engine does it disk-optimally by itself.\n\n    const task = this.getOrSetNewTask(rInfo, callerId);\n    task.attach(w, callerId);\n\n    const result = task.getBlob();\n    if (!result.done) {\n      return undefined;\n    }\n    if (result.result.ok) {\n      return result.result.value;\n    }\n    throw result.result.error;\n  }\n\n  private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n    const key = blobKey(rInfo.id);\n\n    const inMemoryTask = this.keyToDownload.get(key);\n    if (inMemoryTask) {\n      return inMemoryTask;\n    }\n\n    // schedule the blob downloading, then it'll be added to the cache.\n    const fPath = path.resolve(this.saveDir, key);\n\n    const newTask = new DownloadBlobTask(\n      this.logger,\n      this.clientDownload,\n      rInfo,\n      newLocalHandle(fPath, this.signer),\n      fPath,\n    );\n    this.keyToDownload.set(key, newTask);\n\n    this.downloadQueue.push({\n      fn: () => this.downloadBlob(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask;\n  }\n\n  private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n    await task.download();\n    const blob = task.getBlob();\n    if (blob.done && blob.result.ok) {\n      this.cache.addCache(task, callerId);\n    }\n  }\n\n  /** Gets on demand blob. */\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: undefined,\n    fromBytes?: number,\n    toBytes?: number,\n  ): Computable<RemoteBlobHandleAndSize>;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx: ComputableCtx,\n    fromBytes?: number,\n    toBytes?: number,\n  ): RemoteBlobHandleAndSize;\n  public getOnDemandBlob(\n    res: OnDemandBlobResourceSnapshot | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): ComputableStableDefined<RemoteBlobHandleAndSize> | RemoteBlobHandleAndSize | undefined {\n    if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n    const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n      ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n      : res;\n\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n    // note that the watcher is not needed,\n    // the handler never changes.\n    const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n    return result;\n  }\n\n  private getOnDemandBlobNoCtx(\n    info: OnDemandBlobResourceSnapshot,\n    callerId: string,\n  ): RemoteBlobHandleAndSize {\n    validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n    let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n    if (blob === undefined) {\n      blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n      this.keyToOnDemand.set(blobKey(info.id), blob);\n    }\n\n    blob.attach(callerId);\n\n    return blob.getHandle();\n  }\n\n  /** Gets a path from a handle. */\n  public getLocalPath(handle: LocalBlobHandle): string {\n    const { path } = parseLocalHandle(handle, this.signer);\n    return path;\n  }\n\n  /** Gets a content of a blob by a handle. */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options?: GetContentOptions,\n  ): Promise<Uint8Array>;\n  /** @deprecated Use {@link getContent} with {@link GetContentOptions} instead */\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    range?: RangeBytes,\n  ): Promise<Uint8Array>;\n  public async getContent(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    optionsOrRange?: GetContentOptions | RangeBytes,\n  ): Promise<Uint8Array> {\n    let options: GetContentOptions = {};\n    if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n      if (\"range\" in optionsOrRange) {\n        options = optionsOrRange;\n      } else {\n        const range = optionsOrRange as RangeBytes;\n        validateRangeBytes(range, `getContent`);\n        options = { range };\n      }\n    }\n\n    const request = () =>\n      this.withContent(handle, {\n        ...options,\n        handler: async (content) => {\n          const chunks: Uint8Array[] = [];\n          for await (const chunk of content) {\n            options.signal?.throwIfAborted();\n            chunks.push(chunk);\n          }\n          return Buffer.concat(chunks);\n        },\n      });\n\n    try {\n      return await request();\n    } catch (error) {\n      if (isOffByOneError(error)) {\n        return await request();\n      }\n      throw error;\n    }\n  }\n\n  /** Gets a content stream of a blob by a handle and calls handler with it. */\n  public async withContent<T>(\n    handle: LocalBlobHandle | RemoteBlobHandle,\n    options: GetContentOptions & {\n      handler: ContentHandler<T>;\n    },\n  ): Promise<T> {\n    const { range, signal, handler } = options;\n\n    if (isLocalBlobHandle(handle)) {\n      return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n    }\n\n    if (isRemoteBlobHandle(handle)) {\n      const result = parseRemoteHandle(handle, this.signer);\n\n      const key = blobKey(result.info.id);\n      const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n      signal?.throwIfAborted();\n\n      if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n      return await this.clientDownload.withBlobContent(\n        result.info,\n        { signal },\n        options,\n        async (content, size) => {\n          const [handlerStream, cacheStream] = content.tee();\n\n          const handlerPromise = handler(handlerStream, size);\n          const _cachePromise = buffer(cacheStream)\n            .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n            .catch(() => {\n              // Ignore cache errors - they shouldn't affect the main handler result\n              // This prevents unhandled promise rejections when the stream fails\n            });\n\n          return await handlerPromise;\n        },\n      );\n    }\n\n    throw new Error(\"Malformed remote handle\");\n  }\n\n  /**\n   * Creates computable that will return blob content once it is downloaded.\n   * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n   */\n  public getComputableContent(\n    res: ResourceInfo | PlTreeEntry,\n    range?: RangeBytes,\n  ): ComputableStableDefined<Uint8Array> {\n    if (range) {\n      validateRangeBytes(range, `getComputableContent`);\n    }\n\n    return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n      postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n    }).withStableType();\n  }\n\n  /** Returns all logs and schedules a job that reads remain logs.\n   * Notifies when a new portion of the log appeared. */\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx: ComputableCtx,\n  ): Computable<string | undefined>;\n  public getLastLogs(\n    res: ResourceInfo | PlTreeEntry,\n    lines: number,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n    if (result == undefined)\n      ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n    return result;\n  }\n\n  private getLastLogsNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    lines: number,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, lines);\n      this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns a last line that has patternToSearch.\n   * Notifies when a new line appeared or EOF reached. */\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n  ): Computable<string | undefined>;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx: ComputableCtx,\n  ): string | undefined;\n  public getProgressLog(\n    res: ResourceInfo | PlTreeEntry,\n    patternToSearch: string,\n    ctx?: ComputableCtx,\n  ): Computable<string | undefined> | string | undefined {\n    if (ctx == undefined)\n      return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n    const callerId = randomUUID();\n    ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n    const result = this.getProgressLogNoCtx(\n      ctx.watcher,\n      r as ResourceSnapshot,\n      patternToSearch,\n      callerId,\n    );\n    if (result === undefined)\n      ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n    return result;\n  }\n\n  private getProgressLogNoCtx(\n    w: Watcher,\n    rInfo: ResourceSnapshot,\n    patternToSearch: string,\n    callerId: string,\n  ): string | undefined {\n    validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n    const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n    if (blob == undefined) return undefined;\n    const { path } = parseLocalHandle(blob.handle, this.signer);\n\n    let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n    if (logGetter == undefined) {\n      const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n      this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n      logGetter = newLogGetter;\n    }\n\n    const result = logGetter.getOrSchedule(w);\n    if (result.error) throw result.error;\n\n    return result.log;\n  }\n\n  /** Returns an Id of a smart object, that can read logs directly from\n   * the platform. */\n  public getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<AnyLogHandle>;\n  public getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n  public getLogHandle(\n    res: ResourceInfo | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<AnyLogHandle> | AnyLogHandle {\n    if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n    const r = treeEntryToResourceInfo(res, ctx);\n\n    return this.getLogHandleNoCtx(r as ResourceSnapshot);\n  }\n\n  private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n    validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n    return newLogHandle(false, rInfo);\n  }\n\n  public async lastLines(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number, // if 0n, then start from the end.\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.lastLines(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  public async readText(\n    handle: ReadyLogHandle,\n    lineCount: number,\n    offsetBytes?: number,\n    searchStr?: string,\n  ): Promise<StreamingApiResponse> {\n    const resp = await this.clientLogs.readText(\n      getResourceInfoFromLogHandle(handle),\n      lineCount,\n      BigInt(offsetBytes ?? 0),\n      searchStr,\n    );\n\n    return {\n      live: false,\n      shouldUpdateHandle: false,\n      data: resp.data,\n      size: Number(resp.size),\n      newOffset: Number(resp.newOffset),\n    };\n  }\n\n  private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n    const task = this.keyToDownload.get(blobKey(rInfo.id));\n    if (task == undefined) {\n      return;\n    }\n\n    if (this.cache.existsFile(blobKey(rInfo.id))) {\n      const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n      await Promise.all(\n        toDelete.map(async (cachedFile) => {\n          await fsp.rm(cachedFile.path);\n\n          this.cache.removeCache(cachedFile);\n\n          this.removeTask(\n            mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n            `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted) {\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n      }\n    }\n  }\n\n  private removeTask(task: DownloadBlobTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n    this.keyToDownload.delete(pathToKey(task.path));\n    this.idToLastLines.delete(blobKey(task.rInfo.id));\n    this.idToProgressLog.delete(blobKey(task.rInfo.id));\n  }\n\n  private async releaseOnDemandBlob(blobId: SignedResourceId, callerId: string) {\n    const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n    if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    this.keyToDownload.forEach((task, key) => {\n      this.keyToDownload.delete(key);\n      task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n    });\n  }\n\n  async dispose(): Promise<void> {\n    await this.rangesCache.dispose();\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.dispose();\n  }\n}\n\n/** Keeps a counter to the on demand handle. */\nclass OnDemandBlobHolder {\n  private readonly counter = new CallersCounter();\n\n  constructor(\n    private readonly size: number,\n    private readonly handle: RemoteBlobHandle,\n  ) {}\n\n  public getHandle(): RemoteBlobHandleAndSize {\n    return { handle: this.handle, size: this.size };\n  }\n\n  public attach(callerId: string) {\n    this.counter.inc(callerId);\n  }\n\n  public release(callerId: string): boolean {\n    return this.counter.dec(callerId);\n  }\n}\n\nclass LastLinesGetter {\n  private updater: Updater;\n  private log: string | undefined;\n  private readonly change: ChangeSource = new ChangeSource();\n  private error: any | undefined = undefined;\n\n  constructor(\n    private readonly path: string,\n    private readonly lines: number,\n    private readonly patternToSearch?: string,\n  ) {\n    this.updater = new Updater(async () => this.update());\n  }\n\n  getOrSchedule(w: Watcher): {\n    log: string | undefined;\n    error?: any | undefined;\n  } {\n    this.change.attachWatcher(w);\n\n    this.updater.schedule();\n\n    return {\n      log: this.log,\n      error: this.error,\n    };\n  }\n\n  async update(): Promise<void> {\n    try {\n      const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n      if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n      this.log = newLogs;\n    } catch (e: any) {\n      if (isNotFoundError(e)) {\n        // No resource\n        this.log = \"\";\n        this.error = e;\n        this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n        return;\n      }\n\n      throw e;\n    }\n  }\n}\n\n/** Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n  fPath: string,\n  nLines: number,\n  patternToSearch?: string,\n): Promise<string> {\n  let inStream: fs.ReadStream | undefined;\n  let rl: readline.Interface | undefined;\n\n  try {\n    inStream = fs.createReadStream(fPath);\n    rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n    const lines = new Denque();\n\n    for await (const line of rl) {\n      if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n      lines.push(line);\n      if (lines.length > nLines) {\n        lines.shift();\n      }\n    }\n\n    // last EOL is for keeping backward compat with platforma implementation.\n    return lines.toArray().join(os.EOL) + os.EOL;\n  } finally {\n    // Cleanup resources in finally block to ensure they're always cleaned up\n    try {\n      if (rl) {\n        rl.close();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error closing readline interface:\", cleanupError);\n    }\n\n    try {\n      if (inStream && !inStream.destroyed) {\n        inStream.destroy();\n      }\n    } catch (cleanupError) {\n      console.error(\"Error destroying read stream:\", cleanupError);\n    }\n  }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith(\"Blob/\")) {\n    let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n    if (rType.name == \"Blob\")\n      message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n    throw new WrongResourceTypeError(message);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,gCAAuD,IAAI,KAAK;;;CAIhE;CACA;;CAGA;CAEA,gCAAyD,IAAI,KAAK;CAElE,gCAAsD,IAAI,KAAK;CAC/D,kCAAwD,IAAI,KAAK;CAEjE;CAEA,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;AAPiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,aAAA;AAEA,OAAA,iBAAA;AACA,OAAA,SAAA;AACA,OAAA,MAAA;AAEjB,OAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAI,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAI,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAI,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUA,OAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,kBACE,KACA,KACqF;AACrF,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,QAAQ,wBAAwB,KAAK,IAAI;EAE/C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,uBACE,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAM,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQA,OAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAI,iBAClB,KAAK,QACL,KAAK,gBACL,OACA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,gBACE,KACA,KACwF;AACxF,MAAI,QAAQ,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,QAAsC,cAAc,IAAI,GAC1D,qBAAqB,KAAK,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,qBACE,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,KAAA,GAAW;AACtB,UAAO,IAAI,mBAAmB,QAAQ,KAAK,EAAE,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAAS,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,sBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;EAIvB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAI,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAGY;EACZ,MAAM,EAAE,OAAO,QAAQ,YAAY;AAEnC,MAAI,kBAAkB,OAAO,CAC3B,QAAO,MAAM,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAI,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAAS,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAM,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AACvF,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAM,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;IACvB,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,WAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,qBACE,KACA,OACqC;AACrC,MAAI,MACF,oBAAmB,OAAO,uBAAuB;AAGnD,SAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAA,GACtE,CAAC,CAAC,gBAAgB;;CAcrB,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAE9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,KAAA,EACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAC9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,aACE,KACA,KACyC;AACzC,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,IAAI,wBAAwB,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,kBAA0B,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAO,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,KAAA,EACV;AAGF,MAAI,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAM,IAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,WACH,OAAO,KAAK,eAAe,UAAU,WAAW,KAAK,CAAC,EACtD,YAAY,wBAAwB,WAAW,CAAC,oCACrB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,WAAmB,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAO,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAA0B,UAAkB;AAE5E,MADgB,KAAK,cAAc,IAAI,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAO,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,UAA2B,IAAI,gBAAgB;CAE/C,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,YAA4C;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,OAAc,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAe,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB;CACA;CACA,SAAwC,IAAI,cAAc;CAC1D,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;AAHiB,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;AAEjB,OAAK,UAAU,IAAI,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,OAAI,gBAAgB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAWC,KAAG,iBAAiB,MAAM;AACrC,OAAK,SAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAI,QAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAK,GAAG,IAAI,GAAG,GAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAI,uBAAuB,QAAQ"}

package/dist/drivers/download_blob_url/driver.cjs

@@ -119,7 +119,8 @@ var DownloadBlobToURLDriver = class {
 		this.idToDownload.delete(require_driver_id.newId(task.rInfo.id, task.format));
 	}
 	getFilePath(id, format) {
-		return node_path.join(this.saveDir, `${String(BigInt(id))}_${format}`);
+		const { globalId } = (0, _milaboratories_pl_client.parseSignedResourceId)(id);
+		return node_path.join(this.saveDir, `${String(globalId)}_${format}`);
 	}
 };
 //#endregion

package/dist/drivers/download_blob_url/driver.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","getPathForFolderURL","Computable","makeDownloadableBlobSnapshot","newId","nonRecoverableError","rmRFDir","DownloadAndUnarchiveTask","path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { ResourceId } from \"@milaboratories/pl-client\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n    return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAiCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,OAAA,GAAA,gCAAA,aAAgB,IAAI,CAClB,QAAOC,YAAAA,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,SAAA,GAAA,wBAAA,eAAgD,IAAI,GACtDC,iBAAAA,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAIC,kBAAAA,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAgB,QAAuB,UAAiC;EACxF,MAAM,OAAO,KAAK,aAAa,IAAID,kBAAAA,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAME,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAIC,aAAAA,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAIH,kBAAAA,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,aAAA,GAAA,0BAAA,oBAA+B,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAOA,kBAAAA,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAgB,QAA+B;AACjE,SAAOI,UAAK,KAAK,KAAK,SAAS,GAAG,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG,SAAS"}
+{"version":3,"file":"driver.cjs","names":["TaskProcessor","FilesCache","getPathForFolderURL","Computable","makeDownloadableBlobSnapshot","newId","nonRecoverableError","rmRFDir","DownloadAndUnarchiveTask","path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAIA,2BAAAA,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAIC,oBAAAA,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,OAAA,GAAA,gCAAA,aAAgB,IAAI,CAClB,QAAOC,YAAAA,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAOC,2BAAAA,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,SAAA,GAAA,wBAAA,eAAgD,IAAI,GACtDC,iBAAAA,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAIC,kBAAAA,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAID,kBAAAA,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAME,aAAAA,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,qCAAA,GAAA,0BAAA,yBACE,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAMA,aAAAA,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAIC,aAAAA,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAIH,kBAAAA,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,aAAA,GAAA,0BAAA,oBAA+B,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAOA,kBAAAA,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,GAAG;AAC9C,SAAOI,UAAK,KAAK,KAAK,SAAS,GAAG,OAAO,SAAS,CAAC,GAAG,SAAS"}

package/dist/drivers/download_blob_url/driver.d.ts

@@ -1,7 +1,7 @@
 import { ClientDownload } from "../../clients/download.js";
 import { DownloadableBlobSnapshot } from "./snapshot.js";
 import { DownloadAndUnarchiveTask } from "./task.js";
-import { ResourceId } from "@milaboratories/pl-client";
+import { SignedResourceId } from "@milaboratories/pl-client";
 import { MiLogger, Signer } from "@milaboratories/ts-helpers";
 import { Computable, ComputableCtx } from "@milaboratories/computable";
 import { ArchiveFormat, BlobToURLDriver, FolderURL } from "@milaboratories/pl-model-common";
@@ -38,7 +38,7 @@ declare class DownloadBlobToURLDriver implements BlobToURLDriver {
   downloadUrl(task: DownloadAndUnarchiveTask, callerId: string): Promise<void>;
   /** Removes a directory and aborts a downloading task when all callers
    * are not interested in it. */
-  releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void>;
+  releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void>;
   /** Removes all files from a hard drive. */
   releaseAll(): Promise<void>;
   private setNewTask;

package/dist/drivers/download_blob_url/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"mappings":";;;;;;;;;;KA0BY,0BAAA;EACV,kBAAA;EACA,oBAAA;AAAA;;;cAKW,uBAAA,YAAmC,eAAA;EAAA,iBAS3B,MAAA;EAAA,iBACA,MAAA;EAAA,iBACA,cAAA;EAAA,iBACA,OAAA;EAAA,iBACA,IAAA;EAAA,QAZX,YAAA;EAAA,QACA,aAAA;EAgDD;;EAAA,QA5CC,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EACR,cAAA,EAAgB,cAAA,EAChB,OAAA,UACA,IAAA,GAAM,0BAAA;EAelB,IAAA,CAAA;EA4BA;;;EAfP,wBAAA,CAAyB,GAAA,EAAK,SAAA;EAQ9B,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,EACR,GAAA,EAAK,aAAA,GACJ,SAAA;EAEH,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,GACP,UAAA,CAAW,SAAA;EAAA,QA8BN,4BAAA;EA+Bc;EARhB,WAAA,CAAY,IAAA,EAAM,wBAAA,EAA0B,QAAA,WAAgB,OAAA;EAQU;;EAAtE,WAAA,CAAY,EAAA,EAAI,UAAA,EAAY,MAAA,EAAQ,aAAA,EAAe,QAAA,WAAmB,OAAA;EAvHf;EAsJvD,UAAA,CAAA,GAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAqBA,UAAA;EAAA,QAMA,WAAA;AAAA"}
+{"version":3,"file":"driver.d.ts","names":[],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"mappings":";;;;;;;;;;KA8BY,0BAAA;EACV,kBAAA;EACA,oBAAA;AAAA;;;cAKW,uBAAA,YAAmC,eAAA;EAAA,iBAS3B,MAAA;EAAA,iBACA,MAAA;EAAA,iBACA,cAAA;EAAA,iBACA,OAAA;EAAA,iBACA,IAAA;EAAA,QAZX,YAAA;EAAA,QACA,aAAA;EAgDD;;EAAA,QA5CC,KAAA;cAGW,MAAA,EAAQ,QAAA,EACR,MAAA,EAAQ,MAAA,EACR,cAAA,EAAgB,cAAA,EAChB,OAAA,UACA,IAAA,GAAM,0BAAA;EAelB,IAAA,CAAA;EA4BA;;;EAfP,wBAAA,CAAyB,GAAA,EAAK,SAAA;EAQ9B,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,EACR,GAAA,EAAK,aAAA,GACJ,SAAA;EAEH,uBAAA,CACE,GAAA,EAAK,wBAAA,GAA2B,WAAA,EAChC,MAAA,EAAQ,aAAA,GACP,UAAA,CAAW,SAAA;EAAA,QA8BN,4BAAA;EA+Bc;EARhB,WAAA,CAAY,IAAA,EAAM,wBAAA,EAA0B,QAAA,WAAgB,OAAA;EAQgB;;EAA5E,WAAA,CAAY,EAAA,EAAI,gBAAA,EAAkB,MAAA,EAAQ,aAAA,EAAe,QAAA,WAAmB,OAAA;EAvHrB;EAsJvD,UAAA,CAAA,GAAU,OAAA;EAAA,QAgBR,UAAA;EAAA,QAqBA,UAAA;EAAA,QAMA,WAAA;AAAA"}

package/dist/drivers/download_blob_url/driver.js

@@ -4,7 +4,7 @@ import { makeDownloadableBlobSnapshot } from "./snapshot.js";
 import { getPathForFolderURL } from "../urls/url.js";
 import { DownloadAndUnarchiveTask, rmRFDir } from "./task.js";
 import { newId } from "./driver_id.js";
-import { resourceIdToString, stringifyWithResourceId } from "@milaboratories/pl-client";
+import { parseSignedResourceId, resourceIdToString, stringifyWithResourceId } from "@milaboratories/pl-client";
 import { TaskProcessor } from "@milaboratories/ts-helpers";
 import * as path$1 from "node:path";
 import { Computable } from "@milaboratories/computable";
@@ -117,7 +117,8 @@ var DownloadBlobToURLDriver = class {
 		this.idToDownload.delete(newId(task.rInfo.id, task.format));
 	}
 	getFilePath(id, format) {
-		return path$1.join(this.saveDir, `${String(BigInt(id))}_${format}`);
+		const { globalId } = parseSignedResourceId(id);
+		return path$1.join(this.saveDir, `${String(globalId)}_${format}`);
 	}
 };
 //#endregion

package/dist/drivers/download_blob_url/driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { ResourceId } from \"@milaboratories/pl-client\";\nimport { resourceIdToString, stringifyWithResourceId } from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: ResourceId, format: ArchiveFormat): string {\n    return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,MAAI,YAAY,IAAI,CAClB,QAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,QAAkC,cAAc,IAAI,GACtD,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,WAAW,YAAY;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAgB,QAAuB,UAAiC;EACxF,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAgB,QAA+B;AACjE,SAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG,SAAS"}
+{"version":3,"file":"driver.js","names":["path"],"sources":["../../../src/drivers/download_blob_url/driver.ts"],"sourcesContent":["import type { ComputableCtx, Watcher } from \"@milaboratories/computable\";\nimport { Computable } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport { randomUUID } from \"node:crypto\";\nimport * as path from \"node:path\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport {\n  resourceIdToString,\n  stringifyWithResourceId,\n  parseSignedResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n  type ArchiveFormat,\n  type BlobToURLDriver,\n  type FolderURL,\n  isFolderURL,\n} from \"@milaboratories/pl-model-common\";\nimport type { DownloadableBlobSnapshot } from \"./snapshot\";\nimport { makeDownloadableBlobSnapshot } from \"./snapshot\";\nimport type { PlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { isPlTreeEntry } from \"@milaboratories/pl-tree\";\nimport { DownloadAndUnarchiveTask, rmRFDir } from \"./task\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport { getPathForFolderURL } from \"../urls/url\";\nimport type { Id } from \"./driver_id\";\nimport { newId } from \"./driver_id\";\nimport { nonRecoverableError } from \"../download_blob/download_blob_task\";\n\nexport type DownloadBlobToURLDriverOps = {\n  cacheSoftSizeBytes: number;\n  nConcurrentDownloads: number;\n};\n\n/** Downloads .tar, .tar.gz or zip archives,\n * extracts them into saveDir and gets a url for it. */\nexport class DownloadBlobToURLDriver implements BlobToURLDriver {\n  private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();\n  private downloadQueue: TaskProcessor;\n\n  /** Writes and removes files to a hard drive and holds a counter for every\n   * file that should be kept. */\n  private cache: FilesCache<DownloadAndUnarchiveTask>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientDownload: ClientDownload,\n    private readonly saveDir: string,\n    private readonly opts: DownloadBlobToURLDriverOps = {\n      cacheSoftSizeBytes: 50 * 1024 * 1024,\n      nConcurrentDownloads: 50,\n    },\n  ) {\n    this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 10000,\n      maxDelay: 30000,\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n    this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);\n  }\n\n  public info(): any {\n    return {\n      saveDir: this.saveDir,\n      opts: this.opts,\n      idToDownloadSize: this.idToDownload.size,\n      idToDownloadKeys: this.idToDownload.keys(),\n      idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),\n    };\n  }\n\n  /**\n   * @returns full path to the referenced file\n   */\n  getPathForCustomProtocol(url: FolderURL): string {\n    if (isFolderURL(url)) {\n      return getPathForFolderURL(this.signer, url, this.saveDir);\n    }\n\n    throw new Error(`getPathForCustomProtocol: ${url} is invalid`);\n  }\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx: ComputableCtx,\n  ): FolderURL | undefined;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n  ): Computable<FolderURL | undefined>;\n\n  extractArchiveAndGetURL(\n    res: DownloadableBlobSnapshot | PlTreeEntry,\n    format: ArchiveFormat,\n    ctx?: ComputableCtx,\n  ): Computable<FolderURL | undefined> | FolderURL | undefined {\n    // wrap result as computable, if we were not given an existing computable context\n    if (ctx === undefined)\n      return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));\n\n    const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)\n      ? makeDownloadableBlobSnapshot(res, ctx)\n      : res;\n\n    const callerId = randomUUID();\n\n    ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));\n\n    const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);\n    if (result?.url === undefined)\n      ctx.markUnstable(\n        `a path to the downloaded archive might be undefined. The current result: ${result}`,\n      );\n\n    if (result?.error !== undefined) throw result?.error;\n\n    return result?.url;\n  }\n\n  private extractArchiveAndGetURLNoCtx(\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    w: Watcher,\n    callerId: string,\n  ) {\n    const task = this.idToDownload.get(newId(rInfo.id, format));\n\n    if (task != undefined) {\n      task.attach(w, callerId);\n      return task.getURL();\n    }\n\n    const newTask = this.setNewTask(w, rInfo, format, callerId);\n    this.downloadQueue.push({\n      fn: async () => this.downloadUrl(newTask, callerId),\n      recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n    });\n\n    return newTask.getURL();\n  }\n\n  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */\n  async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {\n    await task.download();\n    // Might be undefined if a error happened\n    if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);\n  }\n\n  /** Removes a directory and aborts a downloading task when all callers\n   * are not interested in it. */\n  async releasePath(id: SignedResourceId, format: ArchiveFormat, callerId: string): Promise<void> {\n    const task = this.idToDownload.get(newId(id, format));\n    if (task == undefined) return;\n\n    if (this.cache.existsFile(task.path)) {\n      const toDelete = this.cache.removeFile(task.path, callerId);\n\n      await Promise.all(\n        toDelete.map(async (task: DownloadAndUnarchiveTask) => {\n          await rmRFDir(task.path);\n          this.cache.removeCache(task);\n\n          this.removeTask(\n            task,\n            `the task ${stringifyWithResourceId(task.info())} was removed` +\n              `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,\n          );\n        }),\n      );\n    } else {\n      // The task is still in a downloading queue.\n      const deleted = task.counter.dec(callerId);\n      if (deleted)\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n        );\n    }\n  }\n\n  /** Removes all files from a hard drive. */\n  async releaseAll() {\n    this.downloadQueue.stop();\n\n    await Promise.all(\n      Array.from(this.idToDownload.entries()).map(async ([_, task]) => {\n        await rmRFDir(task.path);\n        this.cache.removeCache(task);\n\n        this.removeTask(\n          task,\n          `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,\n        );\n      }),\n    );\n  }\n\n  private setNewTask(\n    w: Watcher,\n    rInfo: DownloadableBlobSnapshot,\n    format: ArchiveFormat,\n    callerId: string,\n  ) {\n    const result = new DownloadAndUnarchiveTask(\n      this.logger,\n      this.signer,\n      this.saveDir,\n      this.getFilePath(rInfo.id, format),\n      rInfo,\n      format,\n      this.clientDownload,\n    );\n    result.attach(w, callerId);\n    this.idToDownload.set(newId(rInfo.id, format), result);\n\n    return result;\n  }\n\n  private removeTask(task: DownloadAndUnarchiveTask, reason: string) {\n    task.abort(reason);\n    task.change.markChanged(`task for ${resourceIdToString(task.rInfo.id)} removed: ${reason}`);\n    this.idToDownload.delete(newId(task.rInfo.id, task.format));\n  }\n\n  private getFilePath(id: SignedResourceId, format: ArchiveFormat): string {\n    const { globalId } = parseSignedResourceId(id);\n    return path.join(this.saveDir, `${String(globalId)}_${format}`);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqCA,IAAa,0BAAb,MAAgE;CAC9D,+BAA0D,IAAI,KAAK;CACnE;;;CAIA;CAEA,YACE,QACA,QACA,gBACA,SACA,OAAoD;EAClD,oBAAoB,KAAK,OAAO;EAChC,sBAAsB;EACvB,EACD;AARiB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,UAAA;AACA,OAAA,OAAA;AAKjB,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,KAAK,KAAK,sBAAsB;GAClF,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AACF,OAAK,QAAQ,IAAI,WAAW,KAAK,KAAK,mBAAmB;;CAG3D,OAAmB;AACjB,SAAO;GACL,SAAS,KAAK;GACd,MAAM,KAAK;GACX,kBAAkB,KAAK,aAAa;GACpC,kBAAkB,KAAK,aAAa,MAAM;GAC1C,cAAc,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;GAC7F;;;;;CAMH,yBAAyB,KAAwB;AAC/C,MAAI,YAAY,IAAI,CAClB,QAAO,oBAAoB,KAAK,QAAQ,KAAK,KAAK,QAAQ;AAG5D,QAAM,IAAI,MAAM,6BAA6B,IAAI,aAAa;;CAchE,wBACE,KACA,QACA,KAC2D;AAE3D,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,MAAM,KAAK,wBAAwB,KAAK,QAAQ,EAAE,CAAC;EAE7E,MAAM,QAAkC,cAAc,IAAI,GACtD,6BAA6B,KAAK,IAAI,GACtC;EAEJ,MAAM,WAAW,YAAY;AAE7B,MAAI,mBAAmB,KAAK,YAAY,MAAM,IAAI,QAAQ,SAAS,CAAC;EAEpE,MAAM,SAAS,KAAK,6BAA6B,OAAO,QAAQ,IAAI,SAAS,SAAS;AACtF,MAAI,QAAQ,QAAQ,KAAA,EAClB,KAAI,aACF,4EAA4E,SAC7E;AAEH,MAAI,QAAQ,UAAU,KAAA,EAAW,OAAM,QAAQ;AAE/C,SAAO,QAAQ;;CAGjB,6BACE,OACA,QACA,GACA,UACA;EACA,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,CAAC;AAE3D,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,OAAO,GAAG,SAAS;AACxB,UAAO,KAAK,QAAQ;;EAGtB,MAAM,UAAU,KAAK,WAAW,GAAG,OAAO,QAAQ,SAAS;AAC3D,OAAK,cAAc,KAAK;GACtB,IAAI,YAAY,KAAK,YAAY,SAAS,SAAS;GACnD,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO,QAAQ,QAAQ;;;CAIzB,MAAM,YAAY,MAAgC,UAAkB;AAClE,QAAM,KAAK,UAAU;AAErB,MAAI,KAAK,QAAQ,EAAE,OAAO,KAAA,EAAW,MAAK,MAAM,SAAS,MAAM,SAAS;;;;CAK1E,MAAM,YAAY,IAAsB,QAAuB,UAAiC;EAC9F,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC;AACrD,MAAI,QAAQ,KAAA,EAAW;AAEvB,MAAI,KAAK,MAAM,WAAW,KAAK,KAAK,EAAE;GACpC,MAAM,WAAW,KAAK,MAAM,WAAW,KAAK,MAAM,SAAS;AAE3D,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,SAAmC;AACrD,UAAM,QAAQ,KAAK,KAAK;AACxB,SAAK,MAAM,YAAY,KAAK;AAE5B,SAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,oCACtB,wBAAwB,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,GAClF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;;CAKP,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,QAAM,QAAQ,IACZ,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,UAAU;AAC/D,SAAM,QAAQ,KAAK,KAAK;AACxB,QAAK,MAAM,YAAY,KAAK;AAE5B,QAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,0CAClD;IACD,CACH;;CAGH,WACE,GACA,OACA,QACA,UACA;EACA,MAAM,SAAS,IAAI,yBACjB,KAAK,QACL,KAAK,QACL,KAAK,SACL,KAAK,YAAY,MAAM,IAAI,OAAO,EAClC,OACA,QACA,KAAK,eACN;AACD,SAAO,OAAO,GAAG,SAAS;AAC1B,OAAK,aAAa,IAAI,MAAM,MAAM,IAAI,OAAO,EAAE,OAAO;AAEtD,SAAO;;CAGT,WAAmB,MAAgC,QAAgB;AACjE,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,YAAY,mBAAmB,KAAK,MAAM,GAAG,CAAC,YAAY,SAAS;AAC3F,OAAK,aAAa,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,OAAO,CAAC;;CAG7D,YAAoB,IAAsB,QAA+B;EACvE,MAAM,EAAE,aAAa,sBAAsB,GAAG;AAC9C,SAAOA,OAAK,KAAK,KAAK,SAAS,GAAG,OAAO,SAAS,CAAC,GAAG,SAAS"}

package/dist/drivers/download_blob_url/driver_id.cjs

@@ -1,6 +1,9 @@
+require("../../_virtual/_rolldown/runtime.cjs");
+let _milaboratories_pl_client = require("@milaboratories/pl-client");
 //#region src/drivers/download_blob_url/driver_id.ts
 function newId(id, format) {
-	return `id:${String(BigInt(id))}-${format}`;
+	const { globalId } = (0, _milaboratories_pl_client.parseSignedResourceId)(id);
+	return `id:${String(globalId)}-${format}`;
 }
 //#endregion
 exports.newId = newId;

package/dist/drivers/download_blob_url/driver_id.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { ResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: ResourceId, format: ArchiveFormat): Id {\n  return `id:${String(BigInt(id))}-${format}`;\n}\n\n// export function\n"],"mappings":";AAMA,SAAgB,MAAM,IAAgB,QAA2B;AAC/D,QAAO,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG"}
+{"version":3,"file":"driver_id.cjs","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,GAAG;AAC9C,QAAO,MAAM,OAAO,SAAS,CAAC,GAAG"}

package/dist/drivers/download_blob_url/driver_id.js

@@ -1,6 +1,8 @@
+import { parseSignedResourceId } from "@milaboratories/pl-client";
 //#region src/drivers/download_blob_url/driver_id.ts
 function newId(id, format) {
-	return `id:${String(BigInt(id))}-${format}`;
+	const { globalId } = parseSignedResourceId(id);
+	return `id:${String(globalId)}-${format}`;
 }
 //#endregion
 export { newId };

package/dist/drivers/download_blob_url/driver_id.js.map

@@ -1 +1 @@
-{"version":3,"file":"driver_id.js","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { ResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: ResourceId, format: ArchiveFormat): Id {\n  return `id:${String(BigInt(id))}-${format}`;\n}\n\n// export function\n"],"mappings":";AAMA,SAAgB,MAAM,IAAgB,QAA2B;AAC/D,QAAO,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,GAAG"}
+{"version":3,"file":"driver_id.js","names":[],"sources":["../../../src/drivers/download_blob_url/driver_id.ts"],"sourcesContent":["import type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\nimport type { ArchiveFormat } from \"@milaboratories/pl-model-common\";\n\n/** A key in the driver task queue. */\nexport type Id = string;\n\nexport function newId(id: SignedResourceId, format: ArchiveFormat): Id {\n  const { globalId } = parseSignedResourceId(id);\n  return `id:${String(globalId)}-${format}`;\n}\n\n// export function\n"],"mappings":";;AAOA,SAAgB,MAAM,IAAsB,QAA2B;CACrE,MAAM,EAAE,aAAa,sBAAsB,GAAG;AAC9C,QAAO,MAAM,OAAO,SAAS,CAAC,GAAG"}

package/dist/drivers/download_url/task.cjs

@@ -47,7 +47,7 @@ var DownloadByUrlTask = class {
 			this.setDone(size);
 			this.change.markChanged(`download of ${this.url} finished`);
 		} catch (e) {
-			if (e instanceof URLAborted || require_download_errors.isDownloadNetworkError400(e)) {
+			if (e instanceof URLAborted || require_download_errors.isDownloadNetworkError400(e) || this.signalCtl.signal.aborted) {
 				this.setError(e);
 				this.change.markChanged(`download of ${this.url} failed`);
 				await rmRFDir(this.path);

package/dist/drivers/download_url/task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (e instanceof URLAborted || isDownloadNetworkError400(e)) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAIA,2BAAAA,gBAAgB;CACvC,SAAkB,IAAIC,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OAAI,aAAa,cAAcC,wBAAAA,0BAA0B,EAAE,EAAE;AAC3D,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,SAAA,GAAA,2BAAA,iBAAsBC,UAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAASC,YAAAA,UAAU,MAAMC,UAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAMC,iBAAI,MAAM,MAAM;IACtB,MAAM,QAAQC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAKC,YAAAA,cAAc,KAAK,QAAQ,KAAK,UAAA,GAAA,2BAAA,UAAkB,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAMH,iBAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQH,UAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMG,iBAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAMA,iBAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}
+{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (\n        e instanceof URLAborted ||\n        isDownloadNetworkError400(e) ||\n        this.signalCtl.signal.aborted\n      ) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAIA,2BAAAA,gBAAgB;CACvC,SAAkB,IAAIC,2BAAAA,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OACE,aAAa,cACbC,wBAAAA,0BAA0B,EAAE,IAC5B,KAAK,UAAU,OAAO,SACtB;AACA,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,SAAA,GAAA,2BAAA,iBAAsBC,UAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,OAAA,GAAA,2BAAA,YAAiB,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAASC,YAAAA,UAAU,MAAMC,UAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,UAAA,GAAA,2BAAA,sBAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAMC,iBAAI,MAAM,MAAM;IACtB,MAAM,QAAQC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAKC,YAAAA,cAAc,KAAK,QAAQ,KAAK,UAAA,GAAA,2BAAA,UAAkB,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAMH,iBAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQH,UAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAMG,iBAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAMA,iBAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}

package/dist/drivers/download_url/task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_url/task.ts"],"mappings":";;;;;;AAqBA;AAAA,cAAa,iBAAA;EAAA,iBASQ,MAAA;EAAA,SACR,IAAA;EAAA,SACA,GAAA,EAAK,GAAA;EAAA,SACL,MAAA,EAAQ,MAAA;EAAA,SACR,OAAA;EAAA,SAZF,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;cAGmB,MAAA,EAAQ,QAAA,EAChB,IAAA,UACA,GAAA,EAAK,GAAA,EACL,MAAA,EAAQ,MAAA,EACR,OAAA;EAGJ,IAAA,CAAA;;;;;;;EAUP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAS,cAAA,EAAgB,oBAAA,EAAsB,UAAA,YAAmB,OAAA;EAAA,QAkB1D,gBAAA;EAmCd,MAAA,CAAA,GAAU,SAAA;EAAA,QAWF,OAAA;EAAA,QAKA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA"}
+{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_url/task.ts"],"mappings":";;;;;;AAqBA;AAAA,cAAa,iBAAA;EAAA,iBASQ,MAAA;EAAA,SACR,IAAA;EAAA,SACA,GAAA,EAAK,GAAA;EAAA,SACL,MAAA,EAAQ,MAAA;EAAA,SACR,OAAA;EAAA,SAZF,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;cAGmB,MAAA,EAAQ,QAAA,EAChB,IAAA,UACA,GAAA,EAAK,GAAA,EACL,MAAA,EAAQ,MAAA,EACR,OAAA;EAGJ,IAAA,CAAA;;;;;;;EAUP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAS,cAAA,EAAgB,oBAAA,EAAsB,UAAA,YAAmB,OAAA;EAAA,QAsB1D,gBAAA;EAmCd,MAAA,CAAA,GAAU,SAAA;EAAA,QAWF,OAAA;EAAA,QAKA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA"}

package/dist/drivers/download_url/task.js

@@ -42,7 +42,7 @@ var DownloadByUrlTask = class {
 			this.setDone(size);
 			this.change.markChanged(`download of ${this.url} finished`);
 		} catch (e) {
-			if (e instanceof URLAborted || isDownloadNetworkError400(e)) {
+			if (e instanceof URLAborted || isDownloadNetworkError400(e) || this.signalCtl.signal.aborted) {
 				this.setError(e);
 				this.change.markChanged(`download of ${this.url} failed`);
 				await rmRFDir(this.path);

package/dist/drivers/download_url/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (e instanceof URLAborted || isDownloadNetworkError400(e)) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OAAI,aAAa,cAAc,0BAA0B,EAAE,EAAE;AAC3D,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,QAAM,gBAAgBA,OAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,MAAM,WAAW,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAM,IAAI,MAAM,MAAM;IACtB,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAK,cAAc,KAAK,QAAQ,KAAK,SAAS,SAAS,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQA,OAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}
+{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n  CallersCounter,\n  createPathAtomically,\n  ensureDirExists,\n  fileExists,\n  notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Downloads and extracts an archive to a directory. */\nexport class DownloadByUrlTask {\n  readonly counter = new CallersCounter();\n  readonly change = new ChangeSource();\n  private readonly signalCtl = new AbortController();\n  error: string | undefined;\n  done = false;\n  size = 0;\n\n  constructor(\n    private readonly logger: MiLogger,\n    readonly path: string,\n    readonly url: URL,\n    readonly signer: Signer,\n    readonly saveDir: string,\n  ) {}\n\n  public info() {\n    return {\n      url: this.url.toString(),\n      path: this.path,\n      done: this.done,\n      size: this.size,\n      error: this.error,\n    };\n  }\n\n  attach(w: Watcher, callerId: string) {\n    this.counter.inc(callerId);\n    if (!this.done) this.change.attachWatcher(w);\n  }\n\n  async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n    try {\n      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n      this.setDone(size);\n      this.change.markChanged(`download of ${this.url} finished`);\n    } catch (e: unknown) {\n      if (\n        e instanceof URLAborted ||\n        isDownloadNetworkError400(e) ||\n        this.signalCtl.signal.aborted\n      ) {\n        this.setError(e);\n        this.change.markChanged(`download of ${this.url} failed`);\n        // Just in case we were half-way extracting an archive.\n        await rmRFDir(this.path);\n        return;\n      }\n\n      throw e;\n    }\n  }\n\n  private async downloadAndUntar(\n    clientDownload: RemoteFileDownloader,\n    withGunzip: boolean,\n    signal: AbortSignal,\n  ): Promise<number> {\n    await ensureDirExists(path.dirname(this.path));\n\n    if (await fileExists(this.path)) {\n      return await dirSize(this.path);\n    }\n\n    const size = await clientDownload.withContent(\n      this.url.toString(),\n      {},\n      { signal },\n      async (content, size) => {\n        let processedContent = content;\n        if (withGunzip) {\n          const gunzip = Transform.toWeb(zlib.createGunzip());\n          processedContent = content.pipeThrough(gunzip, { signal });\n        }\n\n        await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n          await fsp.mkdir(fPath); // throws if a directory already exists.\n          const untar = Writable.toWeb(tar.extract(fPath));\n          await processedContent.pipeTo(untar, { signal });\n        });\n\n        return size;\n      },\n    );\n\n    return size;\n  }\n\n  getUrl(): UrlResult | undefined {\n    if (this.done)\n      return {\n        url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n      };\n\n    if (this.error) return { error: this.error };\n\n    return undefined;\n  }\n\n  private setDone(size: number) {\n    this.done = true;\n    this.size = size;\n  }\n\n  private setError(e: any) {\n    this.error = String(e);\n  }\n\n  abort(reason: string) {\n    this.signalCtl.abort(new URLAborted(reason));\n  }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n  name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n  const files = await fsp.readdir(dir, { withFileTypes: true });\n  const sizes = await Promise.all(\n    files.map(async (file: any) => {\n      const fPath = path.join(dir, file.name);\n\n      if (file.isDirectory()) return await dirSize(fPath);\n\n      const stat = await fsp.stat(fPath);\n      return stat.size;\n    }),\n  );\n\n  return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n  await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;AAqBA,IAAa,oBAAb,MAA+B;CAC7B,UAAmB,IAAI,gBAAgB;CACvC,SAAkB,IAAI,cAAc;CACpC,YAA6B,IAAI,iBAAiB;CAClD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,KACA,QACA,SACA;AALiB,OAAA,SAAA;AACR,OAAA,OAAA;AACA,OAAA,MAAA;AACA,OAAA,SAAA;AACA,OAAA,UAAA;;CAGX,OAAc;AACZ,SAAO;GACL,KAAK,KAAK,IAAI,UAAU;GACxB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;GACb;;CAGH,OAAO,GAAY,UAAkB;AACnC,OAAK,QAAQ,IAAI,SAAS;AAC1B,MAAI,CAAC,KAAK,KAAM,MAAK,OAAO,cAAc,EAAE;;CAG9C,MAAM,SAAS,gBAAsC,YAAqB;AACxE,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,iBAAiB,gBAAgB,YAAY,KAAK,UAAU,OAAO;AAC3F,QAAK,QAAQ,KAAK;AAClB,QAAK,OAAO,YAAY,eAAe,KAAK,IAAI,WAAW;WACpD,GAAY;AACnB,OACE,aAAa,cACb,0BAA0B,EAAE,IAC5B,KAAK,UAAU,OAAO,SACtB;AACA,SAAK,SAAS,EAAE;AAChB,SAAK,OAAO,YAAY,eAAe,KAAK,IAAI,SAAS;AAEzD,UAAM,QAAQ,KAAK,KAAK;AACxB;;AAGF,SAAM;;;CAIV,MAAc,iBACZ,gBACA,YACA,QACiB;AACjB,QAAM,gBAAgBA,OAAK,QAAQ,KAAK,KAAK,CAAC;AAE9C,MAAI,MAAM,WAAW,KAAK,KAAK,CAC7B,QAAO,MAAM,QAAQ,KAAK,KAAK;AAwBjC,SArBa,MAAM,eAAe,YAChC,KAAK,IAAI,UAAU,EACnB,EAAE,EACF,EAAE,QAAQ,EACV,OAAO,SAAS,SAAS;GACvB,IAAI,mBAAmB;AACvB,OAAI,YAAY;IACd,MAAM,SAAS,UAAU,MAAM,KAAK,cAAc,CAAC;AACnD,uBAAmB,QAAQ,YAAY,QAAQ,EAAE,QAAQ,CAAC;;AAG5D,SAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;AAC1E,UAAM,IAAI,MAAM,MAAM;IACtB,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,MAAM,CAAC;AAChD,UAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC;KAChD;AAEF,UAAO;IAEV;;CAKH,SAAgC;AAC9B,MAAI,KAAK,KACP,QAAO,EACL,KAAK,cAAc,KAAK,QAAQ,KAAK,SAAS,SAAS,KAAK,KAAK,CAAC,EACnE;AAEH,MAAI,KAAK,MAAO,QAAO,EAAE,OAAO,KAAK,OAAO;;CAK9C,QAAgB,MAAc;AAC5B,OAAK,OAAO;AACZ,OAAK,OAAO;;CAGd,SAAiB,GAAQ;AACvB,OAAK,QAAQ,OAAO,EAAE;;CAGxB,MAAM,QAAgB;AACpB,OAAK,UAAU,MAAM,IAAI,WAAW,OAAO,CAAC;;;;AAKhD,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;;;AAIT,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,MAAM,CAAC;AAY7D,SAXc,MAAM,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQA,OAAK,KAAK,KAAK,KAAK,KAAK;AAEvC,MAAI,KAAK,aAAa,CAAE,QAAO,MAAM,QAAQ,MAAM;AAGnD,UADa,MAAM,IAAI,KAAK,MAAM,EACtB;GACZ,CACH,EAEY,QAAQ,KAAU,SAAc,MAAM,MAAM,EAAE;;;AAI7D,eAAsB,QAAQ,MAAc;AAC1C,OAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;EAAM,CAAC"}

package/dist/drivers/helpers/download_remote_handle.cjs

@@ -2,9 +2,11 @@ require("../../_virtual/_rolldown/runtime.cjs");
 const require_types = require("../types.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 //#region src/drivers/helpers/download_remote_handle.ts
-const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<size>\d+?))#(?<signature>.*)$/;
+const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<resourceSig>[A-Za-z0-9_-]*)\/(?<size>\d+?))#(?<signature>.*)$/;
 function newRemoteHandle(rInfo, signer) {
-	let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${require_types.getSize(rInfo)}`;
+	const { globalId, signature: resourceSig } = (0, _milaboratories_pl_client.parseSignedResourceId)(rInfo.id);
+	const sigSegment = (0, _milaboratories_pl_client.signatureToBase64Url)(resourceSig);
+	let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${require_types.getSize(rInfo)}`;
 	return `blob+remote://download/${content}#${signer.sign(content)}`;
 }
 function isRemoteBlobHandle(handle) {
@@ -13,11 +15,12 @@ function isRemoteBlobHandle(handle) {
 function parseRemoteHandle(handle, signer) {
 	const parsed = handle.match(remoteHandleRegex);
 	if (parsed === null) throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);
-	const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups;
+	const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } = parsed.groups;
 	signer.verify(content, signature, `Signature verification failed for ${handle}`);
+	const sig = resourceSig ? (0, _milaboratories_pl_client.base64UrlToSignature)(resourceSig) : void 0;
 	return {
 		info: {
-			id: (0, _milaboratories_pl_client.bigintToResourceId)(BigInt(resourceId)),
+			id: (0, _milaboratories_pl_client.createSignedResourceId)(BigInt(resourceId), sig),
 			type: {
 				name: resourceType,
 				version: resourceVersion

package/dist/drivers/helpers/download_remote_handle.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport { bigintToResourceId } from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  return {\n    info: {\n      id: bigintToResourceId(BigInt(resourceId)),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;;AAWA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,OAAO,MAAM,GAAG,CAAC,GAAGA,cAAAA,QAAQ,MAAM;AAE5F,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,MAAM,cAAc,OAAO;AAEvF,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;AAEhF,QAAO;EACL,MAAM;GACJ,KAAA,GAAA,0BAAA,oBAAuB,OAAO,WAAW,CAAC;GAC1C,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}
+{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n  createSignedResourceId,\n  parseSignedResourceId,\n  signatureToBase64Url,\n  base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n  const sigSegment = signatureToBase64Url(resourceSig);\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n    parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n  return {\n    info: {\n      id: createSignedResourceId(BigInt(resourceId), sig),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,iBAAA,GAAA,0BAAA,uBAAsC,MAAM,GAAG;CAC5E,MAAM,cAAA,GAAA,0BAAA,sBAAkC,YAAY;CACpD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAGA,cAAAA,QAAQ,MAAM;AAElG,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;AAET,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;CAEhF,MAAM,MAAM,eAAA,GAAA,0BAAA,sBAAmC,YAAY,GAAG,KAAA;AAE9D,QAAO;EACL,MAAM;GACJ,KAAA,GAAA,0BAAA,wBAA2B,OAAO,WAAW,EAAE,IAAI;GACnD,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}

package/dist/drivers/helpers/download_remote_handle.js

@@ -1,9 +1,11 @@
 import { getSize } from "../types.js";
-import { bigintToResourceId } from "@milaboratories/pl-client";
+import { base64UrlToSignature, createSignedResourceId, parseSignedResourceId, signatureToBase64Url } from "@milaboratories/pl-client";
 //#region src/drivers/helpers/download_remote_handle.ts
-const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<size>\d+?))#(?<signature>.*)$/;
+const remoteHandleRegex = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<resourceSig>[A-Za-z0-9_-]*)\/(?<size>\d+?))#(?<signature>.*)$/;
 function newRemoteHandle(rInfo, signer) {
-	let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;
+	const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);
+	const sigSegment = signatureToBase64Url(resourceSig);
+	let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;
 	return `blob+remote://download/${content}#${signer.sign(content)}`;
 }
 function isRemoteBlobHandle(handle) {
@@ -12,11 +14,12 @@ function isRemoteBlobHandle(handle) {
 function parseRemoteHandle(handle, signer) {
 	const parsed = handle.match(remoteHandleRegex);
 	if (parsed === null) throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);
-	const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups;
+	const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } = parsed.groups;
 	signer.verify(content, signature, `Signature verification failed for ${handle}`);
+	const sig = resourceSig ? base64UrlToSignature(resourceSig) : void 0;
 	return {
 		info: {
-			id: bigintToResourceId(BigInt(resourceId)),
+			id: createSignedResourceId(BigInt(resourceId), sig),
 			type: {
 				name: resourceType,
 				version: resourceVersion

package/dist/drivers/helpers/download_remote_handle.js.map

@@ -1 +1 @@
-{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport { bigintToResourceId } from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  return {\n    info: {\n      id: bigintToResourceId(BigInt(resourceId)),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;AAWA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,OAAO,MAAM,GAAG,CAAC,GAAG,QAAQ,MAAM;AAE5F,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,MAAM,cAAc,OAAO;AAEvF,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;AAEhF,QAAO;EACL,MAAM;GACJ,IAAI,mBAAmB,OAAO,WAAW,CAAC;GAC1C,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}
+{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n  createSignedResourceId,\n  parseSignedResourceId,\n  signatureToBase64Url,\n  base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n  /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n  rInfo: OnDemandBlobResourceSnapshot,\n  signer: Signer,\n): RemoteBlobHandle {\n  const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n  const sigSegment = signatureToBase64Url(resourceSig);\n  let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n  return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n  return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n  handle: RemoteBlobHandle,\n  signer: Signer,\n): {\n  info: ResourceInfo;\n  size: number;\n} {\n  const parsed = handle.match(remoteHandleRegex);\n  if (parsed === null) {\n    throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n  }\n\n  const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n    parsed.groups!;\n\n  signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n  const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n  return {\n    info: {\n      id: createSignedResourceId(BigInt(resourceId), sig),\n      type: { name: resourceType, version: resourceVersion },\n    },\n    size: Number(size),\n  };\n}\n"],"mappings":";;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,gBAAgB,sBAAsB,MAAM,GAAG;CAC5E,MAAM,aAAa,qBAAqB,YAAY;CACpD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,MAAM;AAElG,QAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,QAAQ;;AAGlE,SAAgB,mBAAmB,QAA4C;AAC7E,QAAO,QAAQ,OAAO,MAAM,kBAAkB,CAAC;;AAGjD,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,kBAAkB;AAC9C,KAAI,WAAW,KACb,OAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,SAAS;CAG9E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;AAET,QAAO,OAAO,SAAS,WAAW,qCAAqC,SAAS;CAEhF,MAAM,MAAM,cAAc,qBAAqB,YAAY,GAAG,KAAA;AAE9D,QAAO;EACL,MAAM;GACJ,IAAI,uBAAuB,OAAO,WAAW,EAAE,IAAI;GACnD,MAAM;IAAE,MAAM;IAAc,SAAS;IAAiB;GACvD;EACD,MAAM,OAAO,KAAK;EACnB"}