@milaboratories/pl-drivers 1.12.7 → 1.12.9

package/dist/drivers/upload.js.map

@@ -1 +1 @@
-{"version":3,"file":"upload.js","names":[],"sources":["../../src/drivers/upload.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport type { Watcher, ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ClientProgress } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport type { PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n} from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { IndexResourceSnapshot, UploadResourceSnapshot } from \"./types\";\nimport { nonRecoverableError, UploadTask } from \"./upload_task\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport function makeBlobImportSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): ImportResourceSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  if (node.resourceType.name.startsWith(\"BlobUpload\"))\n    return makeResourceSnapshot(node, UploadResourceSnapshot);\n  return makeResourceSnapshot(node, IndexResourceSnapshot);\n}\n\nexport type UploadDriverOps = PollingOps & {\n  /** How much parts of a file can be multipart-uploaded to S3 at once. */\n  nConcurrentPartUploads: number;\n  /** How much upload/indexing statuses of blobs can the driver ask\n   * from the platform gRPC at once. */\n  nConcurrentGetProgresses: number;\n};\n\n// TODO: add abort signal to Upload Tasks.\n\n/** Uploads blobs in a queue and holds counters, so it can stop not-needed\n * uploads.\n * Handles both Index and Upload blobs,\n * the client needs to pass concrete blobs from `handle` field. */\nexport class UploadDriver {\n  private readonly idToProgress: Map<ResourceId, UploadTask> = new Map();\n\n  /** Holds a queue that upload blobs. */\n  private readonly uploadQueue: TaskProcessor;\n  private readonly hooks: PollingComputableHooks;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly opts: UploadDriverOps = {\n      nConcurrentPartUploads: 10,\n      nConcurrentGetProgresses: 10,\n      pollingInterval: 1000,\n      stopPollingDelay: 1000,\n    },\n  ) {\n    this.uploadQueue = new TaskProcessor(this.logger, 1, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 20,\n      maxDelay: 15000, // 15 seconds\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n\n    this.hooks = new PollingComputableHooks(\n      () => this.startUpdating(),\n      () => this.stopUpdating(),\n      { stopDebounce: opts.stopPollingDelay },\n      (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n    );\n  }\n\n  /** Returns a progress id and schedules an upload task if it's necessary. */\n  getProgressId(\n    handleResource: ImportResourceSnapshot | PlTreeEntry,\n  ): Computable<sdk.ImportProgress>;\n  getProgressId(\n    handleResource: ImportResourceSnapshot | PlTreeEntry,\n    ctx: ComputableCtx,\n  ): sdk.ImportProgress;\n  getProgressId(\n    handleResource: ImportResourceSnapshot | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<sdk.ImportProgress> | sdk.ImportProgress {\n    if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));\n\n    const rInfo: ImportResourceSnapshot = isPlTreeEntry(handleResource)\n      ? makeBlobImportSnapshot(handleResource, ctx)\n      : handleResource;\n\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.release(rInfo.id, callerId));\n\n    const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);\n\n    return result;\n  }\n\n  private getProgressIdNoCtx(\n    w: Watcher,\n    res: ImportResourceSnapshot,\n    callerId: string,\n  ): sdk.ImportProgress {\n    validateResourceType(\"getProgressId\", res.type);\n\n    const task = this.idToProgress.get(res.id);\n\n    if (task != undefined) {\n      task.setDoneIfOutputSet(res);\n      return task.getProgress(w, callerId);\n    }\n\n    const newTask = new UploadTask(\n      this.logger,\n      this.clientBlob,\n      this.clientProgress,\n      this.opts.nConcurrentPartUploads,\n      this.signer,\n      res,\n    );\n\n    this.idToProgress.set(res.id, newTask);\n\n    if (newTask.shouldScheduleUpload()) {\n      this.uploadQueue.push({\n        fn: () => newTask.uploadBlobTask(),\n        recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n      });\n    }\n\n    newTask.setDoneIfOutputSet(res);\n    return newTask.getProgress(w, callerId);\n  }\n\n  /** Decrement counters for the file and remove an uploading if counter == 0. */\n  private async release(id: ResourceId, callerId: string) {\n    const task = this.idToProgress.get(id);\n    if (task === undefined) return;\n\n    const deleted = task.decCounter(callerId);\n    if (deleted) this.idToProgress.delete(id);\n  }\n\n  /** Must be called when the driver is closing. */\n  public async releaseAll() {\n    this.uploadQueue.stop();\n  }\n\n  private scheduledOnNextState: ScheduledRefresh[] = [];\n\n  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n    this.scheduledOnNextState.push({ resolve, reject });\n  }\n\n  /** Called from observer */\n  private startUpdating(): void {\n    this.keepRunning = true;\n    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n  }\n\n  /** Called from observer */\n  private stopUpdating(): void {\n    this.keepRunning = false;\n  }\n\n  /** Stops polling loop and waits for it to finish */\n  public async terminate(): Promise<void> {\n    this.stopUpdating();\n    if (this.currentLoop !== undefined) {\n      await this.currentLoop;\n    }\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.terminate();\n  }\n\n  /** If true, main loop will continue polling pl state. */\n  private keepRunning = false;\n  /** Actual state of main loop. */\n  private currentLoop: Promise<void> | undefined = undefined;\n\n  private async mainLoop() {\n    while (this.keepRunning) {\n      const toNotify = this.scheduledOnNextState;\n      this.scheduledOnNextState = [];\n\n      try {\n        await asyncPool(\n          this.opts.nConcurrentGetProgresses,\n          this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),\n        );\n\n        toNotify.forEach((n) => n.resolve());\n      } catch (e: any) {\n        console.error(e);\n        toNotify.forEach((n) => n.reject(e));\n      }\n\n      if (!this.keepRunning) break;\n      await scheduler.wait(this.opts.pollingInterval);\n    }\n\n    this.currentLoop = undefined;\n  }\n\n  private getAllNotDoneProgresses(): Array<UploadTask> {\n    return Array.from(this.idToProgress.entries())\n      .filter(([_, p]) => !isProgressDone(p.progress))\n      .map(([_, p]) => p);\n  }\n}\n\nfunction isProgressDone(p: sdk.ImportProgress) {\n  return p.done && (p.status?.progress ?? 0.0) >= 1.0;\n}\n\ntype ScheduledRefresh = {\n  resolve: () => void;\n  reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith(\"BlobUpload\") && !rType.name.startsWith(\"BlobIndex\")) {\n    throw new WrongResourceTypeError(\n      `${methodName}: wrong resource type: ${rType.name}, ` +\n        `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;AAsBA,SAAgB,uBACd,iBACA,KACwB;CACxB,MAAM,OAAO,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB;AAEN,KAAI,KAAK,aAAa,KAAK,WAAW,aAAa,CACjD,QAAO,qBAAqB,MAAM,uBAAuB;AAC3D,QAAO,qBAAqB,MAAM,sBAAsB;;;;;;AAiB1D,IAAa,eAAb,MAA0B;CACxB,AAAiB,+BAA4C,IAAI,KAAK;;CAGtE,AAAiB;CACjB,AAAiB;CAEjB,YACE,AAAiB,QACjB,AAAiB,QACjB,AAAiB,YACjB,AAAiB,gBACjB,AAAiB,OAAwB;EACvC,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB,EACD;EAViB;EACA;EACA;EACA;EACA;AAOjB,OAAK,cAAc,IAAI,cAAc,KAAK,QAAQ,GAAG;GACnD,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AAEF,OAAK,QAAQ,IAAI,6BACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CAWH,cACE,gBACA,KACqD;AACrD,MAAI,OAAO,OAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,cAAc,gBAAgB,IAAI,CAAC;EAE9F,MAAM,QAAgC,cAAc,eAAe,GAC/D,uBAAuB,gBAAgB,IAAI,GAC3C;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,QAAQ,MAAM,IAAI,SAAS,CAAC;AAIxD,SAFe,KAAK,mBAAmB,IAAI,SAAS,OAAO,SAAS;;CAKtE,AAAQ,mBACN,GACA,KACA,UACoB;AACpB,uBAAqB,iBAAiB,IAAI,KAAK;EAE/C,MAAM,OAAO,KAAK,aAAa,IAAI,IAAI,GAAG;AAE1C,MAAI,QAAQ,QAAW;AACrB,QAAK,mBAAmB,IAAI;AAC5B,UAAO,KAAK,YAAY,GAAG,SAAS;;EAGtC,MAAM,UAAU,IAAI,WAClB,KAAK,QACL,KAAK,YACL,KAAK,gBACL,KAAK,KAAK,wBACV,KAAK,QACL,IACD;AAED,OAAK,aAAa,IAAI,IAAI,IAAI,QAAQ;AAEtC,MAAI,QAAQ,sBAAsB,CAChC,MAAK,YAAY,KAAK;GACpB,UAAU,QAAQ,gBAAgB;GAClC,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAGJ,UAAQ,mBAAmB,IAAI;AAC/B,SAAO,QAAQ,YAAY,GAAG,SAAS;;;CAIzC,MAAc,QAAQ,IAAgB,UAAkB;EACtD,MAAM,OAAO,KAAK,aAAa,IAAI,GAAG;AACtC,MAAI,SAAS,OAAW;AAGxB,MADgB,KAAK,WAAW,SAAS,CAC5B,MAAK,aAAa,OAAO,GAAG;;;CAI3C,MAAa,aAAa;AACxB,OAAK,YAAY,MAAM;;CAGzB,AAAQ,uBAA2C,EAAE;CAErD,AAAQ,oBAAoB,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,AAAQ,gBAAsB;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,OAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,AAAQ,eAAqB;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,OACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,AAAQ,cAAc;;CAEtB,AAAQ,cAAyC;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;AACF,UAAM,UACJ,KAAK,KAAK,0BACV,KAAK,yBAAyB,CAAC,KAAK,MAAM,YAAY,MAAM,EAAE,cAAc,CAAC,CAC9E;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAM,UAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc;;CAGrB,AAAQ,0BAA6C;AACnD,SAAO,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAC3C,QAAQ,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,SAAS,CAAC,CAC/C,KAAK,CAAC,GAAG,OAAO,EAAE;;;AAIzB,SAAS,eAAe,GAAuB;AAC7C,QAAO,EAAE,SAAS,EAAE,QAAQ,YAAY,MAAQ;;AAQlD,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,aAAa,IAAI,CAAC,MAAM,KAAK,WAAW,YAAY,CAC7E,OAAM,IAAI,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,oEAEnD"}
+{"version":3,"file":"upload.js","names":[],"sources":["../../src/drivers/upload.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type { ResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport type { Watcher, ComputableCtx } from \"@milaboratories/computable\";\nimport { Computable, PollingComputableHooks } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { ClientProgress } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport type { PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport {\n  isPlTreeEntry,\n  isPlTreeEntryAccessor,\n  makeResourceSnapshot,\n} from \"@milaboratories/pl-tree\";\nimport { scheduler } from \"node:timers/promises\";\nimport type { PollingOps } from \"./helpers/polling_ops\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { IndexResourceSnapshot, UploadResourceSnapshot } from \"./types\";\nimport { nonRecoverableError, UploadTask } from \"./upload_task\";\nimport { WrongResourceTypeError } from \"./helpers/helpers\";\n\nexport function makeBlobImportSnapshot(\n  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,\n  ctx: ComputableCtx,\n): ImportResourceSnapshot {\n  const node = isPlTreeEntry(entryOrAccessor)\n    ? ctx.accessor(entryOrAccessor).node()\n    : isPlTreeEntryAccessor(entryOrAccessor)\n      ? entryOrAccessor.node()\n      : entryOrAccessor;\n\n  if (node.resourceType.name.startsWith(\"BlobUpload\"))\n    return makeResourceSnapshot(node, UploadResourceSnapshot);\n  return makeResourceSnapshot(node, IndexResourceSnapshot);\n}\n\nexport type UploadDriverOps = PollingOps & {\n  /** How much parts of a file can be multipart-uploaded to S3 at once. */\n  nConcurrentPartUploads: number;\n  /** How much upload/indexing statuses of blobs can the driver ask\n   * from the platform gRPC at once. */\n  nConcurrentGetProgresses: number;\n};\n\n// TODO: add abort signal to Upload Tasks.\n\n/** Uploads blobs in a queue and holds counters, so it can stop not-needed\n * uploads.\n * Handles both Index and Upload blobs,\n * the client needs to pass concrete blobs from `handle` field. */\nexport class UploadDriver {\n  private readonly idToProgress: Map<ResourceId, UploadTask> = new Map();\n\n  /** Holds a queue that upload blobs. */\n  private readonly uploadQueue: TaskProcessor;\n  private readonly hooks: PollingComputableHooks;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly signer: Signer,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly opts: UploadDriverOps = {\n      nConcurrentPartUploads: 10,\n      nConcurrentGetProgresses: 10,\n      pollingInterval: 1000,\n      stopPollingDelay: 1000,\n    },\n  ) {\n    this.uploadQueue = new TaskProcessor(this.logger, 1, {\n      type: \"exponentialWithMaxDelayBackoff\",\n      initialDelay: 20,\n      maxDelay: 15000, // 15 seconds\n      backoffMultiplier: 1.5,\n      jitter: 0.5,\n    });\n\n    this.hooks = new PollingComputableHooks(\n      () => this.startUpdating(),\n      () => this.stopUpdating(),\n      { stopDebounce: opts.stopPollingDelay },\n      (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n    );\n  }\n\n  /** Returns a progress id and schedules an upload task if it's necessary. */\n  getProgressId(\n    handleResource: ImportResourceSnapshot | PlTreeEntry,\n  ): Computable<sdk.ImportProgress>;\n  getProgressId(\n    handleResource: ImportResourceSnapshot | PlTreeEntry,\n    ctx: ComputableCtx,\n  ): sdk.ImportProgress;\n  getProgressId(\n    handleResource: ImportResourceSnapshot | PlTreeEntry,\n    ctx?: ComputableCtx,\n  ): Computable<sdk.ImportProgress> | sdk.ImportProgress {\n    if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));\n\n    const rInfo: ImportResourceSnapshot = isPlTreeEntry(handleResource)\n      ? makeBlobImportSnapshot(handleResource, ctx)\n      : handleResource;\n\n    const callerId = randomUUID();\n    ctx.attacheHooks(this.hooks);\n    ctx.addOnDestroy(() => this.release(rInfo.id, callerId));\n\n    const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);\n\n    return result;\n  }\n\n  private getProgressIdNoCtx(\n    w: Watcher,\n    res: ImportResourceSnapshot,\n    callerId: string,\n  ): sdk.ImportProgress {\n    validateResourceType(\"getProgressId\", res.type);\n\n    const task = this.idToProgress.get(res.id);\n\n    if (task != undefined) {\n      task.setDoneIfOutputSet(res);\n      return task.getProgress(w, callerId);\n    }\n\n    const newTask = new UploadTask(\n      this.logger,\n      this.clientBlob,\n      this.clientProgress,\n      this.opts.nConcurrentPartUploads,\n      this.signer,\n      res,\n    );\n\n    this.idToProgress.set(res.id, newTask);\n\n    if (newTask.shouldScheduleUpload()) {\n      this.uploadQueue.push({\n        fn: () => newTask.uploadBlobTask(),\n        recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n      });\n    }\n\n    newTask.setDoneIfOutputSet(res);\n    return newTask.getProgress(w, callerId);\n  }\n\n  /** Decrement counters for the file and remove an uploading if counter == 0. */\n  private async release(id: ResourceId, callerId: string) {\n    const task = this.idToProgress.get(id);\n    if (task === undefined) return;\n\n    const deleted = task.decCounter(callerId);\n    if (deleted) this.idToProgress.delete(id);\n  }\n\n  /** Must be called when the driver is closing. */\n  public async releaseAll() {\n    this.uploadQueue.stop();\n  }\n\n  private scheduledOnNextState: ScheduledRefresh[] = [];\n\n  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n    this.scheduledOnNextState.push({ resolve, reject });\n  }\n\n  /** Called from observer */\n  private startUpdating(): void {\n    this.keepRunning = true;\n    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n  }\n\n  /** Called from observer */\n  private stopUpdating(): void {\n    this.keepRunning = false;\n  }\n\n  /** Stops polling loop and waits for it to finish */\n  public async terminate(): Promise<void> {\n    this.stopUpdating();\n    if (this.currentLoop !== undefined) {\n      await this.currentLoop;\n    }\n  }\n\n  async [Symbol.asyncDispose](): Promise<void> {\n    await this.terminate();\n  }\n\n  /** If true, main loop will continue polling pl state. */\n  private keepRunning = false;\n  /** Actual state of main loop. */\n  private currentLoop: Promise<void> | undefined = undefined;\n\n  private async mainLoop() {\n    while (this.keepRunning) {\n      const toNotify = this.scheduledOnNextState;\n      this.scheduledOnNextState = [];\n\n      try {\n        await asyncPool(\n          this.opts.nConcurrentGetProgresses,\n          this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),\n        );\n\n        toNotify.forEach((n) => n.resolve());\n      } catch (e: any) {\n        console.error(e);\n        toNotify.forEach((n) => n.reject(e));\n      }\n\n      if (!this.keepRunning) break;\n      await scheduler.wait(this.opts.pollingInterval);\n    }\n\n    this.currentLoop = undefined;\n  }\n\n  private getAllNotDoneProgresses(): Array<UploadTask> {\n    return Array.from(this.idToProgress.entries())\n      .filter(([_, p]) => !isProgressDone(p.progress))\n      .map(([_, p]) => p);\n  }\n}\n\nfunction isProgressDone(p: sdk.ImportProgress) {\n  return p.done && (p.status?.progress ?? 0.0) >= 1.0;\n}\n\ntype ScheduledRefresh = {\n  resolve: () => void;\n  reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n  if (!rType.name.startsWith(\"BlobUpload\") && !rType.name.startsWith(\"BlobIndex\")) {\n    throw new WrongResourceTypeError(\n      `${methodName}: wrong resource type: ${rType.name}, ` +\n        `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,\n    );\n  }\n}\n"],"mappings":";;;;;;;;;AAsBA,SAAgB,uBACd,iBACA,KACwB;CACxB,MAAM,OAAO,cAAc,gBAAgB,GACvC,IAAI,SAAS,gBAAgB,CAAC,MAAM,GACpC,sBAAsB,gBAAgB,GACpC,gBAAgB,MAAM,GACtB;AAEN,KAAI,KAAK,aAAa,KAAK,WAAW,aAAa,CACjD,QAAO,qBAAqB,MAAM,uBAAuB;AAC3D,QAAO,qBAAqB,MAAM,sBAAsB;;;;;;AAiB1D,IAAa,eAAb,MAA0B;CACxB,+BAA6D,IAAI,KAAK;;CAGtE;CACA;CAEA,YACE,QACA,QACA,YACA,gBACA,OAAyC;EACvC,wBAAwB;EACxB,0BAA0B;EAC1B,iBAAiB;EACjB,kBAAkB;EACnB,EACD;AAViB,OAAA,SAAA;AACA,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,iBAAA;AACA,OAAA,OAAA;AAOjB,OAAK,cAAc,IAAI,cAAc,KAAK,QAAQ,GAAG;GACnD,MAAM;GACN,cAAc;GACd,UAAU;GACV,mBAAmB;GACnB,QAAQ;GACT,CAAC;AAEF,OAAK,QAAQ,IAAI,6BACT,KAAK,eAAe,QACpB,KAAK,cAAc,EACzB,EAAE,cAAc,KAAK,kBAAkB,GACtC,SAAS,WAAW,KAAK,oBAAoB,SAAS,OAAO,CAC/D;;CAWH,cACE,gBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,cAAc,gBAAgB,IAAI,CAAC;EAE9F,MAAM,QAAgC,cAAc,eAAe,GAC/D,uBAAuB,gBAAgB,IAAI,GAC3C;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,aAAa,KAAK,MAAM;AAC5B,MAAI,mBAAmB,KAAK,QAAQ,MAAM,IAAI,SAAS,CAAC;AAIxD,SAFe,KAAK,mBAAmB,IAAI,SAAS,OAAO,SAAS;;CAKtE,mBACE,GACA,KACA,UACoB;AACpB,uBAAqB,iBAAiB,IAAI,KAAK;EAE/C,MAAM,OAAO,KAAK,aAAa,IAAI,IAAI,GAAG;AAE1C,MAAI,QAAQ,KAAA,GAAW;AACrB,QAAK,mBAAmB,IAAI;AAC5B,UAAO,KAAK,YAAY,GAAG,SAAS;;EAGtC,MAAM,UAAU,IAAI,WAClB,KAAK,QACL,KAAK,YACL,KAAK,gBACL,KAAK,KAAK,wBACV,KAAK,QACL,IACD;AAED,OAAK,aAAa,IAAI,IAAI,IAAI,QAAQ;AAEtC,MAAI,QAAQ,sBAAsB,CAChC,MAAK,YAAY,KAAK;GACpB,UAAU,QAAQ,gBAAgB;GAClC,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAGJ,UAAQ,mBAAmB,IAAI;AAC/B,SAAO,QAAQ,YAAY,GAAG,SAAS;;;CAIzC,MAAc,QAAQ,IAAgB,UAAkB;EACtD,MAAM,OAAO,KAAK,aAAa,IAAI,GAAG;AACtC,MAAI,SAAS,KAAA,EAAW;AAGxB,MADgB,KAAK,WAAW,SAAS,CAC5B,MAAK,aAAa,OAAO,GAAG;;;CAI3C,MAAa,aAAa;AACxB,OAAK,YAAY,MAAM;;CAGzB,uBAAmD,EAAE;CAErD,oBAA4B,SAAqB,QAAkC;AACjF,OAAK,qBAAqB,KAAK;GAAE;GAAS;GAAQ,CAAC;;;CAIrD,gBAA8B;AAC5B,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EAAW,MAAK,cAAc,KAAK,UAAU;;;CAIxE,eAA6B;AAC3B,OAAK,cAAc;;;CAIrB,MAAa,YAA2B;AACtC,OAAK,cAAc;AACnB,MAAI,KAAK,gBAAgB,KAAA,EACvB,OAAM,KAAK;;CAIf,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,WAAW;;;CAIxB,cAAsB;;CAEtB,cAAiD,KAAA;CAEjD,MAAc,WAAW;AACvB,SAAO,KAAK,aAAa;GACvB,MAAM,WAAW,KAAK;AACtB,QAAK,uBAAuB,EAAE;AAE9B,OAAI;AACF,UAAM,UACJ,KAAK,KAAK,0BACV,KAAK,yBAAyB,CAAC,KAAK,MAAM,YAAY,MAAM,EAAE,cAAc,CAAC,CAC9E;AAED,aAAS,SAAS,MAAM,EAAE,SAAS,CAAC;YAC7B,GAAQ;AACf,YAAQ,MAAM,EAAE;AAChB,aAAS,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC;;AAGtC,OAAI,CAAC,KAAK,YAAa;AACvB,SAAM,UAAU,KAAK,KAAK,KAAK,gBAAgB;;AAGjD,OAAK,cAAc,KAAA;;CAGrB,0BAAqD;AACnD,SAAO,MAAM,KAAK,KAAK,aAAa,SAAS,CAAC,CAC3C,QAAQ,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,SAAS,CAAC,CAC/C,KAAK,CAAC,GAAG,OAAO,EAAE;;;AAIzB,SAAS,eAAe,GAAuB;AAC7C,QAAO,EAAE,SAAS,EAAE,QAAQ,YAAY,MAAQ;;AAQlD,SAAS,qBAAqB,YAAoB,OAAqB;AACrE,KAAI,CAAC,MAAM,KAAK,WAAW,aAAa,IAAI,CAAC,MAAM,KAAK,WAAW,YAAY,CAC7E,OAAM,IAAI,uBACR,GAAG,WAAW,yBAAyB,MAAM,KAAK,oEAEnD"}

package/dist/drivers/upload_task.cjs

@@ -1,12 +1,11 @@
-const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
-const require_upload = require('../clients/upload.cjs');
-const require_types = require('./types.cjs');
+const require_runtime = require("../_virtual/_rolldown/runtime.cjs");
+const require_upload = require("../clients/upload.cjs");
+const require_types = require("./types.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let _milaboratories_computable = require("@milaboratories/computable");
 let node_assert = require("node:assert");
 node_assert = require_runtime.__toESM(node_assert);
-
 //#region src/drivers/upload_task.ts
 /** Holds all info needed to upload a file and a status of uploading
 * and indexing. Also, has a method to update a status of the progress.
@@ -255,7 +254,6 @@ function decreaseConcurrency(logger, current, min) {
 	if (newConcurrency != current) logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`);
 	return newConcurrency;
 }
-
 //#endregion
 exports.UploadTask = UploadTask;
 exports.isMyUpload = isMyUpload;
@@ -264,4 +262,5 @@ exports.isTerminalUploadError = isTerminalUploadError;
 exports.isUpload = isUpload;
 exports.nonRecoverableError = nonRecoverableError;
 exports.uploadBlob = uploadBlob;
+
 //# sourceMappingURL=upload_task.cjs.map

package/dist/drivers/upload_task.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"upload_task.cjs","names":["ChangeSource","CallersCounter","ImportFileHandleUploadData","MTimeError","UnexpectedEOF","NoFileForUploading","BadRequestError"],"sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport {\n  isTimeoutError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { AsyncPoolController, MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientProgress, ProgressStatus } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from \"../clients/upload\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { ImportFileHandleUploadData } from \"./types\";\nimport assert from \"node:assert\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n  private nMaxUploads: number;\n  private nPartsWithThisUploadSpeed = 0;\n  private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n  /** If this is upload progress this field will be defined */\n  private uploadData?: ImportFileHandleUploadData;\n  public progress: sdk.ImportProgress;\n\n  /** If failed, then getting a progress is terminally failed. */\n  public failed?: boolean;\n\n  /** True if the blob was existed.\n   * At this case, the task will show progress == 1.0. */\n  private alreadyExisted = false;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly maxNConcurrentPartsUpload: number,\n    signer: Signer,\n    public readonly res: ImportResourceSnapshot,\n  ) {\n    this.nMaxUploads = this.maxNConcurrentPartsUpload;\n    const { uploadData, progress } = newProgress(res, signer);\n    this.uploadData = uploadData;\n    this.progress = progress;\n  }\n\n  public getProgress(w: Watcher, callerId: string) {\n    this.incCounter(w, callerId);\n\n    if (this.failed) {\n      this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n      throw new Error(this.progress.lastError);\n    }\n\n    return cloneProgress(this.progress);\n  }\n\n  public shouldScheduleUpload(): boolean {\n    return isMyUpload(this.progress);\n  }\n\n  /** Uploads a blob if it's not BlobIndex. */\n  public async uploadBlobTask() {\n    try {\n      await uploadBlob(\n        this.logger,\n        this.clientBlob,\n        this.res,\n        this.uploadData!,\n        this.isComputableDone.bind(this),\n        {\n          nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n          nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n          currentSpeed: this.nMaxUploads,\n          maxSpeed: this.maxNConcurrentPartsUpload,\n        },\n      );\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n    } catch (e: any) {\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(`terminal error while uploading a blob: ${e}`);\n        this.change.markChanged(\n          `blob upload for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n\n        return;\n      }\n\n      this.logger.error(`error while uploading a blob: ${e}`);\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n      if (nonRecoverableError(e)) {\n        this.setTerminalError(e);\n        return;\n      }\n\n      this.setRetriableError(e);\n\n      if (isHeadersTimeoutError(e)) {\n        // we probably have a slow internet, we need to slow things a bit.\n        this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n      }\n\n      throw e;\n    }\n  }\n\n  public async updateStatus() {\n    try {\n      // we do it with timeout in case we have slow internet.\n      const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n      const oldStatus = this.progress.status;\n      const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n      this.progress.status = newStatus;\n      this.setDone(status.done);\n\n      if (status.done || this.progress.status.progress != oldStatus?.progress) {\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n      }\n    } catch (e: any) {\n      this.setRetriableError(e);\n\n      if (isTimeoutError(e)) {\n        this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);\n        return;\n      }\n\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(\n          `terminal error while updating BlobImport status: ${e}, ${stringifyWithResourceId(this.res)}`,\n        );\n        this.change.markChanged(\n          `upload status for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n        return;\n      }\n\n      this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n      // It was a terminal error, but when a connection drops,\n      // this will stop the whole task, so we make it retryable.\n      // It was like that:\n      // this.change.markChanged();\n      // this.setTerminalError(e);\n    }\n  }\n\n  /** Set non-terminal error, that task can be retried. */\n  private setRetriableError(e: unknown) {\n    this.progress.lastError = String(e);\n  }\n\n  /** Set a terminal error, the task will throw a error instead of a progress. */\n  private setTerminalError(e: unknown) {\n    this.progress.lastError = String(e);\n    this.progress.done = false;\n    this.failed = true;\n  }\n\n  public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n    if (isImportResourceOutputSet(res)) {\n      this.setDone(true);\n      this.alreadyExisted = true;\n    }\n  }\n\n  private setDone(done: boolean) {\n    this.progress.done = done;\n    if (done) this.progress.lastError = undefined;\n  }\n\n  public incCounter(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  public decCounter(callerId: string) {\n    return this.counter.dec(callerId);\n  }\n\n  private isComputableDone() {\n    return this.counter.isZero();\n  }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n  logger: MiLogger,\n  clientBlob: ClientUpload,\n  res: ResourceInfo,\n  uploadData: ImportFileHandleUploadData,\n  isDoneFn: () => boolean,\n  speed: {\n    nPartsWithThisUploadSpeed: number;\n    nPartsToIncreaseUpload: number;\n    currentSpeed: number;\n    maxSpeed: number;\n  },\n) {\n  assert(isUpload(res), \"the upload operation can be done only for BlobUploads\");\n  const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n  if (isDoneFn()) return;\n  const parts = await clientBlob.initUpload(res, { timeout });\n  logger.info(\n    `started to upload blob ${res.id},` +\n      ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},` +\n      ` number of concurrent uploads: ${speed.currentSpeed}`,\n  );\n\n  const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n    if (isDoneFn()) return;\n    await clientBlob.partUpload(\n      res,\n      uploadData.localPath,\n      BigInt(uploadData.modificationTime),\n      part,\n      parts.checksumAlgorithm,\n      parts.checksumHeader,\n      { timeout },\n    );\n    logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n    // if we had a network freeze, it will be increased slowly.\n    speed.nPartsWithThisUploadSpeed++;\n    if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n      speed.nPartsWithThisUploadSpeed = 0;\n      speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n      controller.setConcurrency(speed.currentSpeed);\n    }\n  };\n\n  await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n  if (isDoneFn()) return;\n  await clientBlob.finalize(res, { timeout });\n\n  logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n  let isUploadSignMatch: boolean | undefined;\n  let uploadData: ImportFileHandleUploadData | undefined;\n  if (isUpload(res)) {\n    uploadData = ImportFileHandleUploadData.parse(res.data);\n    isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n  }\n\n  return {\n    uploadData: uploadData,\n    progress: {\n      done: false,\n      status: undefined,\n      isUpload: isUpload(res),\n      isUploadSignMatch: isUploadSignMatch,\n      lastError: undefined,\n    } satisfies sdk.ImportProgress,\n  };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n  return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n  const cloned: sdk.ImportProgress = {\n    done: progress.done,\n    isUpload: progress.isUpload,\n    isUploadSignMatch: progress.isUploadSignMatch,\n    lastError: progress.lastError,\n  };\n\n  if (progress.status)\n    cloned.status = {\n      progress: progress.status.progress,\n      bytesProcessed: progress.status.bytesProcessed,\n      bytesTotal: progress.status.bytesTotal,\n    };\n\n  return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n  return \"blob\" in res.fields\n    ? res.fields.blob !== undefined\n    : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n  return res.type.name.startsWith(\"BlobUpload\");\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n  try {\n    signer.verify(path, signature);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n  return {\n    progress: proto.progress ?? 0,\n    bytesProcessed: Number(proto.bytesProcessed),\n    bytesTotal: Number(proto.bytesTotal),\n  };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n  if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n    return {\n      progress: 1.0,\n      bytesProcessed: Number(status.bytesTotal),\n      bytesTotal: Number(status.bytesTotal),\n    };\n  }\n\n  return status;\n}\n\nexport function isTerminalUploadError(e: any) {\n  if (e?.name !== \"RpcError\") return false;\n  switch (e.code) {\n    case \"NOT_FOUND\":\n    case \"ABORTED\":\n    case \"ALREADY_EXISTS\":\n      return true;\n    default:\n      return false;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof MTimeError ||\n    e instanceof UnexpectedEOF ||\n    e instanceof NoFileForUploading ||\n    e instanceof BadRequestError\n  );\n}\n\nfunction isHeadersTimeoutError(e: any) {\n  return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n  const newConcurrency = Math.min(current + 2, max);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n  const newConcurrency = Math.max(Math.round(current / 2), min);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n"],"mappings":";;;;;;;;;;;;;AAqBA,IAAa,aAAb,MAAwB;CACtB,AAAiB,SAAuB,IAAIA,yCAAc;CAC1D,AAAiB,UAA0B,IAAIC,2CAAgB;CAC/D,AAAQ;CACR,AAAQ,4BAA4B;CACpC,AAAQ,yBAAyB;;CAGjC,AAAQ;CACR,AAAO;;CAGP,AAAO;;;CAIP,AAAQ,iBAAiB;CAEzB,YACE,AAAiB,QACjB,AAAiB,YACjB,AAAiB,gBACjB,AAAiB,2BACjB,QACA,AAAgB,KAChB;EANiB;EACA;EACA;EACA;EAED;AAEhB,OAAK,cAAc,KAAK;EACxB,MAAM,EAAE,YAAY,aAAa,YAAY,KAAK,OAAO;AACzD,OAAK,aAAa;AAClB,OAAK,WAAW;;CAGlB,AAAO,YAAY,GAAY,UAAkB;AAC/C,OAAK,WAAW,GAAG,SAAS;AAE5B,MAAI,KAAK,QAAQ;AACf,QAAK,OAAO,MAAM,gCAAgC,KAAK,SAAS,YAAY;AAC5E,SAAM,IAAI,MAAM,KAAK,SAAS,UAAU;;AAG1C,SAAO,cAAc,KAAK,SAAS;;CAGrC,AAAO,uBAAgC;AACrC,SAAO,WAAW,KAAK,SAAS;;;CAIlC,MAAa,iBAAiB;AAC5B,MAAI;AACF,SAAM,WACJ,KAAK,QACL,KAAK,YACL,KAAK,KACL,KAAK,YACL,KAAK,iBAAiB,KAAK,KAAK,EAChC;IACE,2BAA2B,KAAK;IAChC,wBAAwB,KAAK;IAC7B,cAAc,KAAK;IACnB,UAAU,KAAK;IAChB,CACF;AACD,QAAK,OAAO,YAAY,qEAAsC,KAAK,IAAI,GAAG,CAAC,WAAW;WAC/E,GAAQ;AACf,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KAAK,0CAA0C,IAAI;AAC/D,SAAK,OAAO,YACV,qEAAsC,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OAClE;AACD,SAAK,QAAQ,KAAK;AAElB;;AAGF,QAAK,OAAO,MAAM,iCAAiC,IAAI;AACvD,QAAK,OAAO,YAAY,qEAAsC,KAAK,IAAI,GAAG,CAAC,SAAS;AAEpF,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,iBAAiB,EAAE;AACxB;;AAGF,QAAK,kBAAkB,EAAE;AAEzB,OAAI,sBAAsB,EAAE,CAE1B,MAAK,cAAc,oBAAoB,KAAK,QAAQ,KAAK,aAAa,EAAE;AAG1E,SAAM;;;CAIV,MAAa,eAAe;AAC1B,MAAI;GAEF,MAAM,SAAS,MAAM,KAAK,eAAe,UAAU,KAAK,KAAK,EAAE,SAAS,KAAO,CAAC;GAEhF,MAAM,YAAY,KAAK,SAAS;GAChC,MAAM,YAAY,sBAAsB,KAAK,gBAAgB,cAAc,OAAO,CAAC;AACnF,QAAK,SAAS,SAAS;AACvB,QAAK,QAAQ,OAAO,KAAK;AAEzB,OAAI,OAAO,QAAQ,KAAK,SAAS,OAAO,YAAY,WAAW,SAC7D,MAAK,OAAO,YAAY,uEAAwC,KAAK,IAAI,GAAG,CAAC,UAAU;WAElF,GAAQ;AACf,QAAK,kBAAkB,EAAE;AAEzB,qDAAmB,EAAE,EAAE;AACrB,SAAK,OAAO,KAAK,2DAA2D,EAAE,UAAU;AACxF;;AAGF,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KACV,oDAAoD,EAAE,2DAA4B,KAAK,IAAI,GAC5F;AACD,SAAK,OAAO,YACV,uEAAwC,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OACpE;AACD,SAAK,QAAQ,KAAK;AAClB;;AAGF,QAAK,OAAO,MAAM,0DAA0D,IAAI;;;;CAUpF,AAAQ,kBAAkB,GAAY;AACpC,OAAK,SAAS,YAAY,OAAO,EAAE;;;CAIrC,AAAQ,iBAAiB,GAAY;AACnC,OAAK,SAAS,YAAY,OAAO,EAAE;AACnC,OAAK,SAAS,OAAO;AACrB,OAAK,SAAS;;CAGhB,AAAO,mBAAmB,KAA6B;AACrD,MAAI,0BAA0B,IAAI,EAAE;AAClC,QAAK,QAAQ,KAAK;AAClB,QAAK,iBAAiB;;;CAI1B,AAAQ,QAAQ,MAAe;AAC7B,OAAK,SAAS,OAAO;AACrB,MAAI,KAAM,MAAK,SAAS,YAAY;;CAGtC,AAAO,WAAW,GAAY,UAAkB;AAC9C,OAAK,OAAO,cAAc,EAAE;AAC5B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,AAAO,WAAW,UAAkB;AAClC,SAAO,KAAK,QAAQ,IAAI,SAAS;;CAGnC,AAAQ,mBAAmB;AACzB,SAAO,KAAK,QAAQ,QAAQ;;;;AAKhC,eAAsB,WACpB,QACA,YACA,KACA,YACA,UACA,OAMA;AACA,0BAAO,SAAS,IAAI,EAAE,wDAAwD;CAC9E,MAAM,UAAU;AAEhB,KAAI,UAAU,CAAE;CAChB,MAAM,QAAQ,MAAM,WAAW,WAAW,KAAK,EAAE,SAAS,CAAC;AAC3D,QAAO,KACL,0BAA0B,IAAI,GAAG,mBACZ,MAAM,QAAQ,oBAAoB,MAAM,SAAS,OAAO,kCACzC,MAAM,eAC3C;CAED,MAAM,gBAAgB,SAAiB,OAAO,eAAoC;AAChF,MAAI,UAAU,CAAE;AAChB,QAAM,WAAW,WACf,KACA,WAAW,WACX,OAAO,WAAW,iBAAiB,EACnC,MACA,MAAM,mBACN,MAAM,gBACN,EAAE,SAAS,CACZ;AACD,SAAO,KAAK,kBAAkB,KAAK,GAAG,MAAM,QAAQ,gBAAgB,IAAI,KAAK;AAG7E,QAAM;AACN,MAAI,MAAM,6BAA6B,MAAM,wBAAwB;AACnE,SAAM,4BAA4B;AAClC,SAAM,eAAe,oBAAoB,QAAQ,MAAM,cAAc,MAAM,SAAS;AACpF,cAAW,eAAe,MAAM,aAAa;;;AAIjD,iDAAgB,MAAM,cAAc,MAAM,SAAS,IAAI,aAAa,CAAC;AAErE,KAAI,UAAU,CAAE;AAChB,OAAM,WAAW,SAAS,KAAK,EAAE,SAAS,CAAC;AAE3C,QAAO,KAAK,yBAAyB,IAAI,GAAG,YAAY;;AAG1D,SAAS,YAAY,KAA6B,QAAgB;CAChE,IAAI;CACJ,IAAI;AACJ,KAAI,SAAS,IAAI,EAAE;AACjB,eAAaC,yCAA2B,MAAM,IAAI,KAAK;AACvD,sBAAoB,YAAY,QAAQ,WAAW,WAAW,WAAW,cAAc;;AAGzF,QAAO;EACO;EACZ,UAAU;GACR,MAAM;GACN,QAAQ;GACR,UAAU,SAAS,IAAI;GACJ;GACnB,WAAW;GACZ;EACF;;;AAIH,SAAgB,WAAW,GAAgC;AACzD,QAAO,EAAE,aAAa,EAAE,qBAAqB;;;;;;;AAQ/C,SAAS,cAAc,UAAkD;CACvE,MAAM,SAA6B;EACjC,MAAM,SAAS;EACf,UAAU,SAAS;EACnB,mBAAmB,SAAS;EAC5B,WAAW,SAAS;EACrB;AAED,KAAI,SAAS,OACX,QAAO,SAAS;EACd,UAAU,SAAS,OAAO;EAC1B,gBAAgB,SAAS,OAAO;EAChC,YAAY,SAAS,OAAO;EAC7B;AAEH,QAAO;;AAGT,SAAS,0BAA0B,KAA6B;AAC9D,QAAO,UAAU,IAAI,SACjB,IAAI,OAAO,SAAS,SACpB,IAAI,OAAO,gBAAgB;;AAGjC,SAAgB,SAAS,KAAmB;AAC1C,QAAO,IAAI,KAAK,KAAK,WAAW,aAAa;;AAG/C,SAAgB,YAAY,QAAgB,MAAc,WAA4B;AACpF,KAAI;AACF,SAAO,OAAO,MAAM,UAAU;AAC9B,SAAO;SACD;AACN,SAAO;;;AAIX,SAAS,cAAc,OAAyC;AAC9D,QAAO;EACL,UAAU,MAAM,YAAY;EAC5B,gBAAgB,OAAO,MAAM,eAAe;EAC5C,YAAY,OAAO,MAAM,WAAW;EACrC;;;;;;;AAQH,SAAS,sBAAsB,gBAAyB,QAA0B;AAChF,KAAI,kBAAkB,OAAO,cAAc,KAAK,OAAO,kBAAkB,EACvE,QAAO;EACL,UAAU;EACV,gBAAgB,OAAO,OAAO,WAAW;EACzC,YAAY,OAAO,OAAO,WAAW;EACtC;AAGH,QAAO;;AAGT,SAAgB,sBAAsB,GAAQ;AAC5C,KAAI,GAAG,SAAS,WAAY,QAAO;AACnC,SAAQ,EAAE,MAAV;EACE,KAAK;EACL,KAAK;EACL,KAAK,iBACH,QAAO;EACT,QACE,QAAO;;;AAIb,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAaC,6BACb,aAAaC,gCACb,aAAaC,qCACb,aAAaC;;AAIjB,SAAS,sBAAsB,GAAQ;AACrC,QAAQ,GAAa,QAAQ,SAAS,0BAA0B;;;AAIlE,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,UAAU,GAAG,IAAI;AACjD,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO;;;;AAKT,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,KAAK,MAAM,UAAU,EAAE,EAAE,IAAI;AAC7D,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO"}
+{"version":3,"file":"upload_task.cjs","names":["ChangeSource","CallersCounter","ImportFileHandleUploadData","MTimeError","UnexpectedEOF","NoFileForUploading","BadRequestError"],"sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport {\n  isTimeoutError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { AsyncPoolController, MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientProgress, ProgressStatus } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from \"../clients/upload\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { ImportFileHandleUploadData } from \"./types\";\nimport assert from \"node:assert\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n  private nMaxUploads: number;\n  private nPartsWithThisUploadSpeed = 0;\n  private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n  /** If this is upload progress this field will be defined */\n  private uploadData?: ImportFileHandleUploadData;\n  public progress: sdk.ImportProgress;\n\n  /** If failed, then getting a progress is terminally failed. */\n  public failed?: boolean;\n\n  /** True if the blob was existed.\n   * At this case, the task will show progress == 1.0. */\n  private alreadyExisted = false;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly maxNConcurrentPartsUpload: number,\n    signer: Signer,\n    public readonly res: ImportResourceSnapshot,\n  ) {\n    this.nMaxUploads = this.maxNConcurrentPartsUpload;\n    const { uploadData, progress } = newProgress(res, signer);\n    this.uploadData = uploadData;\n    this.progress = progress;\n  }\n\n  public getProgress(w: Watcher, callerId: string) {\n    this.incCounter(w, callerId);\n\n    if (this.failed) {\n      this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n      throw new Error(this.progress.lastError);\n    }\n\n    return cloneProgress(this.progress);\n  }\n\n  public shouldScheduleUpload(): boolean {\n    return isMyUpload(this.progress);\n  }\n\n  /** Uploads a blob if it's not BlobIndex. */\n  public async uploadBlobTask() {\n    try {\n      await uploadBlob(\n        this.logger,\n        this.clientBlob,\n        this.res,\n        this.uploadData!,\n        this.isComputableDone.bind(this),\n        {\n          nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n          nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n          currentSpeed: this.nMaxUploads,\n          maxSpeed: this.maxNConcurrentPartsUpload,\n        },\n      );\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n    } catch (e: any) {\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(`terminal error while uploading a blob: ${e}`);\n        this.change.markChanged(\n          `blob upload for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n\n        return;\n      }\n\n      this.logger.error(`error while uploading a blob: ${e}`);\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n      if (nonRecoverableError(e)) {\n        this.setTerminalError(e);\n        return;\n      }\n\n      this.setRetriableError(e);\n\n      if (isHeadersTimeoutError(e)) {\n        // we probably have a slow internet, we need to slow things a bit.\n        this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n      }\n\n      throw e;\n    }\n  }\n\n  public async updateStatus() {\n    try {\n      // we do it with timeout in case we have slow internet.\n      const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n      const oldStatus = this.progress.status;\n      const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n      this.progress.status = newStatus;\n      this.setDone(status.done);\n\n      if (status.done || this.progress.status.progress != oldStatus?.progress) {\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n      }\n    } catch (e: any) {\n      this.setRetriableError(e);\n\n      if (isTimeoutError(e)) {\n        this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);\n        return;\n      }\n\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(\n          `terminal error while updating BlobImport status: ${e}, ${stringifyWithResourceId(this.res)}`,\n        );\n        this.change.markChanged(\n          `upload status for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n        return;\n      }\n\n      this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n      // It was a terminal error, but when a connection drops,\n      // this will stop the whole task, so we make it retryable.\n      // It was like that:\n      // this.change.markChanged();\n      // this.setTerminalError(e);\n    }\n  }\n\n  /** Set non-terminal error, that task can be retried. */\n  private setRetriableError(e: unknown) {\n    this.progress.lastError = String(e);\n  }\n\n  /** Set a terminal error, the task will throw a error instead of a progress. */\n  private setTerminalError(e: unknown) {\n    this.progress.lastError = String(e);\n    this.progress.done = false;\n    this.failed = true;\n  }\n\n  public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n    if (isImportResourceOutputSet(res)) {\n      this.setDone(true);\n      this.alreadyExisted = true;\n    }\n  }\n\n  private setDone(done: boolean) {\n    this.progress.done = done;\n    if (done) this.progress.lastError = undefined;\n  }\n\n  public incCounter(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  public decCounter(callerId: string) {\n    return this.counter.dec(callerId);\n  }\n\n  private isComputableDone() {\n    return this.counter.isZero();\n  }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n  logger: MiLogger,\n  clientBlob: ClientUpload,\n  res: ResourceInfo,\n  uploadData: ImportFileHandleUploadData,\n  isDoneFn: () => boolean,\n  speed: {\n    nPartsWithThisUploadSpeed: number;\n    nPartsToIncreaseUpload: number;\n    currentSpeed: number;\n    maxSpeed: number;\n  },\n) {\n  assert(isUpload(res), \"the upload operation can be done only for BlobUploads\");\n  const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n  if (isDoneFn()) return;\n  const parts = await clientBlob.initUpload(res, { timeout });\n  logger.info(\n    `started to upload blob ${res.id},` +\n      ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},` +\n      ` number of concurrent uploads: ${speed.currentSpeed}`,\n  );\n\n  const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n    if (isDoneFn()) return;\n    await clientBlob.partUpload(\n      res,\n      uploadData.localPath,\n      BigInt(uploadData.modificationTime),\n      part,\n      parts.checksumAlgorithm,\n      parts.checksumHeader,\n      { timeout },\n    );\n    logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n    // if we had a network freeze, it will be increased slowly.\n    speed.nPartsWithThisUploadSpeed++;\n    if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n      speed.nPartsWithThisUploadSpeed = 0;\n      speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n      controller.setConcurrency(speed.currentSpeed);\n    }\n  };\n\n  await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n  if (isDoneFn()) return;\n  await clientBlob.finalize(res, { timeout });\n\n  logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n  let isUploadSignMatch: boolean | undefined;\n  let uploadData: ImportFileHandleUploadData | undefined;\n  if (isUpload(res)) {\n    uploadData = ImportFileHandleUploadData.parse(res.data);\n    isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n  }\n\n  return {\n    uploadData: uploadData,\n    progress: {\n      done: false,\n      status: undefined,\n      isUpload: isUpload(res),\n      isUploadSignMatch: isUploadSignMatch,\n      lastError: undefined,\n    } satisfies sdk.ImportProgress,\n  };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n  return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n  const cloned: sdk.ImportProgress = {\n    done: progress.done,\n    isUpload: progress.isUpload,\n    isUploadSignMatch: progress.isUploadSignMatch,\n    lastError: progress.lastError,\n  };\n\n  if (progress.status)\n    cloned.status = {\n      progress: progress.status.progress,\n      bytesProcessed: progress.status.bytesProcessed,\n      bytesTotal: progress.status.bytesTotal,\n    };\n\n  return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n  return \"blob\" in res.fields\n    ? res.fields.blob !== undefined\n    : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n  return res.type.name.startsWith(\"BlobUpload\");\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n  try {\n    signer.verify(path, signature);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n  return {\n    progress: proto.progress ?? 0,\n    bytesProcessed: Number(proto.bytesProcessed),\n    bytesTotal: Number(proto.bytesTotal),\n  };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n  if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n    return {\n      progress: 1.0,\n      bytesProcessed: Number(status.bytesTotal),\n      bytesTotal: Number(status.bytesTotal),\n    };\n  }\n\n  return status;\n}\n\nexport function isTerminalUploadError(e: any) {\n  if (e?.name !== \"RpcError\") return false;\n  switch (e.code) {\n    case \"NOT_FOUND\":\n    case \"ABORTED\":\n    case \"ALREADY_EXISTS\":\n      return true;\n    default:\n      return false;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof MTimeError ||\n    e instanceof UnexpectedEOF ||\n    e instanceof NoFileForUploading ||\n    e instanceof BadRequestError\n  );\n}\n\nfunction isHeadersTimeoutError(e: any) {\n  return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n  const newConcurrency = Math.min(current + 2, max);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n  const newConcurrency = Math.max(Math.round(current / 2), min);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n"],"mappings":";;;;;;;;;;;;AAqBA,IAAa,aAAb,MAAwB;CACtB,SAAwC,IAAIA,2BAAAA,cAAc;CAC1D,UAA2C,IAAIC,2BAAAA,gBAAgB;CAC/D;CACA,4BAAoC;CACpC,yBAAiC;;CAGjC;CACA;;CAGA;;;CAIA,iBAAyB;CAEzB,YACE,QACA,YACA,gBACA,2BACA,QACA,KACA;AANiB,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,iBAAA;AACA,OAAA,4BAAA;AAED,OAAA,MAAA;AAEhB,OAAK,cAAc,KAAK;EACxB,MAAM,EAAE,YAAY,aAAa,YAAY,KAAK,OAAO;AACzD,OAAK,aAAa;AAClB,OAAK,WAAW;;CAGlB,YAAmB,GAAY,UAAkB;AAC/C,OAAK,WAAW,GAAG,SAAS;AAE5B,MAAI,KAAK,QAAQ;AACf,QAAK,OAAO,MAAM,gCAAgC,KAAK,SAAS,YAAY;AAC5E,SAAM,IAAI,MAAM,KAAK,SAAS,UAAU;;AAG1C,SAAO,cAAc,KAAK,SAAS;;CAGrC,uBAAuC;AACrC,SAAO,WAAW,KAAK,SAAS;;;CAIlC,MAAa,iBAAiB;AAC5B,MAAI;AACF,SAAM,WACJ,KAAK,QACL,KAAK,YACL,KAAK,KACL,KAAK,YACL,KAAK,iBAAiB,KAAK,KAAK,EAChC;IACE,2BAA2B,KAAK;IAChC,wBAAwB,KAAK;IAC7B,cAAc,KAAK;IACnB,UAAU,KAAK;IAChB,CACF;AACD,QAAK,OAAO,YAAY,oBAAA,GAAA,0BAAA,oBAAsC,KAAK,IAAI,GAAG,CAAC,WAAW;WAC/E,GAAQ;AACf,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KAAK,0CAA0C,IAAI;AAC/D,SAAK,OAAO,YACV,oBAAA,GAAA,0BAAA,oBAAsC,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OAClE;AACD,SAAK,QAAQ,KAAK;AAElB;;AAGF,QAAK,OAAO,MAAM,iCAAiC,IAAI;AACvD,QAAK,OAAO,YAAY,oBAAA,GAAA,0BAAA,oBAAsC,KAAK,IAAI,GAAG,CAAC,SAAS;AAEpF,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,iBAAiB,EAAE;AACxB;;AAGF,QAAK,kBAAkB,EAAE;AAEzB,OAAI,sBAAsB,EAAE,CAE1B,MAAK,cAAc,oBAAoB,KAAK,QAAQ,KAAK,aAAa,EAAE;AAG1E,SAAM;;;CAIV,MAAa,eAAe;AAC1B,MAAI;GAEF,MAAM,SAAS,MAAM,KAAK,eAAe,UAAU,KAAK,KAAK,EAAE,SAAS,KAAO,CAAC;GAEhF,MAAM,YAAY,KAAK,SAAS;GAChC,MAAM,YAAY,sBAAsB,KAAK,gBAAgB,cAAc,OAAO,CAAC;AACnF,QAAK,SAAS,SAAS;AACvB,QAAK,QAAQ,OAAO,KAAK;AAEzB,OAAI,OAAO,QAAQ,KAAK,SAAS,OAAO,YAAY,WAAW,SAC7D,MAAK,OAAO,YAAY,sBAAA,GAAA,0BAAA,oBAAwC,KAAK,IAAI,GAAG,CAAC,UAAU;WAElF,GAAQ;AACf,QAAK,kBAAkB,EAAE;AAEzB,QAAA,GAAA,0BAAA,gBAAmB,EAAE,EAAE;AACrB,SAAK,OAAO,KAAK,2DAA2D,EAAE,UAAU;AACxF;;AAGF,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KACV,oDAAoD,EAAE,KAAA,GAAA,0BAAA,yBAA4B,KAAK,IAAI,GAC5F;AACD,SAAK,OAAO,YACV,sBAAA,GAAA,0BAAA,oBAAwC,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OACpE;AACD,SAAK,QAAQ,KAAK;AAClB;;AAGF,QAAK,OAAO,MAAM,0DAA0D,IAAI;;;;CAUpF,kBAA0B,GAAY;AACpC,OAAK,SAAS,YAAY,OAAO,EAAE;;;CAIrC,iBAAyB,GAAY;AACnC,OAAK,SAAS,YAAY,OAAO,EAAE;AACnC,OAAK,SAAS,OAAO;AACrB,OAAK,SAAS;;CAGhB,mBAA0B,KAA6B;AACrD,MAAI,0BAA0B,IAAI,EAAE;AAClC,QAAK,QAAQ,KAAK;AAClB,QAAK,iBAAiB;;;CAI1B,QAAgB,MAAe;AAC7B,OAAK,SAAS,OAAO;AACrB,MAAI,KAAM,MAAK,SAAS,YAAY,KAAA;;CAGtC,WAAkB,GAAY,UAAkB;AAC9C,OAAK,OAAO,cAAc,EAAE;AAC5B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,WAAkB,UAAkB;AAClC,SAAO,KAAK,QAAQ,IAAI,SAAS;;CAGnC,mBAA2B;AACzB,SAAO,KAAK,QAAQ,QAAQ;;;;AAKhC,eAAsB,WACpB,QACA,YACA,KACA,YACA,UACA,OAMA;AACA,EAAA,GAAA,YAAA,SAAO,SAAS,IAAI,EAAE,wDAAwD;CAC9E,MAAM,UAAU;AAEhB,KAAI,UAAU,CAAE;CAChB,MAAM,QAAQ,MAAM,WAAW,WAAW,KAAK,EAAE,SAAS,CAAC;AAC3D,QAAO,KACL,0BAA0B,IAAI,GAAG,mBACZ,MAAM,QAAQ,oBAAoB,MAAM,SAAS,OAAO,kCACzC,MAAM,eAC3C;CAED,MAAM,gBAAgB,SAAiB,OAAO,eAAoC;AAChF,MAAI,UAAU,CAAE;AAChB,QAAM,WAAW,WACf,KACA,WAAW,WACX,OAAO,WAAW,iBAAiB,EACnC,MACA,MAAM,mBACN,MAAM,gBACN,EAAE,SAAS,CACZ;AACD,SAAO,KAAK,kBAAkB,KAAK,GAAG,MAAM,QAAQ,gBAAgB,IAAI,KAAK;AAG7E,QAAM;AACN,MAAI,MAAM,6BAA6B,MAAM,wBAAwB;AACnE,SAAM,4BAA4B;AAClC,SAAM,eAAe,oBAAoB,QAAQ,MAAM,cAAc,MAAM,SAAS;AACpF,cAAW,eAAe,MAAM,aAAa;;;AAIjD,QAAA,GAAA,2BAAA,WAAgB,MAAM,cAAc,MAAM,SAAS,IAAI,aAAa,CAAC;AAErE,KAAI,UAAU,CAAE;AAChB,OAAM,WAAW,SAAS,KAAK,EAAE,SAAS,CAAC;AAE3C,QAAO,KAAK,yBAAyB,IAAI,GAAG,YAAY;;AAG1D,SAAS,YAAY,KAA6B,QAAgB;CAChE,IAAI;CACJ,IAAI;AACJ,KAAI,SAAS,IAAI,EAAE;AACjB,eAAaC,cAAAA,2BAA2B,MAAM,IAAI,KAAK;AACvD,sBAAoB,YAAY,QAAQ,WAAW,WAAW,WAAW,cAAc;;AAGzF,QAAO;EACO;EACZ,UAAU;GACR,MAAM;GACN,QAAQ,KAAA;GACR,UAAU,SAAS,IAAI;GACJ;GACnB,WAAW,KAAA;GACZ;EACF;;;AAIH,SAAgB,WAAW,GAAgC;AACzD,QAAO,EAAE,aAAa,EAAE,qBAAqB;;;;;;;AAQ/C,SAAS,cAAc,UAAkD;CACvE,MAAM,SAA6B;EACjC,MAAM,SAAS;EACf,UAAU,SAAS;EACnB,mBAAmB,SAAS;EAC5B,WAAW,SAAS;EACrB;AAED,KAAI,SAAS,OACX,QAAO,SAAS;EACd,UAAU,SAAS,OAAO;EAC1B,gBAAgB,SAAS,OAAO;EAChC,YAAY,SAAS,OAAO;EAC7B;AAEH,QAAO;;AAGT,SAAS,0BAA0B,KAA6B;AAC9D,QAAO,UAAU,IAAI,SACjB,IAAI,OAAO,SAAS,KAAA,IACpB,IAAI,OAAO,gBAAgB,KAAA;;AAGjC,SAAgB,SAAS,KAAmB;AAC1C,QAAO,IAAI,KAAK,KAAK,WAAW,aAAa;;AAG/C,SAAgB,YAAY,QAAgB,MAAc,WAA4B;AACpF,KAAI;AACF,SAAO,OAAO,MAAM,UAAU;AAC9B,SAAO;SACD;AACN,SAAO;;;AAIX,SAAS,cAAc,OAAyC;AAC9D,QAAO;EACL,UAAU,MAAM,YAAY;EAC5B,gBAAgB,OAAO,MAAM,eAAe;EAC5C,YAAY,OAAO,MAAM,WAAW;EACrC;;;;;;;AAQH,SAAS,sBAAsB,gBAAyB,QAA0B;AAChF,KAAI,kBAAkB,OAAO,cAAc,KAAK,OAAO,kBAAkB,EACvE,QAAO;EACL,UAAU;EACV,gBAAgB,OAAO,OAAO,WAAW;EACzC,YAAY,OAAO,OAAO,WAAW;EACtC;AAGH,QAAO;;AAGT,SAAgB,sBAAsB,GAAQ;AAC5C,KAAI,GAAG,SAAS,WAAY,QAAO;AACnC,SAAQ,EAAE,MAAV;EACE,KAAK;EACL,KAAK;EACL,KAAK,iBACH,QAAO;EACT,QACE,QAAO;;;AAIb,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAaC,eAAAA,cACb,aAAaC,eAAAA,iBACb,aAAaC,eAAAA,sBACb,aAAaC,eAAAA;;AAIjB,SAAS,sBAAsB,GAAQ;AACrC,QAAQ,GAAa,QAAQ,SAAS,0BAA0B;;;AAIlE,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,UAAU,GAAG,IAAI;AACjD,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO;;;;AAKT,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,KAAK,MAAM,UAAU,EAAE,EAAE,IAAI;AAC7D,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO"}

package/dist/drivers/upload_task.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload_task.d.ts","names":[],"sources":["../../src/drivers/upload_task.ts"],"mappings":";;;;;;;;;AAqBA;;;AAAA,cAAa,UAAA;EAAA,iBAmBQ,MAAA;EAAA,iBACA,UAAA;EAAA,iBACA,cAAA;EAAA,iBACA,yBAAA;EAAA,SAED,GAAA,EAAK,sBAAA;EAAA,iBAvBN,MAAA;EAAA,iBACA,OAAA;EAAA,QACT,WAAA;EAAA,QACA,yBAAA;EAAA,QACA,sBAAA;EA8IuB;EAAA,QA3IvB,UAAA;EACD,QAAA,EAAU,GAAA,CAAI,cAAA;EAsJO;EAnJrB,MAAA;EAQY;;EAAA,QAJX,cAAA;cAGW,MAAA,EAAQ,QAAA,EACR,UAAA,EAAY,YAAA,EACZ,cAAA,EAAgB,cAAA,EAChB,yBAAA,UACjB,MAAA,EAAQ,MAAA,EACQ,GAAA,EAAK,sBAAA;EAQhB,WAAA,CAAY,CAAA,EAAG,OAAA,EAAS,QAAA,WAAgB,GAAA,CAAA,cAAA;EAWxC,oBAAA,CAAA;EAzCU;EA8CJ,cAAA,CAAA,GAAc,OAAA;EA8Cd,YAAA,CAAA,GAAY,OAAA;EAzFjB;EAAA,QAmIA,iBAAA;EA/HD;EAAA,QAoIC,gBAAA;EAMD,kBAAA,CAAmB,GAAA,EAAK,sBAAA;EAAA,QAOvB,OAAA;EAKD,UAAA,CAAW,CAAA,EAAG,OAAA,EAAS,QAAA;EAKvB,UAAA,CAAW,QAAA;EAAA,QAIV,gBAAA;AAAA;;iBAMY,UAAA,CACpB,MAAA,EAAQ,QAAA,EACR,UAAA,EAAY,YAAA,EACZ,GAAA,EAAK,YAAA,EACL,UAAA,EAAY,0BAAA,EACZ,QAAA,iBACA,KAAA;EACE,yBAAA;EACA,sBAAA;EACA,YAAA;EACA,QAAA;AAAA,IACD,OAAA;;iBAgEa,UAAA,CAAW,CAAA,EAAG,GAAA,CAAI,cAAA;AAAA,iBAiClB,QAAA,CAAS,GAAA,EAAK,YAAA;AAAA,iBAId,WAAA,CAAY,MAAA,EAAQ,MAAA,EAAQ,IAAA,UAAc,SAAA;AAAA,iBAkC1C,qBAAA,CAAsB,CAAA;AAAA,iBAYtB,mBAAA,CAAoB,CAAA,QAAM,CAAA,IAAA,UAAA,GAAA,aAAA,GAAA,kBAAA,GAAA,eAAA"}

package/dist/drivers/upload_task.js

@@ -4,7 +4,6 @@ import { isTimeoutError, resourceIdToString, stringifyWithResourceId } from "@mi
 import { CallersCounter, asyncPool } from "@milaboratories/ts-helpers";
 import { ChangeSource } from "@milaboratories/computable";
 import assert from "node:assert";
-
 //#region src/drivers/upload_task.ts
 /** Holds all info needed to upload a file and a status of uploading
 * and indexing. Also, has a method to update a status of the progress.
@@ -253,7 +252,7 @@ function decreaseConcurrency(logger, current, min) {
 	if (newConcurrency != current) logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`);
 	return newConcurrency;
 }
-
 //#endregion
 export { UploadTask, isMyUpload, isSignMatch, isTerminalUploadError, isUpload, nonRecoverableError, uploadBlob };
+
 //# sourceMappingURL=upload_task.js.map

package/dist/drivers/upload_task.js.map

@@ -1 +1 @@
-{"version":3,"file":"upload_task.js","names":[],"sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport {\n  isTimeoutError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { AsyncPoolController, MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientProgress, ProgressStatus } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from \"../clients/upload\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { ImportFileHandleUploadData } from \"./types\";\nimport assert from \"node:assert\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n  private nMaxUploads: number;\n  private nPartsWithThisUploadSpeed = 0;\n  private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n  /** If this is upload progress this field will be defined */\n  private uploadData?: ImportFileHandleUploadData;\n  public progress: sdk.ImportProgress;\n\n  /** If failed, then getting a progress is terminally failed. */\n  public failed?: boolean;\n\n  /** True if the blob was existed.\n   * At this case, the task will show progress == 1.0. */\n  private alreadyExisted = false;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly maxNConcurrentPartsUpload: number,\n    signer: Signer,\n    public readonly res: ImportResourceSnapshot,\n  ) {\n    this.nMaxUploads = this.maxNConcurrentPartsUpload;\n    const { uploadData, progress } = newProgress(res, signer);\n    this.uploadData = uploadData;\n    this.progress = progress;\n  }\n\n  public getProgress(w: Watcher, callerId: string) {\n    this.incCounter(w, callerId);\n\n    if (this.failed) {\n      this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n      throw new Error(this.progress.lastError);\n    }\n\n    return cloneProgress(this.progress);\n  }\n\n  public shouldScheduleUpload(): boolean {\n    return isMyUpload(this.progress);\n  }\n\n  /** Uploads a blob if it's not BlobIndex. */\n  public async uploadBlobTask() {\n    try {\n      await uploadBlob(\n        this.logger,\n        this.clientBlob,\n        this.res,\n        this.uploadData!,\n        this.isComputableDone.bind(this),\n        {\n          nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n          nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n          currentSpeed: this.nMaxUploads,\n          maxSpeed: this.maxNConcurrentPartsUpload,\n        },\n      );\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n    } catch (e: any) {\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(`terminal error while uploading a blob: ${e}`);\n        this.change.markChanged(\n          `blob upload for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n\n        return;\n      }\n\n      this.logger.error(`error while uploading a blob: ${e}`);\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n      if (nonRecoverableError(e)) {\n        this.setTerminalError(e);\n        return;\n      }\n\n      this.setRetriableError(e);\n\n      if (isHeadersTimeoutError(e)) {\n        // we probably have a slow internet, we need to slow things a bit.\n        this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n      }\n\n      throw e;\n    }\n  }\n\n  public async updateStatus() {\n    try {\n      // we do it with timeout in case we have slow internet.\n      const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n      const oldStatus = this.progress.status;\n      const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n      this.progress.status = newStatus;\n      this.setDone(status.done);\n\n      if (status.done || this.progress.status.progress != oldStatus?.progress) {\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n      }\n    } catch (e: any) {\n      this.setRetriableError(e);\n\n      if (isTimeoutError(e)) {\n        this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);\n        return;\n      }\n\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(\n          `terminal error while updating BlobImport status: ${e}, ${stringifyWithResourceId(this.res)}`,\n        );\n        this.change.markChanged(\n          `upload status for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n        return;\n      }\n\n      this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n      // It was a terminal error, but when a connection drops,\n      // this will stop the whole task, so we make it retryable.\n      // It was like that:\n      // this.change.markChanged();\n      // this.setTerminalError(e);\n    }\n  }\n\n  /** Set non-terminal error, that task can be retried. */\n  private setRetriableError(e: unknown) {\n    this.progress.lastError = String(e);\n  }\n\n  /** Set a terminal error, the task will throw a error instead of a progress. */\n  private setTerminalError(e: unknown) {\n    this.progress.lastError = String(e);\n    this.progress.done = false;\n    this.failed = true;\n  }\n\n  public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n    if (isImportResourceOutputSet(res)) {\n      this.setDone(true);\n      this.alreadyExisted = true;\n    }\n  }\n\n  private setDone(done: boolean) {\n    this.progress.done = done;\n    if (done) this.progress.lastError = undefined;\n  }\n\n  public incCounter(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  public decCounter(callerId: string) {\n    return this.counter.dec(callerId);\n  }\n\n  private isComputableDone() {\n    return this.counter.isZero();\n  }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n  logger: MiLogger,\n  clientBlob: ClientUpload,\n  res: ResourceInfo,\n  uploadData: ImportFileHandleUploadData,\n  isDoneFn: () => boolean,\n  speed: {\n    nPartsWithThisUploadSpeed: number;\n    nPartsToIncreaseUpload: number;\n    currentSpeed: number;\n    maxSpeed: number;\n  },\n) {\n  assert(isUpload(res), \"the upload operation can be done only for BlobUploads\");\n  const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n  if (isDoneFn()) return;\n  const parts = await clientBlob.initUpload(res, { timeout });\n  logger.info(\n    `started to upload blob ${res.id},` +\n      ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},` +\n      ` number of concurrent uploads: ${speed.currentSpeed}`,\n  );\n\n  const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n    if (isDoneFn()) return;\n    await clientBlob.partUpload(\n      res,\n      uploadData.localPath,\n      BigInt(uploadData.modificationTime),\n      part,\n      parts.checksumAlgorithm,\n      parts.checksumHeader,\n      { timeout },\n    );\n    logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n    // if we had a network freeze, it will be increased slowly.\n    speed.nPartsWithThisUploadSpeed++;\n    if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n      speed.nPartsWithThisUploadSpeed = 0;\n      speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n      controller.setConcurrency(speed.currentSpeed);\n    }\n  };\n\n  await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n  if (isDoneFn()) return;\n  await clientBlob.finalize(res, { timeout });\n\n  logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n  let isUploadSignMatch: boolean | undefined;\n  let uploadData: ImportFileHandleUploadData | undefined;\n  if (isUpload(res)) {\n    uploadData = ImportFileHandleUploadData.parse(res.data);\n    isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n  }\n\n  return {\n    uploadData: uploadData,\n    progress: {\n      done: false,\n      status: undefined,\n      isUpload: isUpload(res),\n      isUploadSignMatch: isUploadSignMatch,\n      lastError: undefined,\n    } satisfies sdk.ImportProgress,\n  };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n  return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n  const cloned: sdk.ImportProgress = {\n    done: progress.done,\n    isUpload: progress.isUpload,\n    isUploadSignMatch: progress.isUploadSignMatch,\n    lastError: progress.lastError,\n  };\n\n  if (progress.status)\n    cloned.status = {\n      progress: progress.status.progress,\n      bytesProcessed: progress.status.bytesProcessed,\n      bytesTotal: progress.status.bytesTotal,\n    };\n\n  return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n  return \"blob\" in res.fields\n    ? res.fields.blob !== undefined\n    : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n  return res.type.name.startsWith(\"BlobUpload\");\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n  try {\n    signer.verify(path, signature);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n  return {\n    progress: proto.progress ?? 0,\n    bytesProcessed: Number(proto.bytesProcessed),\n    bytesTotal: Number(proto.bytesTotal),\n  };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n  if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n    return {\n      progress: 1.0,\n      bytesProcessed: Number(status.bytesTotal),\n      bytesTotal: Number(status.bytesTotal),\n    };\n  }\n\n  return status;\n}\n\nexport function isTerminalUploadError(e: any) {\n  if (e?.name !== \"RpcError\") return false;\n  switch (e.code) {\n    case \"NOT_FOUND\":\n    case \"ABORTED\":\n    case \"ALREADY_EXISTS\":\n      return true;\n    default:\n      return false;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof MTimeError ||\n    e instanceof UnexpectedEOF ||\n    e instanceof NoFileForUploading ||\n    e instanceof BadRequestError\n  );\n}\n\nfunction isHeadersTimeoutError(e: any) {\n  return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n  const newConcurrency = Math.min(current + 2, max);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n  const newConcurrency = Math.max(Math.round(current / 2), min);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n"],"mappings":";;;;;;;;;;;AAqBA,IAAa,aAAb,MAAwB;CACtB,AAAiB,SAAuB,IAAI,cAAc;CAC1D,AAAiB,UAA0B,IAAI,gBAAgB;CAC/D,AAAQ;CACR,AAAQ,4BAA4B;CACpC,AAAQ,yBAAyB;;CAGjC,AAAQ;CACR,AAAO;;CAGP,AAAO;;;CAIP,AAAQ,iBAAiB;CAEzB,YACE,AAAiB,QACjB,AAAiB,YACjB,AAAiB,gBACjB,AAAiB,2BACjB,QACA,AAAgB,KAChB;EANiB;EACA;EACA;EACA;EAED;AAEhB,OAAK,cAAc,KAAK;EACxB,MAAM,EAAE,YAAY,aAAa,YAAY,KAAK,OAAO;AACzD,OAAK,aAAa;AAClB,OAAK,WAAW;;CAGlB,AAAO,YAAY,GAAY,UAAkB;AAC/C,OAAK,WAAW,GAAG,SAAS;AAE5B,MAAI,KAAK,QAAQ;AACf,QAAK,OAAO,MAAM,gCAAgC,KAAK,SAAS,YAAY;AAC5E,SAAM,IAAI,MAAM,KAAK,SAAS,UAAU;;AAG1C,SAAO,cAAc,KAAK,SAAS;;CAGrC,AAAO,uBAAgC;AACrC,SAAO,WAAW,KAAK,SAAS;;;CAIlC,MAAa,iBAAiB;AAC5B,MAAI;AACF,SAAM,WACJ,KAAK,QACL,KAAK,YACL,KAAK,KACL,KAAK,YACL,KAAK,iBAAiB,KAAK,KAAK,EAChC;IACE,2BAA2B,KAAK;IAChC,wBAAwB,KAAK;IAC7B,cAAc,KAAK;IACnB,UAAU,KAAK;IAChB,CACF;AACD,QAAK,OAAO,YAAY,mBAAmB,mBAAmB,KAAK,IAAI,GAAG,CAAC,WAAW;WAC/E,GAAQ;AACf,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KAAK,0CAA0C,IAAI;AAC/D,SAAK,OAAO,YACV,mBAAmB,mBAAmB,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OAClE;AACD,SAAK,QAAQ,KAAK;AAElB;;AAGF,QAAK,OAAO,MAAM,iCAAiC,IAAI;AACvD,QAAK,OAAO,YAAY,mBAAmB,mBAAmB,KAAK,IAAI,GAAG,CAAC,SAAS;AAEpF,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,iBAAiB,EAAE;AACxB;;AAGF,QAAK,kBAAkB,EAAE;AAEzB,OAAI,sBAAsB,EAAE,CAE1B,MAAK,cAAc,oBAAoB,KAAK,QAAQ,KAAK,aAAa,EAAE;AAG1E,SAAM;;;CAIV,MAAa,eAAe;AAC1B,MAAI;GAEF,MAAM,SAAS,MAAM,KAAK,eAAe,UAAU,KAAK,KAAK,EAAE,SAAS,KAAO,CAAC;GAEhF,MAAM,YAAY,KAAK,SAAS;GAChC,MAAM,YAAY,sBAAsB,KAAK,gBAAgB,cAAc,OAAO,CAAC;AACnF,QAAK,SAAS,SAAS;AACvB,QAAK,QAAQ,OAAO,KAAK;AAEzB,OAAI,OAAO,QAAQ,KAAK,SAAS,OAAO,YAAY,WAAW,SAC7D,MAAK,OAAO,YAAY,qBAAqB,mBAAmB,KAAK,IAAI,GAAG,CAAC,UAAU;WAElF,GAAQ;AACf,QAAK,kBAAkB,EAAE;AAEzB,OAAI,eAAe,EAAE,EAAE;AACrB,SAAK,OAAO,KAAK,2DAA2D,EAAE,UAAU;AACxF;;AAGF,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KACV,oDAAoD,EAAE,IAAI,wBAAwB,KAAK,IAAI,GAC5F;AACD,SAAK,OAAO,YACV,qBAAqB,mBAAmB,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OACpE;AACD,SAAK,QAAQ,KAAK;AAClB;;AAGF,QAAK,OAAO,MAAM,0DAA0D,IAAI;;;;CAUpF,AAAQ,kBAAkB,GAAY;AACpC,OAAK,SAAS,YAAY,OAAO,EAAE;;;CAIrC,AAAQ,iBAAiB,GAAY;AACnC,OAAK,SAAS,YAAY,OAAO,EAAE;AACnC,OAAK,SAAS,OAAO;AACrB,OAAK,SAAS;;CAGhB,AAAO,mBAAmB,KAA6B;AACrD,MAAI,0BAA0B,IAAI,EAAE;AAClC,QAAK,QAAQ,KAAK;AAClB,QAAK,iBAAiB;;;CAI1B,AAAQ,QAAQ,MAAe;AAC7B,OAAK,SAAS,OAAO;AACrB,MAAI,KAAM,MAAK,SAAS,YAAY;;CAGtC,AAAO,WAAW,GAAY,UAAkB;AAC9C,OAAK,OAAO,cAAc,EAAE;AAC5B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,AAAO,WAAW,UAAkB;AAClC,SAAO,KAAK,QAAQ,IAAI,SAAS;;CAGnC,AAAQ,mBAAmB;AACzB,SAAO,KAAK,QAAQ,QAAQ;;;;AAKhC,eAAsB,WACpB,QACA,YACA,KACA,YACA,UACA,OAMA;AACA,QAAO,SAAS,IAAI,EAAE,wDAAwD;CAC9E,MAAM,UAAU;AAEhB,KAAI,UAAU,CAAE;CAChB,MAAM,QAAQ,MAAM,WAAW,WAAW,KAAK,EAAE,SAAS,CAAC;AAC3D,QAAO,KACL,0BAA0B,IAAI,GAAG,mBACZ,MAAM,QAAQ,oBAAoB,MAAM,SAAS,OAAO,kCACzC,MAAM,eAC3C;CAED,MAAM,gBAAgB,SAAiB,OAAO,eAAoC;AAChF,MAAI,UAAU,CAAE;AAChB,QAAM,WAAW,WACf,KACA,WAAW,WACX,OAAO,WAAW,iBAAiB,EACnC,MACA,MAAM,mBACN,MAAM,gBACN,EAAE,SAAS,CACZ;AACD,SAAO,KAAK,kBAAkB,KAAK,GAAG,MAAM,QAAQ,gBAAgB,IAAI,KAAK;AAG7E,QAAM;AACN,MAAI,MAAM,6BAA6B,MAAM,wBAAwB;AACnE,SAAM,4BAA4B;AAClC,SAAM,eAAe,oBAAoB,QAAQ,MAAM,cAAc,MAAM,SAAS;AACpF,cAAW,eAAe,MAAM,aAAa;;;AAIjD,OAAM,UAAU,MAAM,cAAc,MAAM,SAAS,IAAI,aAAa,CAAC;AAErE,KAAI,UAAU,CAAE;AAChB,OAAM,WAAW,SAAS,KAAK,EAAE,SAAS,CAAC;AAE3C,QAAO,KAAK,yBAAyB,IAAI,GAAG,YAAY;;AAG1D,SAAS,YAAY,KAA6B,QAAgB;CAChE,IAAI;CACJ,IAAI;AACJ,KAAI,SAAS,IAAI,EAAE;AACjB,eAAa,2BAA2B,MAAM,IAAI,KAAK;AACvD,sBAAoB,YAAY,QAAQ,WAAW,WAAW,WAAW,cAAc;;AAGzF,QAAO;EACO;EACZ,UAAU;GACR,MAAM;GACN,QAAQ;GACR,UAAU,SAAS,IAAI;GACJ;GACnB,WAAW;GACZ;EACF;;;AAIH,SAAgB,WAAW,GAAgC;AACzD,QAAO,EAAE,aAAa,EAAE,qBAAqB;;;;;;;AAQ/C,SAAS,cAAc,UAAkD;CACvE,MAAM,SAA6B;EACjC,MAAM,SAAS;EACf,UAAU,SAAS;EACnB,mBAAmB,SAAS;EAC5B,WAAW,SAAS;EACrB;AAED,KAAI,SAAS,OACX,QAAO,SAAS;EACd,UAAU,SAAS,OAAO;EAC1B,gBAAgB,SAAS,OAAO;EAChC,YAAY,SAAS,OAAO;EAC7B;AAEH,QAAO;;AAGT,SAAS,0BAA0B,KAA6B;AAC9D,QAAO,UAAU,IAAI,SACjB,IAAI,OAAO,SAAS,SACpB,IAAI,OAAO,gBAAgB;;AAGjC,SAAgB,SAAS,KAAmB;AAC1C,QAAO,IAAI,KAAK,KAAK,WAAW,aAAa;;AAG/C,SAAgB,YAAY,QAAgB,MAAc,WAA4B;AACpF,KAAI;AACF,SAAO,OAAO,MAAM,UAAU;AAC9B,SAAO;SACD;AACN,SAAO;;;AAIX,SAAS,cAAc,OAAyC;AAC9D,QAAO;EACL,UAAU,MAAM,YAAY;EAC5B,gBAAgB,OAAO,MAAM,eAAe;EAC5C,YAAY,OAAO,MAAM,WAAW;EACrC;;;;;;;AAQH,SAAS,sBAAsB,gBAAyB,QAA0B;AAChF,KAAI,kBAAkB,OAAO,cAAc,KAAK,OAAO,kBAAkB,EACvE,QAAO;EACL,UAAU;EACV,gBAAgB,OAAO,OAAO,WAAW;EACzC,YAAY,OAAO,OAAO,WAAW;EACtC;AAGH,QAAO;;AAGT,SAAgB,sBAAsB,GAAQ;AAC5C,KAAI,GAAG,SAAS,WAAY,QAAO;AACnC,SAAQ,EAAE,MAAV;EACE,KAAK;EACL,KAAK;EACL,KAAK,iBACH,QAAO;EACT,QACE,QAAO;;;AAIb,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,aAAa,iBACb,aAAa,sBACb,aAAa;;AAIjB,SAAS,sBAAsB,GAAQ;AACrC,QAAQ,GAAa,QAAQ,SAAS,0BAA0B;;;AAIlE,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,UAAU,GAAG,IAAI;AACjD,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO;;;;AAKT,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,KAAK,MAAM,UAAU,EAAE,EAAE,IAAI;AAC7D,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO"}
+{"version":3,"file":"upload_task.js","names":[],"sources":["../../src/drivers/upload_task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport {\n  isTimeoutError,\n  resourceIdToString,\n  stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport type { AsyncPoolController, MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { asyncPool, CallersCounter } from \"@milaboratories/ts-helpers\";\nimport type { ClientProgress, ProgressStatus } from \"../clients/progress\";\nimport type { ClientUpload } from \"../clients/upload\";\nimport { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from \"../clients/upload\";\nimport type { ImportResourceSnapshot } from \"./types\";\nimport { ImportFileHandleUploadData } from \"./types\";\nimport assert from \"node:assert\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\n\n/** Holds all info needed to upload a file and a status of uploading\n * and indexing. Also, has a method to update a status of the progress.\n * And holds a change source. */\nexport class UploadTask {\n  private readonly change: ChangeSource = new ChangeSource();\n  private readonly counter: CallersCounter = new CallersCounter();\n  private nMaxUploads: number;\n  private nPartsWithThisUploadSpeed = 0;\n  private nPartsToIncreaseUpload = 10; // how many parts we have to wait to increase concurrency, 50 mb, 10 parts by 5 mb each.\n\n  /** If this is upload progress this field will be defined */\n  private uploadData?: ImportFileHandleUploadData;\n  public progress: sdk.ImportProgress;\n\n  /** If failed, then getting a progress is terminally failed. */\n  public failed?: boolean;\n\n  /** True if the blob was existed.\n   * At this case, the task will show progress == 1.0. */\n  private alreadyExisted = false;\n\n  constructor(\n    private readonly logger: MiLogger,\n    private readonly clientBlob: ClientUpload,\n    private readonly clientProgress: ClientProgress,\n    private readonly maxNConcurrentPartsUpload: number,\n    signer: Signer,\n    public readonly res: ImportResourceSnapshot,\n  ) {\n    this.nMaxUploads = this.maxNConcurrentPartsUpload;\n    const { uploadData, progress } = newProgress(res, signer);\n    this.uploadData = uploadData;\n    this.progress = progress;\n  }\n\n  public getProgress(w: Watcher, callerId: string) {\n    this.incCounter(w, callerId);\n\n    if (this.failed) {\n      this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);\n      throw new Error(this.progress.lastError);\n    }\n\n    return cloneProgress(this.progress);\n  }\n\n  public shouldScheduleUpload(): boolean {\n    return isMyUpload(this.progress);\n  }\n\n  /** Uploads a blob if it's not BlobIndex. */\n  public async uploadBlobTask() {\n    try {\n      await uploadBlob(\n        this.logger,\n        this.clientBlob,\n        this.res,\n        this.uploadData!,\n        this.isComputableDone.bind(this),\n        {\n          nPartsWithThisUploadSpeed: this.nPartsWithThisUploadSpeed,\n          nPartsToIncreaseUpload: this.nPartsToIncreaseUpload,\n          currentSpeed: this.nMaxUploads,\n          maxSpeed: this.maxNConcurrentPartsUpload,\n        },\n      );\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);\n    } catch (e: any) {\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(`terminal error while uploading a blob: ${e}`);\n        this.change.markChanged(\n          `blob upload for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n\n        return;\n      }\n\n      this.logger.error(`error while uploading a blob: ${e}`);\n      this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} failed`);\n\n      if (nonRecoverableError(e)) {\n        this.setTerminalError(e);\n        return;\n      }\n\n      this.setRetriableError(e);\n\n      if (isHeadersTimeoutError(e)) {\n        // we probably have a slow internet, we need to slow things a bit.\n        this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);\n      }\n\n      throw e;\n    }\n  }\n\n  public async updateStatus() {\n    try {\n      // we do it with timeout in case we have slow internet.\n      const status = await this.clientProgress.getStatus(this.res, { timeout: 10000 });\n\n      const oldStatus = this.progress.status;\n      const newStatus = doneProgressIfExisted(this.alreadyExisted, protoToStatus(status));\n      this.progress.status = newStatus;\n      this.setDone(status.done);\n\n      if (status.done || this.progress.status.progress != oldStatus?.progress) {\n        this.change.markChanged(`upload status for ${resourceIdToString(this.res.id)} changed`);\n      }\n    } catch (e: any) {\n      this.setRetriableError(e);\n\n      if (isTimeoutError(e)) {\n        this.logger.warn(`deadline exceeded while getting a status of BlobImport: ${e.message}`);\n        return;\n      }\n\n      if (isTerminalUploadError(e)) {\n        this.logger.warn(\n          `terminal error while updating BlobImport status: ${e}, ${stringifyWithResourceId(this.res)}`,\n        );\n        this.change.markChanged(\n          `upload status for ${resourceIdToString(this.res.id)} aborted: ${e.code}`,\n        );\n        this.setDone(true);\n        return;\n      }\n\n      this.logger.error(`retryable error while updating a status of BlobImport: ${e}`);\n      // It was a terminal error, but when a connection drops,\n      // this will stop the whole task, so we make it retryable.\n      // It was like that:\n      // this.change.markChanged();\n      // this.setTerminalError(e);\n    }\n  }\n\n  /** Set non-terminal error, that task can be retried. */\n  private setRetriableError(e: unknown) {\n    this.progress.lastError = String(e);\n  }\n\n  /** Set a terminal error, the task will throw a error instead of a progress. */\n  private setTerminalError(e: unknown) {\n    this.progress.lastError = String(e);\n    this.progress.done = false;\n    this.failed = true;\n  }\n\n  public setDoneIfOutputSet(res: ImportResourceSnapshot) {\n    if (isImportResourceOutputSet(res)) {\n      this.setDone(true);\n      this.alreadyExisted = true;\n    }\n  }\n\n  private setDone(done: boolean) {\n    this.progress.done = done;\n    if (done) this.progress.lastError = undefined;\n  }\n\n  public incCounter(w: Watcher, callerId: string) {\n    this.change.attachWatcher(w);\n    this.counter.inc(callerId);\n  }\n\n  public decCounter(callerId: string) {\n    return this.counter.dec(callerId);\n  }\n\n  private isComputableDone() {\n    return this.counter.isZero();\n  }\n}\n\n/** Uploads a blob if it's not BlobIndex. */\nexport async function uploadBlob(\n  logger: MiLogger,\n  clientBlob: ClientUpload,\n  res: ResourceInfo,\n  uploadData: ImportFileHandleUploadData,\n  isDoneFn: () => boolean,\n  speed: {\n    nPartsWithThisUploadSpeed: number;\n    nPartsToIncreaseUpload: number;\n    currentSpeed: number;\n    maxSpeed: number;\n  },\n) {\n  assert(isUpload(res), \"the upload operation can be done only for BlobUploads\");\n  const timeout = 10000; // 10 sec instead of standard 5 sec, things might be slow with a slow connection.\n\n  if (isDoneFn()) return;\n  const parts = await clientBlob.initUpload(res, { timeout });\n  logger.info(\n    `started to upload blob ${res.id},` +\n      ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length},` +\n      ` number of concurrent uploads: ${speed.currentSpeed}`,\n  );\n\n  const partUploadFn = (part: bigint) => async (controller: AsyncPoolController) => {\n    if (isDoneFn()) return;\n    await clientBlob.partUpload(\n      res,\n      uploadData.localPath,\n      BigInt(uploadData.modificationTime),\n      part,\n      parts.checksumAlgorithm,\n      parts.checksumHeader,\n      { timeout },\n    );\n    logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);\n\n    // if we had a network freeze, it will be increased slowly.\n    speed.nPartsWithThisUploadSpeed++;\n    if (speed.nPartsWithThisUploadSpeed >= speed.nPartsToIncreaseUpload) {\n      speed.nPartsWithThisUploadSpeed = 0;\n      speed.currentSpeed = increaseConcurrency(logger, speed.currentSpeed, speed.maxSpeed);\n      controller.setConcurrency(speed.currentSpeed);\n    }\n  };\n\n  await asyncPool(speed.currentSpeed, parts.toUpload.map(partUploadFn));\n\n  if (isDoneFn()) return;\n  await clientBlob.finalize(res, { timeout });\n\n  logger.info(`uploading of resource ${res.id} finished.`);\n}\n\nfunction newProgress(res: ImportResourceSnapshot, signer: Signer) {\n  let isUploadSignMatch: boolean | undefined;\n  let uploadData: ImportFileHandleUploadData | undefined;\n  if (isUpload(res)) {\n    uploadData = ImportFileHandleUploadData.parse(res.data);\n    isUploadSignMatch = isSignMatch(signer, uploadData.localPath, uploadData.pathSignature);\n  }\n\n  return {\n    uploadData: uploadData,\n    progress: {\n      done: false,\n      status: undefined,\n      isUpload: isUpload(res),\n      isUploadSignMatch: isUploadSignMatch,\n      lastError: undefined,\n    } satisfies sdk.ImportProgress,\n  };\n}\n\n/** Returns true if we need to upload the blob that got from this progress. */\nexport function isMyUpload(p: sdk.ImportProgress): boolean {\n  return p.isUpload && (p.isUploadSignMatch ?? false);\n}\n\n/** Creates a deep copy of progress,\n * since we do not want to pass a mutable object\n * to API, it led to bugs before.\n * We do not use '...' cloning syntax\n * for the compiler to fail here if we change API. */\nfunction cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {\n  const cloned: sdk.ImportProgress = {\n    done: progress.done,\n    isUpload: progress.isUpload,\n    isUploadSignMatch: progress.isUploadSignMatch,\n    lastError: progress.lastError,\n  };\n\n  if (progress.status)\n    cloned.status = {\n      progress: progress.status.progress,\n      bytesProcessed: progress.status.bytesProcessed,\n      bytesTotal: progress.status.bytesTotal,\n    };\n\n  return progress;\n}\n\nfunction isImportResourceOutputSet(res: ImportResourceSnapshot) {\n  return \"blob\" in res.fields\n    ? res.fields.blob !== undefined\n    : res.fields.incarnation !== undefined;\n}\n\nexport function isUpload(res: ResourceInfo) {\n  return res.type.name.startsWith(\"BlobUpload\");\n}\n\nexport function isSignMatch(signer: Signer, path: string, signature: string): boolean {\n  try {\n    signer.verify(path, signature);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nfunction protoToStatus(proto: ProgressStatus): sdk.ImportStatus {\n  return {\n    progress: proto.progress ?? 0,\n    bytesProcessed: Number(proto.bytesProcessed),\n    bytesTotal: Number(proto.bytesTotal),\n  };\n}\n\n/** Special hack: if we didn't even start to upload the blob\n * to backend because the result was already there,\n * the backend does show us a status that nothing were uploaded,\n * but we need to show the client that everything is OK.\n * Thus, here we set progress to be 1.0 and all bytes are become processed. */\nfunction doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus) {\n  if (alreadyExisted && status.bytesTotal != 0 && status.bytesProcessed == 0) {\n    return {\n      progress: 1.0,\n      bytesProcessed: Number(status.bytesTotal),\n      bytesTotal: Number(status.bytesTotal),\n    };\n  }\n\n  return status;\n}\n\nexport function isTerminalUploadError(e: any) {\n  if (e?.name !== \"RpcError\") return false;\n  switch (e.code) {\n    case \"NOT_FOUND\":\n    case \"ABORTED\":\n    case \"ALREADY_EXISTS\":\n      return true;\n    default:\n      return false;\n  }\n}\n\nexport function nonRecoverableError(e: any) {\n  return (\n    e instanceof MTimeError ||\n    e instanceof UnexpectedEOF ||\n    e instanceof NoFileForUploading ||\n    e instanceof BadRequestError\n  );\n}\n\nfunction isHeadersTimeoutError(e: any) {\n  return (e as Error)?.message.includes(`UND_ERR_HEADERS_TIMEOUT`);\n}\n\n/** It's called for every upload success so if everyone is succeeded, we'll double the concurrency. */\nfunction increaseConcurrency(logger: MiLogger, current: number, max: number): number {\n  const newConcurrency = Math.min(current + 2, max);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.increaseConcurrency: increased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n\n/** When a error happens, this will half the concurrency level, so the next time\n * we'll try to upload blobs slower. */\nfunction decreaseConcurrency(logger: MiLogger, current: number, min: number): number {\n  const newConcurrency = Math.max(Math.round(current / 2), min);\n  if (newConcurrency != current)\n    logger.info(`uploadTask.decreaseConcurrency: decreased from ${current} to ${newConcurrency}`);\n\n  return newConcurrency;\n}\n"],"mappings":";;;;;;;;;;AAqBA,IAAa,aAAb,MAAwB;CACtB,SAAwC,IAAI,cAAc;CAC1D,UAA2C,IAAI,gBAAgB;CAC/D;CACA,4BAAoC;CACpC,yBAAiC;;CAGjC;CACA;;CAGA;;;CAIA,iBAAyB;CAEzB,YACE,QACA,YACA,gBACA,2BACA,QACA,KACA;AANiB,OAAA,SAAA;AACA,OAAA,aAAA;AACA,OAAA,iBAAA;AACA,OAAA,4BAAA;AAED,OAAA,MAAA;AAEhB,OAAK,cAAc,KAAK;EACxB,MAAM,EAAE,YAAY,aAAa,YAAY,KAAK,OAAO;AACzD,OAAK,aAAa;AAClB,OAAK,WAAW;;CAGlB,YAAmB,GAAY,UAAkB;AAC/C,OAAK,WAAW,GAAG,SAAS;AAE5B,MAAI,KAAK,QAAQ;AACf,QAAK,OAAO,MAAM,gCAAgC,KAAK,SAAS,YAAY;AAC5E,SAAM,IAAI,MAAM,KAAK,SAAS,UAAU;;AAG1C,SAAO,cAAc,KAAK,SAAS;;CAGrC,uBAAuC;AACrC,SAAO,WAAW,KAAK,SAAS;;;CAIlC,MAAa,iBAAiB;AAC5B,MAAI;AACF,SAAM,WACJ,KAAK,QACL,KAAK,YACL,KAAK,KACL,KAAK,YACL,KAAK,iBAAiB,KAAK,KAAK,EAChC;IACE,2BAA2B,KAAK;IAChC,wBAAwB,KAAK;IAC7B,cAAc,KAAK;IACnB,UAAU,KAAK;IAChB,CACF;AACD,QAAK,OAAO,YAAY,mBAAmB,mBAAmB,KAAK,IAAI,GAAG,CAAC,WAAW;WAC/E,GAAQ;AACf,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KAAK,0CAA0C,IAAI;AAC/D,SAAK,OAAO,YACV,mBAAmB,mBAAmB,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OAClE;AACD,SAAK,QAAQ,KAAK;AAElB;;AAGF,QAAK,OAAO,MAAM,iCAAiC,IAAI;AACvD,QAAK,OAAO,YAAY,mBAAmB,mBAAmB,KAAK,IAAI,GAAG,CAAC,SAAS;AAEpF,OAAI,oBAAoB,EAAE,EAAE;AAC1B,SAAK,iBAAiB,EAAE;AACxB;;AAGF,QAAK,kBAAkB,EAAE;AAEzB,OAAI,sBAAsB,EAAE,CAE1B,MAAK,cAAc,oBAAoB,KAAK,QAAQ,KAAK,aAAa,EAAE;AAG1E,SAAM;;;CAIV,MAAa,eAAe;AAC1B,MAAI;GAEF,MAAM,SAAS,MAAM,KAAK,eAAe,UAAU,KAAK,KAAK,EAAE,SAAS,KAAO,CAAC;GAEhF,MAAM,YAAY,KAAK,SAAS;GAChC,MAAM,YAAY,sBAAsB,KAAK,gBAAgB,cAAc,OAAO,CAAC;AACnF,QAAK,SAAS,SAAS;AACvB,QAAK,QAAQ,OAAO,KAAK;AAEzB,OAAI,OAAO,QAAQ,KAAK,SAAS,OAAO,YAAY,WAAW,SAC7D,MAAK,OAAO,YAAY,qBAAqB,mBAAmB,KAAK,IAAI,GAAG,CAAC,UAAU;WAElF,GAAQ;AACf,QAAK,kBAAkB,EAAE;AAEzB,OAAI,eAAe,EAAE,EAAE;AACrB,SAAK,OAAO,KAAK,2DAA2D,EAAE,UAAU;AACxF;;AAGF,OAAI,sBAAsB,EAAE,EAAE;AAC5B,SAAK,OAAO,KACV,oDAAoD,EAAE,IAAI,wBAAwB,KAAK,IAAI,GAC5F;AACD,SAAK,OAAO,YACV,qBAAqB,mBAAmB,KAAK,IAAI,GAAG,CAAC,YAAY,EAAE,OACpE;AACD,SAAK,QAAQ,KAAK;AAClB;;AAGF,QAAK,OAAO,MAAM,0DAA0D,IAAI;;;;CAUpF,kBAA0B,GAAY;AACpC,OAAK,SAAS,YAAY,OAAO,EAAE;;;CAIrC,iBAAyB,GAAY;AACnC,OAAK,SAAS,YAAY,OAAO,EAAE;AACnC,OAAK,SAAS,OAAO;AACrB,OAAK,SAAS;;CAGhB,mBAA0B,KAA6B;AACrD,MAAI,0BAA0B,IAAI,EAAE;AAClC,QAAK,QAAQ,KAAK;AAClB,QAAK,iBAAiB;;;CAI1B,QAAgB,MAAe;AAC7B,OAAK,SAAS,OAAO;AACrB,MAAI,KAAM,MAAK,SAAS,YAAY,KAAA;;CAGtC,WAAkB,GAAY,UAAkB;AAC9C,OAAK,OAAO,cAAc,EAAE;AAC5B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,WAAkB,UAAkB;AAClC,SAAO,KAAK,QAAQ,IAAI,SAAS;;CAGnC,mBAA2B;AACzB,SAAO,KAAK,QAAQ,QAAQ;;;;AAKhC,eAAsB,WACpB,QACA,YACA,KACA,YACA,UACA,OAMA;AACA,QAAO,SAAS,IAAI,EAAE,wDAAwD;CAC9E,MAAM,UAAU;AAEhB,KAAI,UAAU,CAAE;CAChB,MAAM,QAAQ,MAAM,WAAW,WAAW,KAAK,EAAE,SAAS,CAAC;AAC3D,QAAO,KACL,0BAA0B,IAAI,GAAG,mBACZ,MAAM,QAAQ,oBAAoB,MAAM,SAAS,OAAO,kCACzC,MAAM,eAC3C;CAED,MAAM,gBAAgB,SAAiB,OAAO,eAAoC;AAChF,MAAI,UAAU,CAAE;AAChB,QAAM,WAAW,WACf,KACA,WAAW,WACX,OAAO,WAAW,iBAAiB,EACnC,MACA,MAAM,mBACN,MAAM,gBACN,EAAE,SAAS,CACZ;AACD,SAAO,KAAK,kBAAkB,KAAK,GAAG,MAAM,QAAQ,gBAAgB,IAAI,KAAK;AAG7E,QAAM;AACN,MAAI,MAAM,6BAA6B,MAAM,wBAAwB;AACnE,SAAM,4BAA4B;AAClC,SAAM,eAAe,oBAAoB,QAAQ,MAAM,cAAc,MAAM,SAAS;AACpF,cAAW,eAAe,MAAM,aAAa;;;AAIjD,OAAM,UAAU,MAAM,cAAc,MAAM,SAAS,IAAI,aAAa,CAAC;AAErE,KAAI,UAAU,CAAE;AAChB,OAAM,WAAW,SAAS,KAAK,EAAE,SAAS,CAAC;AAE3C,QAAO,KAAK,yBAAyB,IAAI,GAAG,YAAY;;AAG1D,SAAS,YAAY,KAA6B,QAAgB;CAChE,IAAI;CACJ,IAAI;AACJ,KAAI,SAAS,IAAI,EAAE;AACjB,eAAa,2BAA2B,MAAM,IAAI,KAAK;AACvD,sBAAoB,YAAY,QAAQ,WAAW,WAAW,WAAW,cAAc;;AAGzF,QAAO;EACO;EACZ,UAAU;GACR,MAAM;GACN,QAAQ,KAAA;GACR,UAAU,SAAS,IAAI;GACJ;GACnB,WAAW,KAAA;GACZ;EACF;;;AAIH,SAAgB,WAAW,GAAgC;AACzD,QAAO,EAAE,aAAa,EAAE,qBAAqB;;;;;;;AAQ/C,SAAS,cAAc,UAAkD;CACvE,MAAM,SAA6B;EACjC,MAAM,SAAS;EACf,UAAU,SAAS;EACnB,mBAAmB,SAAS;EAC5B,WAAW,SAAS;EACrB;AAED,KAAI,SAAS,OACX,QAAO,SAAS;EACd,UAAU,SAAS,OAAO;EAC1B,gBAAgB,SAAS,OAAO;EAChC,YAAY,SAAS,OAAO;EAC7B;AAEH,QAAO;;AAGT,SAAS,0BAA0B,KAA6B;AAC9D,QAAO,UAAU,IAAI,SACjB,IAAI,OAAO,SAAS,KAAA,IACpB,IAAI,OAAO,gBAAgB,KAAA;;AAGjC,SAAgB,SAAS,KAAmB;AAC1C,QAAO,IAAI,KAAK,KAAK,WAAW,aAAa;;AAG/C,SAAgB,YAAY,QAAgB,MAAc,WAA4B;AACpF,KAAI;AACF,SAAO,OAAO,MAAM,UAAU;AAC9B,SAAO;SACD;AACN,SAAO;;;AAIX,SAAS,cAAc,OAAyC;AAC9D,QAAO;EACL,UAAU,MAAM,YAAY;EAC5B,gBAAgB,OAAO,MAAM,eAAe;EAC5C,YAAY,OAAO,MAAM,WAAW;EACrC;;;;;;;AAQH,SAAS,sBAAsB,gBAAyB,QAA0B;AAChF,KAAI,kBAAkB,OAAO,cAAc,KAAK,OAAO,kBAAkB,EACvE,QAAO;EACL,UAAU;EACV,gBAAgB,OAAO,OAAO,WAAW;EACzC,YAAY,OAAO,OAAO,WAAW;EACtC;AAGH,QAAO;;AAGT,SAAgB,sBAAsB,GAAQ;AAC5C,KAAI,GAAG,SAAS,WAAY,QAAO;AACnC,SAAQ,EAAE,MAAV;EACE,KAAK;EACL,KAAK;EACL,KAAK,iBACH,QAAO;EACT,QACE,QAAO;;;AAIb,SAAgB,oBAAoB,GAAQ;AAC1C,QACE,aAAa,cACb,aAAa,iBACb,aAAa,sBACb,aAAa;;AAIjB,SAAS,sBAAsB,GAAQ;AACrC,QAAQ,GAAa,QAAQ,SAAS,0BAA0B;;;AAIlE,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,UAAU,GAAG,IAAI;AACjD,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO;;;;AAKT,SAAS,oBAAoB,QAAkB,SAAiB,KAAqB;CACnF,MAAM,iBAAiB,KAAK,IAAI,KAAK,MAAM,UAAU,EAAE,EAAE,IAAI;AAC7D,KAAI,kBAAkB,QACpB,QAAO,KAAK,kDAAkD,QAAQ,MAAM,iBAAiB;AAE/F,QAAO"}

package/dist/drivers/urls/url.cjs

@@ -1,7 +1,6 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
 let path = require("path");
 path = require_runtime.__toESM(path);
-
 //#region src/drivers/urls/url.ts
 /** Creates a new plblob+folder URL. */
 function newFolderURL(signer, saveDir, fPath) {
@@ -45,10 +44,10 @@ function parseNestedPathNoEscape(baseDir, userInputPath) {
 	if (!absolutePath.startsWith(normalizedBase)) throw new Error("Path validation failed.");
 	return absolutePath;
 }
-
 //#endregion
 exports.getPathForBlockUIURL = getPathForBlockUIURL;
 exports.getPathForFolderURL = getPathForFolderURL;
 exports.newBlockUIURL = newBlockUIURL;
 exports.newFolderURL = newFolderURL;
+
 //# sourceMappingURL=url.cjs.map

package/dist/drivers/urls/url.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"url.cjs","names":[],"sources":["../../../src/drivers/urls/url.ts"],"sourcesContent":["import type { FolderURL, BlockUIURL } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport path from \"path\";\n\n/** Creates a new plblob+folder URL. */\nexport function newFolderURL(signer: Signer, saveDir: string, fPath: string): FolderURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `plblob+folder://${sign}.${p}.blob`;\n}\n\n/** Creates a new block-ui URL. */\nexport function newBlockUIURL(signer: Signer, saveDir: string, fPath: string): BlockUIURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `block-ui://${sign}.${p}.uidir`;\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForFolderURL(signer: Signer, url: FolderURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForBlockUIURL(signer: Signer, url: BlockUIURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Parses URL,\n * checks the signature,\n * gets the absolute path by the given root directory.\n * If the path is empty, it returns the index.html file.\n * @returns an absolute path for the user. */\nfunction getPath(signer: Signer, url: string, rootDir: string): string {\n  const parsed = new URL(url);\n  const [sign, subfolder, _] = parsed.host.split(\".\");\n\n  signer.verify(\n    subfolder,\n    sign,\n    `signature verification failed for url: ${url}, subfolder: ${subfolder}`,\n  );\n\n  // Decoding changes '%20' to ' ' for example.\n  const pathname = decodeURIComponent(parsed.pathname.slice(1));\n\n  let fPath = parseNestedPathNoEscape(path.join(rootDir, `${subfolder}`), pathname);\n\n  if (parsed.pathname == \"\" || parsed.pathname == \"/\") {\n    fPath = path.join(fPath, \"index.html\");\n  }\n\n  return path.resolve(fPath);\n}\n\n/** Checks that the userInputPath is in baseDir.\n * @returns an absolute path for the user. */\nfunction parseNestedPathNoEscape(baseDir: string, userInputPath: string): string {\n  const absolutePath = path.resolve(baseDir, userInputPath);\n\n  const normalizedBase = path.resolve(baseDir);\n\n  if (!absolutePath.startsWith(normalizedBase)) {\n    throw new Error(\"Path validation failed.\");\n  }\n\n  return absolutePath;\n}\n"],"mappings":";;;;;;AAKA,SAAgB,aAAa,QAAgB,SAAiB,OAA0B;CACtF,MAAM,IAAI,aAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,mBAFM,OAAO,KAAK,EAAE,CAEI,GAAG,EAAE;;;AAItC,SAAgB,cAAc,QAAgB,SAAiB,OAA2B;CACxF,MAAM,IAAI,aAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,cAFM,OAAO,KAAK,EAAE,CAED,GAAG,EAAE;;;;AAKjC,SAAgB,oBAAoB,QAAgB,KAAgB,SAAyB;AAC3F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;AAKtC,SAAgB,qBAAqB,QAAgB,KAAiB,SAAyB;AAC7F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;;;;AAQtC,SAAS,QAAQ,QAAgB,KAAa,SAAyB;CACrE,MAAM,SAAS,IAAI,IAAI,IAAI;CAC3B,MAAM,CAAC,MAAM,WAAW,KAAK,OAAO,KAAK,MAAM,IAAI;AAEnD,QAAO,OACL,WACA,MACA,0CAA0C,IAAI,eAAe,YAC9D;CAGD,MAAM,WAAW,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;CAE7D,IAAI,QAAQ,wBAAwB,aAAK,KAAK,SAAS,GAAG,YAAY,EAAE,SAAS;AAEjF,KAAI,OAAO,YAAY,MAAM,OAAO,YAAY,IAC9C,SAAQ,aAAK,KAAK,OAAO,aAAa;AAGxC,QAAO,aAAK,QAAQ,MAAM;;;;AAK5B,SAAS,wBAAwB,SAAiB,eAA+B;CAC/E,MAAM,eAAe,aAAK,QAAQ,SAAS,cAAc;CAEzD,MAAM,iBAAiB,aAAK,QAAQ,QAAQ;AAE5C,KAAI,CAAC,aAAa,WAAW,eAAe,CAC1C,OAAM,IAAI,MAAM,0BAA0B;AAG5C,QAAO"}
+{"version":3,"file":"url.cjs","names":[],"sources":["../../../src/drivers/urls/url.ts"],"sourcesContent":["import type { FolderURL, BlockUIURL } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport path from \"path\";\n\n/** Creates a new plblob+folder URL. */\nexport function newFolderURL(signer: Signer, saveDir: string, fPath: string): FolderURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `plblob+folder://${sign}.${p}.blob`;\n}\n\n/** Creates a new block-ui URL. */\nexport function newBlockUIURL(signer: Signer, saveDir: string, fPath: string): BlockUIURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `block-ui://${sign}.${p}.uidir`;\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForFolderURL(signer: Signer, url: FolderURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForBlockUIURL(signer: Signer, url: BlockUIURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Parses URL,\n * checks the signature,\n * gets the absolute path by the given root directory.\n * If the path is empty, it returns the index.html file.\n * @returns an absolute path for the user. */\nfunction getPath(signer: Signer, url: string, rootDir: string): string {\n  const parsed = new URL(url);\n  const [sign, subfolder, _] = parsed.host.split(\".\");\n\n  signer.verify(\n    subfolder,\n    sign,\n    `signature verification failed for url: ${url}, subfolder: ${subfolder}`,\n  );\n\n  // Decoding changes '%20' to ' ' for example.\n  const pathname = decodeURIComponent(parsed.pathname.slice(1));\n\n  let fPath = parseNestedPathNoEscape(path.join(rootDir, `${subfolder}`), pathname);\n\n  if (parsed.pathname == \"\" || parsed.pathname == \"/\") {\n    fPath = path.join(fPath, \"index.html\");\n  }\n\n  return path.resolve(fPath);\n}\n\n/** Checks that the userInputPath is in baseDir.\n * @returns an absolute path for the user. */\nfunction parseNestedPathNoEscape(baseDir: string, userInputPath: string): string {\n  const absolutePath = path.resolve(baseDir, userInputPath);\n\n  const normalizedBase = path.resolve(baseDir);\n\n  if (!absolutePath.startsWith(normalizedBase)) {\n    throw new Error(\"Path validation failed.\");\n  }\n\n  return absolutePath;\n}\n"],"mappings":";;;;;AAKA,SAAgB,aAAa,QAAgB,SAAiB,OAA0B;CACtF,MAAM,IAAI,KAAA,QAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,mBAFM,OAAO,KAAK,EAAE,CAEI,GAAG,EAAE;;;AAItC,SAAgB,cAAc,QAAgB,SAAiB,OAA2B;CACxF,MAAM,IAAI,KAAA,QAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,cAFM,OAAO,KAAK,EAAE,CAED,GAAG,EAAE;;;;AAKjC,SAAgB,oBAAoB,QAAgB,KAAgB,SAAyB;AAC3F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;AAKtC,SAAgB,qBAAqB,QAAgB,KAAiB,SAAyB;AAC7F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;;;;AAQtC,SAAS,QAAQ,QAAgB,KAAa,SAAyB;CACrE,MAAM,SAAS,IAAI,IAAI,IAAI;CAC3B,MAAM,CAAC,MAAM,WAAW,KAAK,OAAO,KAAK,MAAM,IAAI;AAEnD,QAAO,OACL,WACA,MACA,0CAA0C,IAAI,eAAe,YAC9D;CAGD,MAAM,WAAW,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;CAE7D,IAAI,QAAQ,wBAAwB,KAAA,QAAK,KAAK,SAAS,GAAG,YAAY,EAAE,SAAS;AAEjF,KAAI,OAAO,YAAY,MAAM,OAAO,YAAY,IAC9C,SAAQ,KAAA,QAAK,KAAK,OAAO,aAAa;AAGxC,QAAO,KAAA,QAAK,QAAQ,MAAM;;;;AAK5B,SAAS,wBAAwB,SAAiB,eAA+B;CAC/E,MAAM,eAAe,KAAA,QAAK,QAAQ,SAAS,cAAc;CAEzD,MAAM,iBAAiB,KAAA,QAAK,QAAQ,QAAQ;AAE5C,KAAI,CAAC,aAAa,WAAW,eAAe,CAC1C,OAAM,IAAI,MAAM,0BAA0B;AAG5C,QAAO"}

package/dist/drivers/urls/url.js

@@ -1,5 +1,4 @@
 import path from "path";
-
 //#region src/drivers/urls/url.ts
 /** Creates a new plblob+folder URL. */
 function newFolderURL(signer, saveDir, fPath) {
@@ -43,7 +42,7 @@ function parseNestedPathNoEscape(baseDir, userInputPath) {
 	if (!absolutePath.startsWith(normalizedBase)) throw new Error("Path validation failed.");
 	return absolutePath;
 }
-
 //#endregion
 export { getPathForBlockUIURL, getPathForFolderURL, newBlockUIURL, newFolderURL };
+
 //# sourceMappingURL=url.js.map

package/dist/drivers/urls/url.js.map

@@ -1 +1 @@
-{"version":3,"file":"url.js","names":[],"sources":["../../../src/drivers/urls/url.ts"],"sourcesContent":["import type { FolderURL, BlockUIURL } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport path from \"path\";\n\n/** Creates a new plblob+folder URL. */\nexport function newFolderURL(signer: Signer, saveDir: string, fPath: string): FolderURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `plblob+folder://${sign}.${p}.blob`;\n}\n\n/** Creates a new block-ui URL. */\nexport function newBlockUIURL(signer: Signer, saveDir: string, fPath: string): BlockUIURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `block-ui://${sign}.${p}.uidir`;\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForFolderURL(signer: Signer, url: FolderURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForBlockUIURL(signer: Signer, url: BlockUIURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Parses URL,\n * checks the signature,\n * gets the absolute path by the given root directory.\n * If the path is empty, it returns the index.html file.\n * @returns an absolute path for the user. */\nfunction getPath(signer: Signer, url: string, rootDir: string): string {\n  const parsed = new URL(url);\n  const [sign, subfolder, _] = parsed.host.split(\".\");\n\n  signer.verify(\n    subfolder,\n    sign,\n    `signature verification failed for url: ${url}, subfolder: ${subfolder}`,\n  );\n\n  // Decoding changes '%20' to ' ' for example.\n  const pathname = decodeURIComponent(parsed.pathname.slice(1));\n\n  let fPath = parseNestedPathNoEscape(path.join(rootDir, `${subfolder}`), pathname);\n\n  if (parsed.pathname == \"\" || parsed.pathname == \"/\") {\n    fPath = path.join(fPath, \"index.html\");\n  }\n\n  return path.resolve(fPath);\n}\n\n/** Checks that the userInputPath is in baseDir.\n * @returns an absolute path for the user. */\nfunction parseNestedPathNoEscape(baseDir: string, userInputPath: string): string {\n  const absolutePath = path.resolve(baseDir, userInputPath);\n\n  const normalizedBase = path.resolve(baseDir);\n\n  if (!absolutePath.startsWith(normalizedBase)) {\n    throw new Error(\"Path validation failed.\");\n  }\n\n  return absolutePath;\n}\n"],"mappings":";;;;AAKA,SAAgB,aAAa,QAAgB,SAAiB,OAA0B;CACtF,MAAM,IAAI,KAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,mBAFM,OAAO,KAAK,EAAE,CAEI,GAAG,EAAE;;;AAItC,SAAgB,cAAc,QAAgB,SAAiB,OAA2B;CACxF,MAAM,IAAI,KAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,cAFM,OAAO,KAAK,EAAE,CAED,GAAG,EAAE;;;;AAKjC,SAAgB,oBAAoB,QAAgB,KAAgB,SAAyB;AAC3F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;AAKtC,SAAgB,qBAAqB,QAAgB,KAAiB,SAAyB;AAC7F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;;;;AAQtC,SAAS,QAAQ,QAAgB,KAAa,SAAyB;CACrE,MAAM,SAAS,IAAI,IAAI,IAAI;CAC3B,MAAM,CAAC,MAAM,WAAW,KAAK,OAAO,KAAK,MAAM,IAAI;AAEnD,QAAO,OACL,WACA,MACA,0CAA0C,IAAI,eAAe,YAC9D;CAGD,MAAM,WAAW,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;CAE7D,IAAI,QAAQ,wBAAwB,KAAK,KAAK,SAAS,GAAG,YAAY,EAAE,SAAS;AAEjF,KAAI,OAAO,YAAY,MAAM,OAAO,YAAY,IAC9C,SAAQ,KAAK,KAAK,OAAO,aAAa;AAGxC,QAAO,KAAK,QAAQ,MAAM;;;;AAK5B,SAAS,wBAAwB,SAAiB,eAA+B;CAC/E,MAAM,eAAe,KAAK,QAAQ,SAAS,cAAc;CAEzD,MAAM,iBAAiB,KAAK,QAAQ,QAAQ;AAE5C,KAAI,CAAC,aAAa,WAAW,eAAe,CAC1C,OAAM,IAAI,MAAM,0BAA0B;AAG5C,QAAO"}
+{"version":3,"file":"url.js","names":[],"sources":["../../../src/drivers/urls/url.ts"],"sourcesContent":["import type { FolderURL, BlockUIURL } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport path from \"path\";\n\n/** Creates a new plblob+folder URL. */\nexport function newFolderURL(signer: Signer, saveDir: string, fPath: string): FolderURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `plblob+folder://${sign}.${p}.blob`;\n}\n\n/** Creates a new block-ui URL. */\nexport function newBlockUIURL(signer: Signer, saveDir: string, fPath: string): BlockUIURL {\n  const p = path.relative(saveDir, fPath);\n  const sign = signer.sign(p);\n\n  return `block-ui://${sign}.${p}.uidir`;\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForFolderURL(signer: Signer, url: FolderURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Checks the signature and path injections.\n * @returns the path to the file inside the root directory. */\nexport function getPathForBlockUIURL(signer: Signer, url: BlockUIURL, rootDir: string): string {\n  return getPath(signer, url, rootDir);\n}\n\n/** Parses URL,\n * checks the signature,\n * gets the absolute path by the given root directory.\n * If the path is empty, it returns the index.html file.\n * @returns an absolute path for the user. */\nfunction getPath(signer: Signer, url: string, rootDir: string): string {\n  const parsed = new URL(url);\n  const [sign, subfolder, _] = parsed.host.split(\".\");\n\n  signer.verify(\n    subfolder,\n    sign,\n    `signature verification failed for url: ${url}, subfolder: ${subfolder}`,\n  );\n\n  // Decoding changes '%20' to ' ' for example.\n  const pathname = decodeURIComponent(parsed.pathname.slice(1));\n\n  let fPath = parseNestedPathNoEscape(path.join(rootDir, `${subfolder}`), pathname);\n\n  if (parsed.pathname == \"\" || parsed.pathname == \"/\") {\n    fPath = path.join(fPath, \"index.html\");\n  }\n\n  return path.resolve(fPath);\n}\n\n/** Checks that the userInputPath is in baseDir.\n * @returns an absolute path for the user. */\nfunction parseNestedPathNoEscape(baseDir: string, userInputPath: string): string {\n  const absolutePath = path.resolve(baseDir, userInputPath);\n\n  const normalizedBase = path.resolve(baseDir);\n\n  if (!absolutePath.startsWith(normalizedBase)) {\n    throw new Error(\"Path validation failed.\");\n  }\n\n  return absolutePath;\n}\n"],"mappings":";;;AAKA,SAAgB,aAAa,QAAgB,SAAiB,OAA0B;CACtF,MAAM,IAAI,KAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,mBAFM,OAAO,KAAK,EAAE,CAEI,GAAG,EAAE;;;AAItC,SAAgB,cAAc,QAAgB,SAAiB,OAA2B;CACxF,MAAM,IAAI,KAAK,SAAS,SAAS,MAAM;AAGvC,QAAO,cAFM,OAAO,KAAK,EAAE,CAED,GAAG,EAAE;;;;AAKjC,SAAgB,oBAAoB,QAAgB,KAAgB,SAAyB;AAC3F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;AAKtC,SAAgB,qBAAqB,QAAgB,KAAiB,SAAyB;AAC7F,QAAO,QAAQ,QAAQ,KAAK,QAAQ;;;;;;;AAQtC,SAAS,QAAQ,QAAgB,KAAa,SAAyB;CACrE,MAAM,SAAS,IAAI,IAAI,IAAI;CAC3B,MAAM,CAAC,MAAM,WAAW,KAAK,OAAO,KAAK,MAAM,IAAI;AAEnD,QAAO,OACL,WACA,MACA,0CAA0C,IAAI,eAAe,YAC9D;CAGD,MAAM,WAAW,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;CAE7D,IAAI,QAAQ,wBAAwB,KAAK,KAAK,SAAS,GAAG,YAAY,EAAE,SAAS;AAEjF,KAAI,OAAO,YAAY,MAAM,OAAO,YAAY,IAC9C,SAAQ,KAAK,KAAK,OAAO,aAAa;AAGxC,QAAO,KAAK,QAAQ,MAAM;;;;AAK5B,SAAS,wBAAwB,SAAiB,eAA+B;CAC/E,MAAM,eAAe,KAAK,QAAQ,SAAS,cAAc;CAEzD,MAAM,iBAAiB,KAAK,QAAQ,QAAQ;AAE5C,KAAI,CAAC,aAAa,WAAW,eAAe,CAC1C,OAAM,IAAI,MAAM,0BAA0B;AAG5C,QAAO"}

package/dist/drivers/virtual_storages.cjs

@@ -1,4 +1,4 @@
-const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
+const require_runtime = require("../_virtual/_rolldown/runtime.cjs");
 let path = require("path");
 path = require_runtime.__toESM(path);
 let os = require("os");
@@ -6,7 +6,6 @@ os = require_runtime.__toESM(os);
 let util = require("util");
 util = require_runtime.__toESM(util);
 let child_process = require("child_process");
-
 //#region src/drivers/virtual_storages.ts
 async function DefaultVirtualLocalStorages() {
 	const home = os.default.homedir();
@@ -35,7 +34,7 @@ async function DefaultVirtualLocalStorages() {
 		}
 	}
 }
-
 //#endregion
 exports.DefaultVirtualLocalStorages = DefaultVirtualLocalStorages;
+
 //# sourceMappingURL=virtual_storages.cjs.map

package/dist/drivers/virtual_storages.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"virtual_storages.cjs","names":["exec"],"sources":["../../src/drivers/virtual_storages.ts"],"sourcesContent":["import path from \"path\";\nimport os from \"os\";\nimport util from \"util\";\nimport { exec } from \"child_process\";\nimport type { VirtualLocalStorageSpec } from \"./types\";\n\nexport async function DefaultVirtualLocalStorages(): Promise<VirtualLocalStorageSpec[]> {\n  const home = os.homedir();\n  if (path.sep == \"/\")\n    return [\n      {\n        name: \"local\",\n        root: \"/\",\n        initialPath: home,\n      },\n    ];\n  else {\n    // determine the drive on which user's home folder is stored\n    const homeRoot = path.parse(home).root; // e.g. C:\\\n    const homeDrive = homeRoot.replaceAll(\":\\\\\", \"\"); // e.g. C drive.\n\n    // code below inspired by\n    // https://stackoverflow.com/a/52411712/769192\n\n    try {\n      const wmic = await util.promisify(exec)(\"wmic logicaldisk get name\");\n      // parsing wmic output\n      const drives = wmic.stdout\n        .split(\"\\r\\n\")\n        .filter((line) => line.includes(\":\"))\n        .map((line) => line.trim().replaceAll(\":\", \"\"));\n\n      return drives.map((drive) => {\n        const isHomeDrive = drive == homeDrive;\n        return {\n          name: `local_disk_${drive}`,\n          root: `${drive}:\\\\`,\n          initialPath: isHomeDrive ? home : `${drive}:\\\\`,\n        };\n      });\n    } catch {\n      return [\n        {\n          name: `local_disk_${homeDrive}`,\n          root: `${homeDrive}:\\\\`,\n          initialPath: home,\n        },\n      ];\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAMA,eAAsB,8BAAkE;CACtF,MAAM,OAAO,WAAG,SAAS;AACzB,KAAI,aAAK,OAAO,IACd,QAAO,CACL;EACE,MAAM;EACN,MAAM;EACN,aAAa;EACd,CACF;MACE;EAGH,MAAM,YADW,aAAK,MAAM,KAAK,CAAC,KACP,WAAW,OAAO,GAAG;AAKhD,MAAI;AAQF,WAPa,MAAM,aAAK,UAAUA,mBAAK,CAAC,4BAA4B,EAEhD,OACjB,MAAM,OAAO,CACb,QAAQ,SAAS,KAAK,SAAS,IAAI,CAAC,CACpC,KAAK,SAAS,KAAK,MAAM,CAAC,WAAW,KAAK,GAAG,CAAC,CAEnC,KAAK,UAAU;IAC3B,MAAM,cAAc,SAAS;AAC7B,WAAO;KACL,MAAM,cAAc;KACpB,MAAM,GAAG,MAAM;KACf,aAAa,cAAc,OAAO,GAAG,MAAM;KAC5C;KACD;UACI;AACN,UAAO,CACL;IACE,MAAM,cAAc;IACpB,MAAM,GAAG,UAAU;IACnB,aAAa;IACd,CACF"}
+{"version":3,"file":"virtual_storages.cjs","names":["exec"],"sources":["../../src/drivers/virtual_storages.ts"],"sourcesContent":["import path from \"path\";\nimport os from \"os\";\nimport util from \"util\";\nimport { exec } from \"child_process\";\nimport type { VirtualLocalStorageSpec } from \"./types\";\n\nexport async function DefaultVirtualLocalStorages(): Promise<VirtualLocalStorageSpec[]> {\n  const home = os.homedir();\n  if (path.sep == \"/\")\n    return [\n      {\n        name: \"local\",\n        root: \"/\",\n        initialPath: home,\n      },\n    ];\n  else {\n    // determine the drive on which user's home folder is stored\n    const homeRoot = path.parse(home).root; // e.g. C:\\\n    const homeDrive = homeRoot.replaceAll(\":\\\\\", \"\"); // e.g. C drive.\n\n    // code below inspired by\n    // https://stackoverflow.com/a/52411712/769192\n\n    try {\n      const wmic = await util.promisify(exec)(\"wmic logicaldisk get name\");\n      // parsing wmic output\n      const drives = wmic.stdout\n        .split(\"\\r\\n\")\n        .filter((line) => line.includes(\":\"))\n        .map((line) => line.trim().replaceAll(\":\", \"\"));\n\n      return drives.map((drive) => {\n        const isHomeDrive = drive == homeDrive;\n        return {\n          name: `local_disk_${drive}`,\n          root: `${drive}:\\\\`,\n          initialPath: isHomeDrive ? home : `${drive}:\\\\`,\n        };\n      });\n    } catch {\n      return [\n        {\n          name: `local_disk_${homeDrive}`,\n          root: `${homeDrive}:\\\\`,\n          initialPath: home,\n        },\n      ];\n    }\n  }\n}\n"],"mappings":";;;;;;;;;AAMA,eAAsB,8BAAkE;CACtF,MAAM,OAAO,GAAA,QAAG,SAAS;AACzB,KAAI,KAAA,QAAK,OAAO,IACd,QAAO,CACL;EACE,MAAM;EACN,MAAM;EACN,aAAa;EACd,CACF;MACE;EAGH,MAAM,YADW,KAAA,QAAK,MAAM,KAAK,CAAC,KACP,WAAW,OAAO,GAAG;AAKhD,MAAI;AAQF,WAPa,MAAM,KAAA,QAAK,UAAUA,cAAAA,KAAK,CAAC,4BAA4B,EAEhD,OACjB,MAAM,OAAO,CACb,QAAQ,SAAS,KAAK,SAAS,IAAI,CAAC,CACpC,KAAK,SAAS,KAAK,MAAM,CAAC,WAAW,KAAK,GAAG,CAAC,CAEnC,KAAK,UAAU;IAC3B,MAAM,cAAc,SAAS;AAC7B,WAAO;KACL,MAAM,cAAc;KACpB,MAAM,GAAG,MAAM;KACf,aAAa,cAAc,OAAO,GAAG,MAAM;KAC5C;KACD;UACI;AACN,UAAO,CACL;IACE,MAAM,cAAc;IACpB,MAAM,GAAG,UAAU;IACnB,aAAa;IACd,CACF"}

package/dist/drivers/virtual_storages.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"virtual_storages.d.ts","names":[],"sources":["../../src/drivers/virtual_storages.ts"],"mappings":";;;iBAMsB,2BAAA,CAAA,GAA+B,OAAA,CAAQ,uBAAA"}

package/dist/drivers/virtual_storages.js

@@ -2,7 +2,6 @@ import path from "path";
 import os from "os";
 import util from "util";
 import { exec } from "child_process";
-
 //#region src/drivers/virtual_storages.ts
 async function DefaultVirtualLocalStorages() {
 	const home = os.homedir();
@@ -31,7 +30,7 @@ async function DefaultVirtualLocalStorages() {
 		}
 	}
 }
-
 //#endregion
 export { DefaultVirtualLocalStorages };
+
 //# sourceMappingURL=virtual_storages.js.map

package/dist/drivers/virtual_storages.js.map

@@ -1 +1 @@
-{"version":3,"file":"virtual_storages.js","names":[],"sources":["../../src/drivers/virtual_storages.ts"],"sourcesContent":["import path from \"path\";\nimport os from \"os\";\nimport util from \"util\";\nimport { exec } from \"child_process\";\nimport type { VirtualLocalStorageSpec } from \"./types\";\n\nexport async function DefaultVirtualLocalStorages(): Promise<VirtualLocalStorageSpec[]> {\n  const home = os.homedir();\n  if (path.sep == \"/\")\n    return [\n      {\n        name: \"local\",\n        root: \"/\",\n        initialPath: home,\n      },\n    ];\n  else {\n    // determine the drive on which user's home folder is stored\n    const homeRoot = path.parse(home).root; // e.g. C:\\\n    const homeDrive = homeRoot.replaceAll(\":\\\\\", \"\"); // e.g. C drive.\n\n    // code below inspired by\n    // https://stackoverflow.com/a/52411712/769192\n\n    try {\n      const wmic = await util.promisify(exec)(\"wmic logicaldisk get name\");\n      // parsing wmic output\n      const drives = wmic.stdout\n        .split(\"\\r\\n\")\n        .filter((line) => line.includes(\":\"))\n        .map((line) => line.trim().replaceAll(\":\", \"\"));\n\n      return drives.map((drive) => {\n        const isHomeDrive = drive == homeDrive;\n        return {\n          name: `local_disk_${drive}`,\n          root: `${drive}:\\\\`,\n          initialPath: isHomeDrive ? home : `${drive}:\\\\`,\n        };\n      });\n    } catch {\n      return [\n        {\n          name: `local_disk_${homeDrive}`,\n          root: `${homeDrive}:\\\\`,\n          initialPath: home,\n        },\n      ];\n    }\n  }\n}\n"],"mappings":";;;;;;AAMA,eAAsB,8BAAkE;CACtF,MAAM,OAAO,GAAG,SAAS;AACzB,KAAI,KAAK,OAAO,IACd,QAAO,CACL;EACE,MAAM;EACN,MAAM;EACN,aAAa;EACd,CACF;MACE;EAGH,MAAM,YADW,KAAK,MAAM,KAAK,CAAC,KACP,WAAW,OAAO,GAAG;AAKhD,MAAI;AAQF,WAPa,MAAM,KAAK,UAAU,KAAK,CAAC,4BAA4B,EAEhD,OACjB,MAAM,OAAO,CACb,QAAQ,SAAS,KAAK,SAAS,IAAI,CAAC,CACpC,KAAK,SAAS,KAAK,MAAM,CAAC,WAAW,KAAK,GAAG,CAAC,CAEnC,KAAK,UAAU;IAC3B,MAAM,cAAc,SAAS;AAC7B,WAAO;KACL,MAAM,cAAc;KACpB,MAAM,GAAG,MAAM;KACf,aAAa,cAAc,OAAO,GAAG,MAAM;KAC5C;KACD;UACI;AACN,UAAO,CACL;IACE,MAAM,cAAc;IACpB,MAAM,GAAG,UAAU;IACnB,aAAa;IACd,CACF"}
+{"version":3,"file":"virtual_storages.js","names":[],"sources":["../../src/drivers/virtual_storages.ts"],"sourcesContent":["import path from \"path\";\nimport os from \"os\";\nimport util from \"util\";\nimport { exec } from \"child_process\";\nimport type { VirtualLocalStorageSpec } from \"./types\";\n\nexport async function DefaultVirtualLocalStorages(): Promise<VirtualLocalStorageSpec[]> {\n  const home = os.homedir();\n  if (path.sep == \"/\")\n    return [\n      {\n        name: \"local\",\n        root: \"/\",\n        initialPath: home,\n      },\n    ];\n  else {\n    // determine the drive on which user's home folder is stored\n    const homeRoot = path.parse(home).root; // e.g. C:\\\n    const homeDrive = homeRoot.replaceAll(\":\\\\\", \"\"); // e.g. C drive.\n\n    // code below inspired by\n    // https://stackoverflow.com/a/52411712/769192\n\n    try {\n      const wmic = await util.promisify(exec)(\"wmic logicaldisk get name\");\n      // parsing wmic output\n      const drives = wmic.stdout\n        .split(\"\\r\\n\")\n        .filter((line) => line.includes(\":\"))\n        .map((line) => line.trim().replaceAll(\":\", \"\"));\n\n      return drives.map((drive) => {\n        const isHomeDrive = drive == homeDrive;\n        return {\n          name: `local_disk_${drive}`,\n          root: `${drive}:\\\\`,\n          initialPath: isHomeDrive ? home : `${drive}:\\\\`,\n        };\n      });\n    } catch {\n      return [\n        {\n          name: `local_disk_${homeDrive}`,\n          root: `${homeDrive}:\\\\`,\n          initialPath: home,\n        },\n      ];\n    }\n  }\n}\n"],"mappings":";;;;;AAMA,eAAsB,8BAAkE;CACtF,MAAM,OAAO,GAAG,SAAS;AACzB,KAAI,KAAK,OAAO,IACd,QAAO,CACL;EACE,MAAM;EACN,MAAM;EACN,aAAa;EACd,CACF;MACE;EAGH,MAAM,YADW,KAAK,MAAM,KAAK,CAAC,KACP,WAAW,OAAO,GAAG;AAKhD,MAAI;AAQF,WAPa,MAAM,KAAK,UAAU,KAAK,CAAC,4BAA4B,EAEhD,OACjB,MAAM,OAAO,CACb,QAAQ,SAAS,KAAK,SAAS,IAAI,CAAC,CACpC,KAAK,SAAS,KAAK,MAAM,CAAC,WAAW,KAAK,GAAG,CAAC,CAEnC,KAAK,UAAU;IAC3B,MAAM,cAAc,SAAS;AAC7B,WAAO;KACL,MAAM,cAAc;KACpB,MAAM,GAAG,MAAM;KACf,aAAa,cAAc,OAAO,GAAG,MAAM;KAC5C;KACD;UACI;AACN,UAAO,CACL;IACE,MAAM,cAAc;IACpB,MAAM,GAAG,UAAU;IACnB,aAAa;IACd,CACF"}

package/dist/helpers/download.cjs

@@ -1,10 +1,9 @@
-const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
-const require_download_errors = require('./download_errors.cjs');
+require("../_virtual/_rolldown/runtime.cjs");
+const require_download_errors = require("./download_errors.cjs");
 let undici = require("undici");
 let node_stream = require("node:stream");
 let node_stream_web = require("node:stream/web");
 let node_stream_consumers = require("node:stream/consumers");
-
 //#region src/helpers/download.ts
 var RemoteFileDownloader = class {
 	offByOneServers = [];
@@ -71,7 +70,7 @@ async function checkStatusCodeOk(statusCode, webBody, url) {
 		throw new require_download_errors.DownloadNetworkError(statusCode, url, beginning);
 	}
 }
-
 //#endregion
 exports.RemoteFileDownloader = RemoteFileDownloader;
+
 //# sourceMappingURL=download.cjs.map

package/dist/helpers/download.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download.cjs","names":["Readable","TransformStream","OffByOneError","DownloadNetworkError400","DownloadNetworkError"],"sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport { Readable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport { TransformStream } from \"node:stream/web\";\nimport { text } from \"node:stream/consumers\";\nimport type { GetContentOptions } from \"@milaboratories/pl-model-common\";\nimport { OffByOneError, DownloadNetworkError400, DownloadNetworkError } from \"./download_errors\";\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers[\"Range\"] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const {\n      statusCode,\n      body,\n      headers: responseHeaders,\n    } = await request(url, {\n      dispatcher: this.httpClient,\n      // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n      // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n      headers: Object.fromEntries(\n        Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]),\n      ),\n      signal: ops.signal,\n      highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders[\"content-length\"]);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(\n          new (class extends TransformStream {\n            constructor(sizeBytes: number, recordOffByOne: () => void) {\n              super({\n                transform(chunk: Uint8Array, controller) {\n                  const truncatedChunk = chunk.slice(0, sizeBytes);\n                  controller.enqueue(truncatedChunk);\n                  sizeBytes -= truncatedChunk.length;\n                  if (!sizeBytes) controller.terminate();\n                },\n                flush(controller) {\n                  // Some backend versions have a bug where they return 1 less byte than requested in range.\n                  // We cannot always request one more byte because if this end byte is the last byte of the file,\n                  // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                  if (sizeBytes === 1) {\n                    recordOffByOne();\n                    controller.error(new OffByOneError());\n                  }\n                },\n              });\n            }\n          })(size, () => this.offByOneServers.push(urlOrigin)),\n        );\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n    if (400 <= statusCode && statusCode < 500)\n      throw new DownloadNetworkError400(statusCode, url, beginning);\n    throw new DownloadNetworkError(statusCode, url, beginning);\n  }\n}\n"],"mappings":";;;;;;;;AAaA,IAAa,uBAAb,MAAkC;CAChC,AAAiB,kBAA4B,EAAE;CAE/C,YAAY,AAAgB,YAAwB;EAAxB;;CAE5B,MAAM,YACJ,KACA,YACA,KACA,SACY;EACZ,MAAM,UAAU,EAAE,GAAG,YAAY;EACjC,MAAM,YAAY,IAAI,IAAI,IAAI,CAAC;AAG/B,MAAI,IAAI,OAAO;GACb,MAAM,WAAW,KAAK,gBAAgB,SAAS,UAAU;AACzD,WAAQ,WAAW,SAAS,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,MAAM,WAAW,IAAI;;EAG/E,MAAM,EACJ,YACA,MACA,SAAS,oBACP,0BAAc,KAAK;GACrB,YAAY,KAAK;GAGjB,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,IAAI,aAAa,EAAE,MAAM,CAAC,CAC1E;GACD,QAAQ,IAAI;GACZ,eAAe,IAAI,OAAO;GAC3B,CAAC;AACF,MAAI,QAAQ,gBAAgB;EAE5B,MAAM,UAAUA,qBAAS,MAAM,KAAK;EACpC,IAAI,iBAAiB;AAErB,MAAI;AACF,SAAM,kBAAkB,YAAY,SAAS,IAAI;AACjD,OAAI,QAAQ,gBAAgB;GAE5B,IAAI,SAAwB;GAE5B,MAAM,gBAAgB,OAAO,gBAAgB,kBAAkB;AAC/D,OAAI,OAAO,MAAM,cAAc,IAAI,kBAAkB,EAInD,UAAS,MAAM,QAAQ,SAAS,EAAE;QAC7B;IAGL,MAAM,OAAO,IAAI,QAAQ,IAAI,MAAM,KAAK,IAAI,MAAM,OAAO;AAwBzD,aAAS,MAAM,QAvBU,QAAQ,YAC/B,IAAK,cAAcC,gCAAgB;KACjC,YAAY,WAAmB,gBAA4B;AACzD,YAAM;OACJ,UAAU,OAAmB,YAAY;QACvC,MAAM,iBAAiB,MAAM,MAAM,GAAG,UAAU;AAChD,mBAAW,QAAQ,eAAe;AAClC,qBAAa,eAAe;AAC5B,YAAI,CAAC,UAAW,YAAW,WAAW;;OAExC,MAAM,YAAY;AAIhB,YAAI,cAAc,GAAG;AACnB,yBAAgB;AAChB,oBAAW,MAAM,IAAIC,uCAAe,CAAC;;;OAG1C,CAAC;;MAEH,YAAY,KAAK,gBAAgB,KAAK,UAAU,CAAC,CACrD,EACwC,KAAK;;AAGhD,oBAAiB;AACjB,UAAO;WACA,OAAO;AAEd,OAAI,CAAC,kBAAkB,CAAC,QAAQ,OAC9B,KAAI;AACF,UAAM,QAAQ,QAAQ;WAChB;AAIV,SAAM;;;;AAKZ,eAAe,kBAAkB,YAAoB,SAAyB,KAAa;AACzF,KAAI,cAAc,OAAO,cAAc,KAA8C;EACnF,MAAM,aAAa,sCAAW,QAAQ,EAAE,UAAU,GAAG,IAAK;AAC1D,MAAI,OAAO,cAAc,aAAa,IACpC,OAAM,IAAIC,gDAAwB,YAAY,KAAK,UAAU;AAC/D,QAAM,IAAIC,6CAAqB,YAAY,KAAK,UAAU"}
+{"version":3,"file":"download.cjs","names":["Readable","TransformStream","OffByOneError","DownloadNetworkError400","DownloadNetworkError"],"sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport { Readable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport { TransformStream } from \"node:stream/web\";\nimport { text } from \"node:stream/consumers\";\nimport type { GetContentOptions } from \"@milaboratories/pl-model-common\";\nimport { OffByOneError, DownloadNetworkError400, DownloadNetworkError } from \"./download_errors\";\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers[\"Range\"] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const {\n      statusCode,\n      body,\n      headers: responseHeaders,\n    } = await request(url, {\n      dispatcher: this.httpClient,\n      // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n      // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n      headers: Object.fromEntries(\n        Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]),\n      ),\n      signal: ops.signal,\n      highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders[\"content-length\"]);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(\n          new (class extends TransformStream {\n            constructor(sizeBytes: number, recordOffByOne: () => void) {\n              super({\n                transform(chunk: Uint8Array, controller) {\n                  const truncatedChunk = chunk.slice(0, sizeBytes);\n                  controller.enqueue(truncatedChunk);\n                  sizeBytes -= truncatedChunk.length;\n                  if (!sizeBytes) controller.terminate();\n                },\n                flush(controller) {\n                  // Some backend versions have a bug where they return 1 less byte than requested in range.\n                  // We cannot always request one more byte because if this end byte is the last byte of the file,\n                  // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                  if (sizeBytes === 1) {\n                    recordOffByOne();\n                    controller.error(new OffByOneError());\n                  }\n                },\n              });\n            }\n          })(size, () => this.offByOneServers.push(urlOrigin)),\n        );\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n    if (400 <= statusCode && statusCode < 500)\n      throw new DownloadNetworkError400(statusCode, url, beginning);\n    throw new DownloadNetworkError(statusCode, url, beginning);\n  }\n}\n"],"mappings":";;;;;;;AAaA,IAAa,uBAAb,MAAkC;CAChC,kBAA6C,EAAE;CAE/C,YAAY,YAAwC;AAAxB,OAAA,aAAA;;CAE5B,MAAM,YACJ,KACA,YACA,KACA,SACY;EACZ,MAAM,UAAU,EAAE,GAAG,YAAY;EACjC,MAAM,YAAY,IAAI,IAAI,IAAI,CAAC;AAG/B,MAAI,IAAI,OAAO;GACb,MAAM,WAAW,KAAK,gBAAgB,SAAS,UAAU;AACzD,WAAQ,WAAW,SAAS,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,MAAM,WAAW,IAAI;;EAG/E,MAAM,EACJ,YACA,MACA,SAAS,oBACP,OAAA,GAAA,OAAA,SAAc,KAAK;GACrB,YAAY,KAAK;GAGjB,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,IAAI,aAAa,EAAE,MAAM,CAAC,CAC1E;GACD,QAAQ,IAAI;GACZ,eAAe,IAAI,OAAO;GAC3B,CAAC;AACF,MAAI,QAAQ,gBAAgB;EAE5B,MAAM,UAAUA,YAAAA,SAAS,MAAM,KAAK;EACpC,IAAI,iBAAiB;AAErB,MAAI;AACF,SAAM,kBAAkB,YAAY,SAAS,IAAI;AACjD,OAAI,QAAQ,gBAAgB;GAE5B,IAAI,SAAwB,KAAA;GAE5B,MAAM,gBAAgB,OAAO,gBAAgB,kBAAkB;AAC/D,OAAI,OAAO,MAAM,cAAc,IAAI,kBAAkB,EAInD,UAAS,MAAM,QAAQ,SAAS,EAAE;QAC7B;IAGL,MAAM,OAAO,IAAI,QAAQ,IAAI,MAAM,KAAK,IAAI,MAAM,OAAO;AAwBzD,aAAS,MAAM,QAvBU,QAAQ,YAC/B,IAAK,cAAcC,gBAAAA,gBAAgB;KACjC,YAAY,WAAmB,gBAA4B;AACzD,YAAM;OACJ,UAAU,OAAmB,YAAY;QACvC,MAAM,iBAAiB,MAAM,MAAM,GAAG,UAAU;AAChD,mBAAW,QAAQ,eAAe;AAClC,qBAAa,eAAe;AAC5B,YAAI,CAAC,UAAW,YAAW,WAAW;;OAExC,MAAM,YAAY;AAIhB,YAAI,cAAc,GAAG;AACnB,yBAAgB;AAChB,oBAAW,MAAM,IAAIC,wBAAAA,eAAe,CAAC;;;OAG1C,CAAC;;MAEH,YAAY,KAAK,gBAAgB,KAAK,UAAU,CAAC,CACrD,EACwC,KAAK;;AAGhD,oBAAiB;AACjB,UAAO;WACA,OAAO;AAEd,OAAI,CAAC,kBAAkB,CAAC,QAAQ,OAC9B,KAAI;AACF,UAAM,QAAQ,QAAQ;WAChB;AAIV,SAAM;;;;AAKZ,eAAe,kBAAkB,YAAoB,SAAyB,KAAa;AACzF,KAAI,cAAc,OAAO,cAAc,KAA8C;EACnF,MAAM,aAAa,OAAA,GAAA,sBAAA,MAAW,QAAQ,EAAE,UAAU,GAAG,IAAK;AAC1D,MAAI,OAAO,cAAc,aAAa,IACpC,OAAM,IAAIC,wBAAAA,wBAAwB,YAAY,KAAK,UAAU;AAC/D,QAAM,IAAIC,wBAAAA,qBAAqB,YAAY,KAAK,UAAU"}

package/dist/helpers/download.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"download.d.ts","names":[],"sources":["../../src/helpers/download.ts"],"mappings":";;;;;KAWY,gBAAA,OAAqB,OAAA,EAAS,cAAA,EAAgB,IAAA,aAAiB,OAAA,CAAQ,CAAA;AAAA,cAEtE,oBAAA;EAAA,SAGiB,UAAA,EAAY,UAAA;EAAA,iBAFvB,eAAA;cAEW,UAAA,EAAY,UAAA;EAElC,WAAA,GAAA,CACJ,GAAA,UACA,UAAA,EAAY,MAAA,kBACZ,GAAA,EAAK,iBAAA,EACL,OAAA,EAAS,gBAAA,CAAe,CAAA,IACvB,OAAA,CAAQ,CAAA;AAAA"}

package/dist/helpers/download.js

@@ -3,7 +3,6 @@ import { request } from "undici";
 import { Readable } from "node:stream";
 import { TransformStream } from "node:stream/web";
 import { text } from "node:stream/consumers";
-
 //#region src/helpers/download.ts
 var RemoteFileDownloader = class {
 	offByOneServers = [];
@@ -70,7 +69,7 @@ async function checkStatusCodeOk(statusCode, webBody, url) {
 		throw new DownloadNetworkError(statusCode, url, beginning);
 	}
 }
-
 //#endregion
 export { RemoteFileDownloader };
+
 //# sourceMappingURL=download.js.map

package/dist/helpers/download.js.map

@@ -1 +1 @@
-{"version":3,"file":"download.js","names":[],"sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport { Readable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport { TransformStream } from \"node:stream/web\";\nimport { text } from \"node:stream/consumers\";\nimport type { GetContentOptions } from \"@milaboratories/pl-model-common\";\nimport { OffByOneError, DownloadNetworkError400, DownloadNetworkError } from \"./download_errors\";\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers[\"Range\"] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const {\n      statusCode,\n      body,\n      headers: responseHeaders,\n    } = await request(url, {\n      dispatcher: this.httpClient,\n      // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n      // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n      headers: Object.fromEntries(\n        Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]),\n      ),\n      signal: ops.signal,\n      highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders[\"content-length\"]);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(\n          new (class extends TransformStream {\n            constructor(sizeBytes: number, recordOffByOne: () => void) {\n              super({\n                transform(chunk: Uint8Array, controller) {\n                  const truncatedChunk = chunk.slice(0, sizeBytes);\n                  controller.enqueue(truncatedChunk);\n                  sizeBytes -= truncatedChunk.length;\n                  if (!sizeBytes) controller.terminate();\n                },\n                flush(controller) {\n                  // Some backend versions have a bug where they return 1 less byte than requested in range.\n                  // We cannot always request one more byte because if this end byte is the last byte of the file,\n                  // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                  if (sizeBytes === 1) {\n                    recordOffByOne();\n                    controller.error(new OffByOneError());\n                  }\n                },\n              });\n            }\n          })(size, () => this.offByOneServers.push(urlOrigin)),\n        );\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n    if (400 <= statusCode && statusCode < 500)\n      throw new DownloadNetworkError400(statusCode, url, beginning);\n    throw new DownloadNetworkError(statusCode, url, beginning);\n  }\n}\n"],"mappings":";;;;;;;AAaA,IAAa,uBAAb,MAAkC;CAChC,AAAiB,kBAA4B,EAAE;CAE/C,YAAY,AAAgB,YAAwB;EAAxB;;CAE5B,MAAM,YACJ,KACA,YACA,KACA,SACY;EACZ,MAAM,UAAU,EAAE,GAAG,YAAY;EACjC,MAAM,YAAY,IAAI,IAAI,IAAI,CAAC;AAG/B,MAAI,IAAI,OAAO;GACb,MAAM,WAAW,KAAK,gBAAgB,SAAS,UAAU;AACzD,WAAQ,WAAW,SAAS,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,MAAM,WAAW,IAAI;;EAG/E,MAAM,EACJ,YACA,MACA,SAAS,oBACP,MAAM,QAAQ,KAAK;GACrB,YAAY,KAAK;GAGjB,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,IAAI,aAAa,EAAE,MAAM,CAAC,CAC1E;GACD,QAAQ,IAAI;GACZ,eAAe,IAAI,OAAO;GAC3B,CAAC;AACF,MAAI,QAAQ,gBAAgB;EAE5B,MAAM,UAAU,SAAS,MAAM,KAAK;EACpC,IAAI,iBAAiB;AAErB,MAAI;AACF,SAAM,kBAAkB,YAAY,SAAS,IAAI;AACjD,OAAI,QAAQ,gBAAgB;GAE5B,IAAI,SAAwB;GAE5B,MAAM,gBAAgB,OAAO,gBAAgB,kBAAkB;AAC/D,OAAI,OAAO,MAAM,cAAc,IAAI,kBAAkB,EAInD,UAAS,MAAM,QAAQ,SAAS,EAAE;QAC7B;IAGL,MAAM,OAAO,IAAI,QAAQ,IAAI,MAAM,KAAK,IAAI,MAAM,OAAO;AAwBzD,aAAS,MAAM,QAvBU,QAAQ,YAC/B,IAAK,cAAc,gBAAgB;KACjC,YAAY,WAAmB,gBAA4B;AACzD,YAAM;OACJ,UAAU,OAAmB,YAAY;QACvC,MAAM,iBAAiB,MAAM,MAAM,GAAG,UAAU;AAChD,mBAAW,QAAQ,eAAe;AAClC,qBAAa,eAAe;AAC5B,YAAI,CAAC,UAAW,YAAW,WAAW;;OAExC,MAAM,YAAY;AAIhB,YAAI,cAAc,GAAG;AACnB,yBAAgB;AAChB,oBAAW,MAAM,IAAI,eAAe,CAAC;;;OAG1C,CAAC;;MAEH,YAAY,KAAK,gBAAgB,KAAK,UAAU,CAAC,CACrD,EACwC,KAAK;;AAGhD,oBAAiB;AACjB,UAAO;WACA,OAAO;AAEd,OAAI,CAAC,kBAAkB,CAAC,QAAQ,OAC9B,KAAI;AACF,UAAM,QAAQ,QAAQ;WAChB;AAIV,SAAM;;;;AAKZ,eAAe,kBAAkB,YAAoB,SAAyB,KAAa;AACzF,KAAI,cAAc,OAAO,cAAc,KAA8C;EACnF,MAAM,aAAa,MAAM,KAAK,QAAQ,EAAE,UAAU,GAAG,IAAK;AAC1D,MAAI,OAAO,cAAc,aAAa,IACpC,OAAM,IAAI,wBAAwB,YAAY,KAAK,UAAU;AAC/D,QAAM,IAAI,qBAAqB,YAAY,KAAK,UAAU"}
+{"version":3,"file":"download.js","names":[],"sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { Dispatcher } from \"undici\";\nimport { request } from \"undici\";\nimport { Readable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport { TransformStream } from \"node:stream/web\";\nimport { text } from \"node:stream/consumers\";\nimport type { GetContentOptions } from \"@milaboratories/pl-model-common\";\nimport { OffByOneError, DownloadNetworkError400, DownloadNetworkError } from \"./download_errors\";\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\nexport class RemoteFileDownloader {\n  private readonly offByOneServers: string[] = [];\n\n  constructor(public readonly httpClient: Dispatcher) {}\n\n  async withContent<T>(\n    url: string,\n    reqHeaders: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const headers = { ...reqHeaders };\n    const urlOrigin = new URL(url).origin;\n\n    // Add range header if specified\n    if (ops.range) {\n      const offByOne = this.offByOneServers.includes(urlOrigin);\n      headers[\"Range\"] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n    }\n\n    const {\n      statusCode,\n      body,\n      headers: responseHeaders,\n    } = await request(url, {\n      dispatcher: this.httpClient,\n      // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n      // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n      headers: Object.fromEntries(\n        Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]),\n      ),\n      signal: ops.signal,\n      highWaterMark: 1 * 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n    });\n    ops.signal?.throwIfAborted();\n\n    const webBody = Readable.toWeb(body);\n    let handlerSuccess = false;\n\n    try {\n      await checkStatusCodeOk(statusCode, webBody, url);\n      ops.signal?.throwIfAborted();\n\n      let result: T | undefined = undefined;\n\n      const contentLength = Number(responseHeaders[\"content-length\"]);\n      if (Number.isNaN(contentLength) || contentLength === 0) {\n        // Some backend versions have a bug that they are not returning content-length header.\n        // In this case `content-length` header is returned as 0.\n        // We should not clip the result stream to 0 bytes in such case.\n        result = await handler(webBody, 0);\n      } else {\n        // Some backend versions have a bug where they return more data than requested in range.\n        // So we have to manually normalize the stream to the expected size.\n        const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n        const normalizedStream = webBody.pipeThrough(\n          new (class extends TransformStream {\n            constructor(sizeBytes: number, recordOffByOne: () => void) {\n              super({\n                transform(chunk: Uint8Array, controller) {\n                  const truncatedChunk = chunk.slice(0, sizeBytes);\n                  controller.enqueue(truncatedChunk);\n                  sizeBytes -= truncatedChunk.length;\n                  if (!sizeBytes) controller.terminate();\n                },\n                flush(controller) {\n                  // Some backend versions have a bug where they return 1 less byte than requested in range.\n                  // We cannot always request one more byte because if this end byte is the last byte of the file,\n                  // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n                  if (sizeBytes === 1) {\n                    recordOffByOne();\n                    controller.error(new OffByOneError());\n                  }\n                },\n              });\n            }\n          })(size, () => this.offByOneServers.push(urlOrigin)),\n        );\n        result = await handler(normalizedStream, size);\n      }\n\n      handlerSuccess = true;\n      return result;\n    } catch (error) {\n      // Cleanup on error (including handler errors)\n      if (!handlerSuccess && !webBody.locked) {\n        try {\n          await webBody.cancel();\n        } catch {\n          // Ignore cleanup errors\n        }\n      }\n      throw error;\n    }\n  }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n  if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n    const beginning = (await text(webBody)).substring(0, 1000);\n    if (400 <= statusCode && statusCode < 500)\n      throw new DownloadNetworkError400(statusCode, url, beginning);\n    throw new DownloadNetworkError(statusCode, url, beginning);\n  }\n}\n"],"mappings":";;;;;;AAaA,IAAa,uBAAb,MAAkC;CAChC,kBAA6C,EAAE;CAE/C,YAAY,YAAwC;AAAxB,OAAA,aAAA;;CAE5B,MAAM,YACJ,KACA,YACA,KACA,SACY;EACZ,MAAM,UAAU,EAAE,GAAG,YAAY;EACjC,MAAM,YAAY,IAAI,IAAI,IAAI,CAAC;AAG/B,MAAI,IAAI,OAAO;GACb,MAAM,WAAW,KAAK,gBAAgB,SAAS,UAAU;AACzD,WAAQ,WAAW,SAAS,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,MAAM,WAAW,IAAI;;EAG/E,MAAM,EACJ,YACA,MACA,SAAS,oBACP,MAAM,QAAQ,KAAK;GACrB,YAAY,KAAK;GAGjB,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,IAAI,aAAa,EAAE,MAAM,CAAC,CAC1E;GACD,QAAQ,IAAI;GACZ,eAAe,IAAI,OAAO;GAC3B,CAAC;AACF,MAAI,QAAQ,gBAAgB;EAE5B,MAAM,UAAU,SAAS,MAAM,KAAK;EACpC,IAAI,iBAAiB;AAErB,MAAI;AACF,SAAM,kBAAkB,YAAY,SAAS,IAAI;AACjD,OAAI,QAAQ,gBAAgB;GAE5B,IAAI,SAAwB,KAAA;GAE5B,MAAM,gBAAgB,OAAO,gBAAgB,kBAAkB;AAC/D,OAAI,OAAO,MAAM,cAAc,IAAI,kBAAkB,EAInD,UAAS,MAAM,QAAQ,SAAS,EAAE;QAC7B;IAGL,MAAM,OAAO,IAAI,QAAQ,IAAI,MAAM,KAAK,IAAI,MAAM,OAAO;AAwBzD,aAAS,MAAM,QAvBU,QAAQ,YAC/B,IAAK,cAAc,gBAAgB;KACjC,YAAY,WAAmB,gBAA4B;AACzD,YAAM;OACJ,UAAU,OAAmB,YAAY;QACvC,MAAM,iBAAiB,MAAM,MAAM,GAAG,UAAU;AAChD,mBAAW,QAAQ,eAAe;AAClC,qBAAa,eAAe;AAC5B,YAAI,CAAC,UAAW,YAAW,WAAW;;OAExC,MAAM,YAAY;AAIhB,YAAI,cAAc,GAAG;AACnB,yBAAgB;AAChB,oBAAW,MAAM,IAAI,eAAe,CAAC;;;OAG1C,CAAC;;MAEH,YAAY,KAAK,gBAAgB,KAAK,UAAU,CAAC,CACrD,EACwC,KAAK;;AAGhD,oBAAiB;AACjB,UAAO;WACA,OAAO;AAEd,OAAI,CAAC,kBAAkB,CAAC,QAAQ,OAC9B,KAAI;AACF,UAAM,QAAQ,QAAQ;WAChB;AAIV,SAAM;;;;AAKZ,eAAe,kBAAkB,YAAoB,SAAyB,KAAa;AACzF,KAAI,cAAc,OAAO,cAAc,KAA8C;EACnF,MAAM,aAAa,MAAM,KAAK,QAAQ,EAAE,UAAU,GAAG,IAAK;AAC1D,MAAI,OAAO,cAAc,aAAa,IACpC,OAAM,IAAI,wBAAwB,YAAY,KAAK,UAAU;AAC/D,QAAM,IAAI,qBAAqB,YAAY,KAAK,UAAU"}

package/dist/helpers/download_errors.cjs

@@ -1,4 +1,3 @@
-
 //#region src/helpers/download_errors.ts
 var DownloadNetworkError = class extends Error {
 	name = "DownloadNetworkError";
@@ -33,7 +32,6 @@ var OffByOneError = class extends Error {
 function isOffByOneError(error) {
 	return error instanceof Error && error.name === "OffByOneError";
 }
-
 //#endregion
 exports.DownloadNetworkError = DownloadNetworkError;
 exports.DownloadNetworkError400 = DownloadNetworkError400;
@@ -41,4 +39,5 @@ exports.OffByOneError = OffByOneError;
 exports.isDownloadNetworkError = isDownloadNetworkError;
 exports.isDownloadNetworkError400 = isDownloadNetworkError400;
 exports.isOffByOneError = isOffByOneError;
+
 //# sourceMappingURL=download_errors.cjs.map

package/dist/helpers/download_errors.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download_errors.cjs","names":[],"sources":["../../src/helpers/download_errors.ts"],"sourcesContent":["export class DownloadNetworkError extends Error {\n  name = \"DownloadNetworkError\";\n\n  statusCode: number;\n  url: string;\n  beginning: string;\n\n  constructor(statusCode: number, url: string, beginning: string) {\n    super(\n      `Http download error: statusCode: ${statusCode} url: ${url.toString()}, beginning of body: ${beginning}`,\n    );\n    this.statusCode = statusCode;\n    this.url = url;\n    this.beginning = beginning;\n  }\n}\n\nexport function isDownloadNetworkError(error: unknown): error is DownloadNetworkError {\n  return error instanceof Error && error.name.startsWith(\"DownloadNetworkError\");\n}\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class DownloadNetworkError400 extends DownloadNetworkError {\n  name = \"DownloadNetworkError400\";\n}\n\nexport function isDownloadNetworkError400(error: unknown): error is DownloadNetworkError400 {\n  return error instanceof Error && error.name === \"DownloadNetworkError400\";\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = \"OffByOneError\";\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === \"OffByOneError\";\n}\n"],"mappings":";;AAAA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;CAEP;CACA;CACA;CAEA,YAAY,YAAoB,KAAa,WAAmB;AAC9D,QACE,oCAAoC,WAAW,QAAQ,IAAI,UAAU,CAAC,uBAAuB,YAC9F;AACD,OAAK,aAAa;AAClB,OAAK,MAAM;AACX,OAAK,YAAY;;;AAIrB,SAAgB,uBAAuB,OAA+C;AACpF,QAAO,iBAAiB,SAAS,MAAM,KAAK,WAAW,uBAAuB;;;AAIhF,IAAa,0BAAb,cAA6C,qBAAqB;CAChE,OAAO;;AAGT,SAAgB,0BAA0B,OAAkD;AAC1F,QAAO,iBAAiB,SAAS,MAAM,SAAS;;;;;;;AAQlD,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAO;;AAGT,SAAgB,gBAAgB,OAAwC;AACtE,QAAO,iBAAiB,SAAS,MAAM,SAAS"}
+{"version":3,"file":"download_errors.cjs","names":[],"sources":["../../src/helpers/download_errors.ts"],"sourcesContent":["export class DownloadNetworkError extends Error {\n  name = \"DownloadNetworkError\";\n\n  statusCode: number;\n  url: string;\n  beginning: string;\n\n  constructor(statusCode: number, url: string, beginning: string) {\n    super(\n      `Http download error: statusCode: ${statusCode} url: ${url.toString()}, beginning of body: ${beginning}`,\n    );\n    this.statusCode = statusCode;\n    this.url = url;\n    this.beginning = beginning;\n  }\n}\n\nexport function isDownloadNetworkError(error: unknown): error is DownloadNetworkError {\n  return error instanceof Error && error.name.startsWith(\"DownloadNetworkError\");\n}\n\n/** Throws when a status code of the downloading URL was in range [400, 500). */\nexport class DownloadNetworkError400 extends DownloadNetworkError {\n  name = \"DownloadNetworkError400\";\n}\n\nexport function isDownloadNetworkError400(error: unknown): error is DownloadNetworkError400 {\n  return error instanceof Error && error.name === \"DownloadNetworkError400\";\n}\n\n/**\n * There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n */\nexport class OffByOneError extends Error {\n  name = \"OffByOneError\";\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n  return error instanceof Error && error.name === \"OffByOneError\";\n}\n"],"mappings":";AAAA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAO;CAEP;CACA;CACA;CAEA,YAAY,YAAoB,KAAa,WAAmB;AAC9D,QACE,oCAAoC,WAAW,QAAQ,IAAI,UAAU,CAAC,uBAAuB,YAC9F;AACD,OAAK,aAAa;AAClB,OAAK,MAAM;AACX,OAAK,YAAY;;;AAIrB,SAAgB,uBAAuB,OAA+C;AACpF,QAAO,iBAAiB,SAAS,MAAM,KAAK,WAAW,uBAAuB;;;AAIhF,IAAa,0BAAb,cAA6C,qBAAqB;CAChE,OAAO;;AAGT,SAAgB,0BAA0B,OAAkD;AAC1F,QAAO,iBAAiB,SAAS,MAAM,SAAS;;;;;;;AAQlD,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAO;;AAGT,SAAgB,gBAAgB,OAAwC;AACtE,QAAO,iBAAiB,SAAS,MAAM,SAAS"}

package/dist/helpers/download_errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"download_errors.d.ts","names":[],"sources":["../../src/helpers/download_errors.ts"],"mappings":";cAAa,oBAAA,SAA6B,KAAA;EACxC,IAAA;EAEA,UAAA;EACA,GAAA;EACA,SAAA;cAEY,UAAA,UAAoB,GAAA,UAAa,SAAA;AAAA;AAAA,iBAU/B,sBAAA,CAAuB,KAAA,YAAiB,KAAA,IAAS,oBAAA;;cAKpD,uBAAA,SAAgC,oBAAA;EAC3C,IAAA;AAAA;AAAA,iBAGc,yBAAA,CAA0B,KAAA,YAAiB,KAAA,IAAS,uBAAA;;;;AATpE;;cAkBa,aAAA,SAAsB,KAAA;EACjC,IAAA;AAAA;AAAA,iBAGc,eAAA,CAAgB,KAAA,YAAiB,KAAA,IAAS,aAAA"}