npm - @milaboratories/pl-drivers - Versions diffs - 1.11.18 → 1.11.20 - Mend

@milaboratories/pl-drivers 1.11.18 → 1.11.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (265) hide show

package/dist/drivers/download_blob_url/task.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.cjs","sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport path from 'path';\nimport fs from 'fs';\nimport * as fsp from 'fs/promises';\nimport { NetworkError400 } from '../../helpers/download';\nimport type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from '@milaboratories/ts-helpers';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { UnknownStorageError, WrongLocalFileUrl, type ClientDownload } from '../../clients/download';\nimport type { ArchiveFormat, FolderURL } from '@milaboratories/pl-model-common';\nimport { newFolderURL } from '../urls/url';\nimport decompress from 'decompress';\nimport { assertNever } from '@protobuf-ts/runtime';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} finished`);\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(`a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`);\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo, \n {}, \n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case 'tar':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case 'tgz':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content\n .pipeThrough(gunzip, { signal })\n .pipeTo(untar, { signal });\n return;\n\n case 'zip':\n this.state!.zipPath = this.path + '.zip';\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: file => !file.path.endsWith('/'),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n }\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = 'URLAborted';\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted\n \|\| e instanceof NetworkError400\n \|\| e instanceof UnknownStorageError\n \|\| e instanceof WrongLocalFileUrl\n // file that we downloads from was moved or deleted.\n \|\| e?.code == 'ENOENT'\n // A resource was deleted.\n \|\| (e.name == 'RpcError' && (e.code == 'NOT_FOUND' \|\| e.code == 'ABORTED'))\n // wrong archive format\n \|\| (String(e).includes('incorrect header check'))\n );\n}\n"],"names":["CallersCounter","ChangeSource","resourceIdToString","stringifyWithResourceId","ensureDirExists","fileExists","createPathAtomically","fsp","Writable","tar","Transform","zlib","assertNever","notEmpty","newFolderURL","NetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA;MACa,wBAAwB,CAAA;AAWhB,IAAA,MAAA;AACA,IAAA,MAAA;AACR,IAAA,OAAA;AACA,IAAA,IAAA;AACA,IAAA,KAAA;AACA,IAAA,MAAA;AACQ,IAAA,cAAA;AAhBV,IAAA,OAAO,GAAG,IAAIA,wBAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAIC,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;AACA,IAAA,GAAG;AACH,IAAA,KAAK;AAEb,IAAA,WAAA,CACmB,MAAgB,EAChB,MAAc,EACtB,OAAe,EACf,IAAY,EACZ,KAA+B,EAC/B,MAAqB,EACb,cAA8B,EAAA;QAN9B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,MAAM,GAAN,MAAM;QACE,IAAA,CAAA,cAAc,GAAd,cAAc;IAC9B;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACpE,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+BC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;AAEpG,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,0BAAA,EAA6BC,gCAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;QACvF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA,uBAAA,EAA0BA,gCAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;AAE5G,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+BD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAElG,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,qBAAqB,CAAC,MAAmB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;AAEf,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,MAAME,yBAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;AAE3C,QAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,IAAI,CAAC;AACpD,QAAA,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;AAC1B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACpD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAM,CAAC,UAAU,GAAG,IAAI;AAE7B,YAAA,MAAMC,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;AACzE,gBAAA,IAAI,CAAC,KAAM,CAAC,QAAQ,GAAG,KAAK;gBAC5B,IAAI,CAAC,KAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM;AAEvC,gBAAA,QAAQ,IAAI,CAAC,MAAM;AACjB,oBAAA,KAAK,KAAK;wBACR,MAAMC,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,wBAAA,MAAM,WAAW,GAAGC,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACtD,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC7C;AAEF,oBAAA,KAAK,KAAK;wBACR,MAAMF,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAGG,qBAAS,CAAC,KAAK,CAACC,eAAI,CAAC,YAAY,EAAE,CAAC;AACnD,wBAAA,MAAM,KAAK,GAAGH,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEhD,wBAAA,MAAM;AACH,6BAAA,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE;AAC9B,6BAAA,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC5B;AAEF,oBAAA,KAAK,KAAK;wBACR,IAAI,CAAC,KAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM;AAExC,wBAAA,MAAM,CAAC,GAAGD,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC;wBACnE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC;AACnC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;;;;;wBAMjC,MAAM,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,EAAE,KAAK,EAAE;AAC3C,4BAAA,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;AACzC,yBAAA,CAAC;AACF,wBAAA,IAAI,CAAC,KAAM,CAAC,eAAe,GAAG,IAAI;AAElC,wBAAA,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC;AACzC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;wBAEjC;AAEF,oBAAA;AACE,wBAAAI,mBAAW,CAAC,IAAI,CAAC,MAAM,CAAC;;AAE9B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAM,CAAC,WAAW,GAAG,IAAI;AAC9B,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,GAAG,EAAEC,kBAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAEjD,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,GAAG,GAAGC,gBAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IAC/D;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAMP,cAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAMA,cAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAMA,cAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;AAgBA;AACA,MAAM,UAAW,SAAQ,KAAK,CAAA;IAC5B,IAAI,GAAG,YAAY;AACpB;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAYQ;AACb,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;AAEX,YAAC,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC;;YAEtE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;AAErD;;;;;;"}
1	+ {"version":3,"file":"task.cjs","sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport path from 'path';\nimport fs from 'fs';\nimport * as fsp from 'fs/promises';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from '@milaboratories/ts-helpers';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { UnknownStorageError, WrongLocalFileUrl, type ClientDownload } from '../../clients/download';\nimport type { ArchiveFormat, FolderURL } from '@milaboratories/pl-model-common';\nimport { newFolderURL } from '../urls/url';\nimport decompress from 'decompress';\nimport { assertNever } from '@protobuf-ts/runtime';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} finished`);\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(`a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`);\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo, \n {}, \n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case 'tar':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case 'tgz':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content\n .pipeThrough(gunzip, { signal })\n .pipeTo(untar, { signal });\n return;\n\n case 'zip':\n this.state!.zipPath = this.path + '.zip';\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: file => !file.path.endsWith('/'),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n }\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = 'URLAborted';\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted\n \|\| isDownloadNetworkError400(e)\n \|\| e instanceof UnknownStorageError\n \|\| e instanceof WrongLocalFileUrl\n // file that we downloads from was moved or deleted.\n \|\| e?.code == 'ENOENT'\n // A resource was deleted.\n \|\| (e.name == 'RpcError' && (e.code == 'NOT_FOUND' \|\| e.code == 'ABORTED'))\n // wrong archive format\n \|\| (String(e).includes('incorrect header check'))\n );\n}\n"],"names":["CallersCounter","ChangeSource","resourceIdToString","stringifyWithResourceId","ensureDirExists","fileExists","createPathAtomically","fsp","Writable","tar","Transform","zlib","assertNever","notEmpty","newFolderURL","isDownloadNetworkError400","UnknownStorageError","WrongLocalFileUrl"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA;MACa,wBAAwB,CAAA;AAWhB,IAAA,MAAA;AACA,IAAA,MAAA;AACR,IAAA,OAAA;AACA,IAAA,IAAA;AACA,IAAA,KAAA;AACA,IAAA,MAAA;AACQ,IAAA,cAAA;AAhBV,IAAA,OAAO,GAAG,IAAIA,wBAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAIC,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;AACA,IAAA,GAAG;AACH,IAAA,KAAK;AAEb,IAAA,WAAA,CACmB,MAAgB,EAChB,MAAc,EACtB,OAAe,EACf,IAAY,EACZ,KAA+B,EAC/B,MAAqB,EACb,cAA8B,EAAA;QAN9B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,MAAM,GAAN,MAAM;QACE,IAAA,CAAA,cAAc,GAAd,cAAc;IAC9B;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACpE,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+BC,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;AAEpG,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,0BAAA,EAA6BC,gCAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;QACvF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA,uBAAA,EAA0BA,gCAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;AAE5G,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+BD,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAElG,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,qBAAqB,CAAC,MAAmB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;AAEf,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,MAAME,yBAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;AAE3C,QAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAMC,oBAAU,CAAC,IAAI,CAAC,IAAI,CAAC;AACpD,QAAA,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;AAC1B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACpD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAM,CAAC,UAAU,GAAG,IAAI;AAE7B,YAAA,MAAMC,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;AACzE,gBAAA,IAAI,CAAC,KAAM,CAAC,QAAQ,GAAG,KAAK;gBAC5B,IAAI,CAAC,KAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM;AAEvC,gBAAA,QAAQ,IAAI,CAAC,MAAM;AACjB,oBAAA,KAAK,KAAK;wBACR,MAAMC,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,wBAAA,MAAM,WAAW,GAAGC,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACtD,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC7C;AAEF,oBAAA,KAAK,KAAK;wBACR,MAAMF,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAGG,qBAAS,CAAC,KAAK,CAACC,eAAI,CAAC,YAAY,EAAE,CAAC;AACnD,wBAAA,MAAM,KAAK,GAAGH,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEhD,wBAAA,MAAM;AACH,6BAAA,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE;AAC9B,6BAAA,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC5B;AAEF,oBAAA,KAAK,KAAK;wBACR,IAAI,CAAC,KAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM;AAExC,wBAAA,MAAM,CAAC,GAAGD,oBAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC;wBACnE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC;AACnC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;;;;;wBAMjC,MAAM,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,EAAE,KAAK,EAAE;AAC3C,4BAAA,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;AACzC,yBAAA,CAAC;AACF,wBAAA,IAAI,CAAC,KAAM,CAAC,eAAe,GAAG,IAAI;AAElC,wBAAA,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC;AACzC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;wBAEjC;AAEF,oBAAA;AACE,wBAAAI,mBAAW,CAAC,IAAI,CAAC,MAAM,CAAC;;AAE9B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAM,CAAC,WAAW,GAAG,IAAI;AAC9B,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,GAAG,EAAEC,kBAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAEjD,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,GAAG,GAAGC,gBAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IAC/D;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAMP,cAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAMA,cAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAMA,cAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;AAgBA;AACA,MAAM,UAAW,SAAQ,KAAK,CAAA;IAC5B,IAAI,GAAG,YAAY;AACpB;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;WACVQ,yCAAyB,CAAC,CAAC;AAC3B,WAAA,CAAC,YAAYC;AACb,WAAA,CAAC,YAAYC;;WAEb,CAAC,EAAE,IAAI,IAAI;;AAEX,YAAC,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC;;YAEtE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;AAErD;;;;;;"}

package/dist/drivers/download_blob_url/task.js CHANGED Viewed

@@ -4,7 +4,7 @@ import * as tar from 'tar-fs';
 import path__default from 'node:path';
 import fs__default from 'node:fs';
 import * as fsp from 'node:fs/promises';
-import { NetworkError400 } from '../../helpers/download.js';
+import { isDownloadNetworkError400 } from '../../helpers/download_errors.js';
 import { ChangeSource } from '@milaboratories/computable';
 import { CallersCounter, ensureDirExists, fileExists, createPathAtomically, notEmpty } from '@milaboratories/ts-helpers';
 import { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download.js';
@@ -169,7 +169,7 @@ class URLAborted extends Error {
 }
 function nonRecoverableError(e) {
     return (e instanceof URLAborted
-        || e instanceof NetworkError400
+        || isDownloadNetworkError400(e)
         || e instanceof UnknownStorageError
         || e instanceof WrongLocalFileUrl
         // file that we downloads from was moved or deleted.

package/dist/drivers/download_blob_url/task.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.js","sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport path from 'path';\nimport fs from 'fs';\nimport * as fsp from 'fs/promises';\nimport { NetworkError400 } from '../../helpers/download';\nimport type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from '@milaboratories/ts-helpers';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { UnknownStorageError, WrongLocalFileUrl, type ClientDownload } from '../../clients/download';\nimport type { ArchiveFormat, FolderURL } from '@milaboratories/pl-model-common';\nimport { newFolderURL } from '../urls/url';\nimport decompress from 'decompress';\nimport { assertNever } from '@protobuf-ts/runtime';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} finished`);\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(`a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`);\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo, \n {}, \n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case 'tar':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case 'tgz':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content\n .pipeThrough(gunzip, { signal })\n .pipeTo(untar, { signal });\n return;\n\n case 'zip':\n this.state!.zipPath = this.path + '.zip';\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: file => !file.path.endsWith('/'),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n }\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = 'URLAborted';\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted\n \|\| e instanceof NetworkError400\n \|\| e instanceof UnknownStorageError\n \|\| e instanceof WrongLocalFileUrl\n // file that we downloads from was moved or deleted.\n \|\| e?.code == 'ENOENT'\n // A resource was deleted.\n \|\| (e.name == 'RpcError' && (e.code == 'NOT_FOUND' \|\| e.code == 'ABORTED'))\n // wrong archive format\n \|\| (String(e).includes('incorrect header check'))\n );\n}\n"],"names":["path","fs"],"mappings":";;;;;;;;;;;;;;;AAwBA;MACa,wBAAwB,CAAA;AAWhB,IAAA,MAAA;AACA,IAAA,MAAA;AACR,IAAA,OAAA;AACA,IAAA,IAAA;AACA,IAAA,KAAA;AACA,IAAA,MAAA;AACQ,IAAA,cAAA;AAhBV,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;AACA,IAAA,GAAG;AACH,IAAA,KAAK;AAEb,IAAA,WAAA,CACmB,MAAgB,EAChB,MAAc,EACtB,OAAe,EACf,IAAY,EACZ,KAA+B,EAC/B,MAAqB,EACb,cAA8B,EAAA;QAN9B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,MAAM,GAAN,MAAM;QACE,IAAA,CAAA,cAAc,GAAd,cAAc;IAC9B;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACpE,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;AAEpG,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,0BAAA,EAA6B,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;QACvF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA,uBAAA,EAA0B,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;AAE5G,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAElG,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,qBAAqB,CAAC,MAAmB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;AAEf,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAGA,aAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;AAE3C,QAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;AACpD,QAAA,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;AAC1B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACpD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAM,CAAC,UAAU,GAAG,IAAI;AAE7B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;AACzE,gBAAA,IAAI,CAAC,KAAM,CAAC,QAAQ,GAAG,KAAK;gBAC5B,IAAI,CAAC,KAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM;AAEvC,gBAAA,QAAQ,IAAI,CAAC,MAAM;AACjB,oBAAA,KAAK,KAAK;wBACR,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,wBAAA,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACtD,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC7C;AAEF,oBAAA,KAAK,KAAK;wBACR,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;AACnD,wBAAA,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEhD,wBAAA,MAAM;AACH,6BAAA,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE;AAC9B,6BAAA,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC5B;AAEF,oBAAA,KAAK,KAAK;wBACR,IAAI,CAAC,KAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM;AAExC,wBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACC,WAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC;wBACnE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC;AACnC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;;;;;wBAMjC,MAAM,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,EAAE,KAAK,EAAE;AAC3C,4BAAA,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;AACzC,yBAAA,CAAC;AACF,wBAAA,IAAI,CAAC,KAAM,CAAC,eAAe,GAAG,IAAI;AAElC,wBAAA,MAAMA,WAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC;AACzC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;wBAEjC;AAEF,oBAAA;AACE,wBAAA,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC;;AAE9B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAM,CAAC,WAAW,GAAG,IAAI;AAC9B,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAEjD,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IAC/D;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAGD,aAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;AAgBA;AACA,MAAM,UAAW,SAAQ,KAAK,CAAA;IAC5B,IAAI,GAAG,YAAY;AACpB;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;AACV,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;AAEX,YAAC,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC;;YAEtE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;AAErD;;;;"}
1	+ {"version":3,"file":"task.js","sources":["../../../src/drivers/download_blob_url/task.ts"],"sourcesContent":["import { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport path from 'path';\nimport fs from 'fs';\nimport * as fsp from 'fs/promises';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type { MiLogger, Signer } from '@milaboratories/ts-helpers';\nimport { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from '@milaboratories/ts-helpers';\nimport type { DownloadableBlobSnapshot } from './snapshot';\nimport { UnknownStorageError, WrongLocalFileUrl, type ClientDownload } from '../../clients/download';\nimport type { ArchiveFormat, FolderURL } from '@milaboratories/pl-model-common';\nimport { newFolderURL } from '../urls/url';\nimport decompress from 'decompress';\nimport { assertNever } from '@protobuf-ts/runtime';\nimport { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';\n\nexport type URLResult = {\n url?: FolderURL;\n error?: string;\n};\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadAndUnarchiveTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n private url: FolderURL \| undefined;\n private state: DownloadCtx \| undefined;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly signer: Signer,\n readonly saveDir: string,\n readonly path: string,\n readonly rInfo: DownloadableBlobSnapshot,\n readonly format: ArchiveFormat,\n private readonly clientDownload: ClientDownload,\n ) {}\n\n /* A debug info of the task. /\n public info() {\n return {\n rInfo: this.rInfo,\n format: this.format,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n taskHistory: this.state,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download() {\n try {\n const size = await this.downloadAndDecompress(this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} finished`);\n\n this.logger.info(`blob to URL task is done: ${stringifyWithResourceId(this.info())}`);\n } catch (e: any) {\n this.logger.warn(`a error was produced: ${e} for blob to URL task: ${stringifyWithResourceId(this.info())}`);\n\n if (nonRecoverableError(e)) {\n this.setError(e);\n this.change.markChanged(`download and decompress for ${resourceIdToString(this.rInfo.id)} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n /* Does the download part and keeps a state of the process. /\n private async downloadAndDecompress(signal: AbortSignal): Promise<number> {\n this.state = {};\n\n this.state.parentDir = path.dirname(this.path);\n await ensureDirExists(this.state.parentDir);\n\n this.state.fileExisted = await fileExists(this.path);\n if (this.state.fileExisted) {\n return await dirSize(this.path);\n }\n\n const size = await this.clientDownload.withBlobContent(\n this.rInfo, \n {}, \n { signal },\n async (content, size) => {\n this.state!.downloaded = true;\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n this.state!.tempPath = fPath;\n this.state!.archiveFormat = this.format;\n\n switch (this.format) {\n case 'tar':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const simpleUntar = Writable.toWeb(tar.extract(fPath));\n await content.pipeTo(simpleUntar, { signal });\n return;\n\n case 'tgz':\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const gunzip = Transform.toWeb(zlib.createGunzip());\n const untar = Writable.toWeb(tar.extract(fPath));\n\n await content\n .pipeThrough(gunzip, { signal })\n .pipeTo(untar, { signal });\n return;\n\n case 'zip':\n this.state!.zipPath = this.path + '.zip';\n\n const f = Writable.toWeb(fs.createWriteStream(this.state!.zipPath));\n await content.pipeTo(f, { signal });\n this.state!.zipPathCreated = true;\n\n // Without this filter it fails with\n // \"EISDIR: illegal operation on a directory\".\n // The workaround is from\n // https://github.com/kevva/decompress/issues/46#issuecomment-525048104\n await decompress(this.state!.zipPath, fPath, {\n filter: file => !file.path.endsWith('/'),\n });\n this.state!.zipDecompressed = true;\n\n await fs.promises.rm(this.state!.zipPath);\n this.state!.zipPathDeleted = true;\n\n return;\n\n default:\n assertNever(this.format);\n }\n });\n\n this.state!.pathCreated = true;\n return size;\n }\n );\n\n return size;\n }\n\n getURL(): URLResult \| undefined {\n if (this.done) return { url: notEmpty(this.url) };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n this.url = newFolderURL(this.signer, this.saveDir, this.path);\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. /\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n\n/* Just a type that adds lots of context when the error happens. /\ntype DownloadCtx = {\n parentDir?: string;\n fileExisted?: boolean;\n downloaded?: boolean;\n archiveFormat?: ArchiveFormat;\n tempPath?: string;\n zipPath?: string;\n zipPathCreated?: boolean;\n zipDecompressed?: boolean;\n zipPathDeleted?: boolean;\n pathCreated?: boolean;\n};\n\n/* Throws when a downloading aborts. */\nclass URLAborted extends Error {\n name = 'URLAborted';\n}\n\nexport function nonRecoverableError(e: any) {\n return (\n e instanceof URLAborted\n \|\| isDownloadNetworkError400(e)\n \|\| e instanceof UnknownStorageError\n \|\| e instanceof WrongLocalFileUrl\n // file that we downloads from was moved or deleted.\n \|\| e?.code == 'ENOENT'\n // A resource was deleted.\n \|\| (e.name == 'RpcError' && (e.code == 'NOT_FOUND' \|\| e.code == 'ABORTED'))\n // wrong archive format\n \|\| (String(e).includes('incorrect header check'))\n );\n}\n"],"names":["path","fs"],"mappings":";;;;;;;;;;;;;;;AAwBA;MACa,wBAAwB,CAAA;AAWhB,IAAA,MAAA;AACA,IAAA,MAAA;AACR,IAAA,OAAA;AACA,IAAA,IAAA;AACA,IAAA,KAAA;AACA,IAAA,MAAA;AACQ,IAAA,cAAA;AAhBV,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;AACA,IAAA,GAAG;AACH,IAAA,KAAK;AAEb,IAAA,WAAA,CACmB,MAAgB,EAChB,MAAc,EACtB,OAAe,EACf,IAAY,EACZ,KAA+B,EAC/B,MAAqB,EACb,cAA8B,EAAA;QAN9B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,MAAM,GAAN,MAAM;QACE,IAAA,CAAA,cAAc,GAAd,cAAc;IAC9B;;IAGI,IAAI,GAAA;QACT,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACpE,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;AAClB,YAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,SAAA,CAAW,CAAC;AAEpG,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,0BAAA,EAA6B,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;QACvF;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA,uBAAA,EAA0B,uBAAuB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA,CAAE,CAAC;AAE5G,YAAA,IAAI,mBAAmB,CAAC,CAAC,CAAC,EAAE;AAC1B,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChB,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,OAAA,CAAS,CAAC;;AAElG,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;;IAGQ,MAAM,qBAAqB,CAAC,MAAmB,EAAA;AACrD,QAAA,IAAI,CAAC,KAAK,GAAG,EAAE;AAEf,QAAA,IAAI,CAAC,KAAK,CAAC,SAAS,GAAGA,aAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;AAE3C,QAAA,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;AACpD,QAAA,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;AAC1B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACpD,IAAI,CAAC,KAAK,EACV,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;AACtB,YAAA,IAAI,CAAC,KAAM,CAAC,UAAU,GAAG,IAAI;AAE7B,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;AACzE,gBAAA,IAAI,CAAC,KAAM,CAAC,QAAQ,GAAG,KAAK;gBAC5B,IAAI,CAAC,KAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM;AAEvC,gBAAA,QAAQ,IAAI,CAAC,MAAM;AACjB,oBAAA,KAAK,KAAK;wBACR,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,wBAAA,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACtD,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC7C;AAEF,oBAAA,KAAK,KAAK;wBACR,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;AACnD,wBAAA,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEhD,wBAAA,MAAM;AACH,6BAAA,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE;AAC9B,6BAAA,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;wBAC5B;AAEF,oBAAA,KAAK,KAAK;wBACR,IAAI,CAAC,KAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM;AAExC,wBAAA,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAACC,WAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC;wBACnE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC;AACnC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;;;;;wBAMjC,MAAM,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,EAAE,KAAK,EAAE;AAC3C,4BAAA,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;AACzC,yBAAA,CAAC;AACF,wBAAA,IAAI,CAAC,KAAM,CAAC,eAAe,GAAG,IAAI;AAElC,wBAAA,MAAMA,WAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,KAAM,CAAC,OAAO,CAAC;AACzC,wBAAA,IAAI,CAAC,KAAM,CAAC,cAAc,GAAG,IAAI;wBAEjC;AAEF,oBAAA;AACE,wBAAA,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC;;AAE9B,YAAA,CAAC,CAAC;AAEF,YAAA,IAAI,CAAC,KAAM,CAAC,WAAW,GAAG,IAAI;AAC9B,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAEjD,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IAC/D;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAGD,aAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;AAgBA;AACA,MAAM,UAAW,SAAQ,KAAK,CAAA;IAC5B,IAAI,GAAG,YAAY;AACpB;AAEK,SAAU,mBAAmB,CAAC,CAAM,EAAA;IACxC,QACE,CAAC,YAAY;WACV,yBAAyB,CAAC,CAAC;AAC3B,WAAA,CAAC,YAAY;AACb,WAAA,CAAC,YAAY;;WAEb,CAAC,EAAE,IAAI,IAAI;;AAEX,YAAC,CAAC,CAAC,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,IAAI,IAAI,SAAS,CAAC;;YAEtE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;AAErD;;;;"}

package/dist/drivers/download_url/task.cjs CHANGED Viewed

@@ -7,7 +7,7 @@ var path = require('node:path');
 var node_stream = require('node:stream');
 var zlib = require('node:zlib');
 var tar = require('tar-fs');
-var download = require('../../helpers/download.cjs');
+var download_errors = require('../../helpers/download_errors.cjs');
 var url = require('../urls/url.cjs');
 function _interopNamespaceDefault(e) {
@@ -73,7 +73,7 @@ class DownloadByUrlTask {
             this.change.markChanged(`download of ${this.url} finished`);
         }
         catch (e) {
-            if (e instanceof URLAborted || e instanceof download.NetworkError400) {
+            if (e instanceof URLAborted || download_errors.isDownloadNetworkError400(e)) {
                 this.setError(e);
                 this.change.markChanged(`download of ${this.url} failed`);
                 // Just in case we were half-way extracting an archive.

package/dist/drivers/download_url/task.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.cjs","sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type {\n MiLogger,\n Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from '@milaboratories/ts-helpers';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport type { RemoteFileDownloader } from '../../helpers/download';\nimport { ~~NetworkError400~~ } from '../../helpers/~~download~~';\nimport type { UrlResult } from './driver';\nimport { newBlockUIURL } from '../urls/url';\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly url: URL,\n readonly signer: Signer,\n readonly saveDir: string,\n ) { }\n\n public info() {\n return {\n url: this.url.toString(),\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.url} finished`);\n } catch (e: unknown) {\n if (e instanceof URLAborted \|\| e ~~instanceof NetworkError400~~) {\n this.setError(e);\n this.change.markChanged(`download of ${this.url} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async downloadAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n const size = await clientDownload.withContent(\n this.url.toString(), \n {}, \n { signal },\n async (content, size) => {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n\n return size;\n }\n );\n\n return size;\n }\n\n getUrl(): UrlResult \| undefined {\n if (this.done) return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Throws when a downloading aborts. /\nexport class URLAborted extends Error {\n name = 'URLAborted';\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"names":["CallersCounter","ChangeSource","~~NetworkError400~~","ensureDirExists","path","fileExists","Transform","zlib","createPathAtomically","fsp","Writable","tar","newBlockUIURL","notEmpty"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,iBAAiB,CAAA;AAST,IAAA,MAAA;AACR,IAAA,IAAA;AACA,IAAA,GAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;AAZF,IAAA,OAAO,GAAG,IAAIA,wBAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAIC,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;IAER,WAAA,CACmB,MAAgB,EACxB,IAAY,EACZ,GAAQ,EACR,MAAc,EACd,OAAe,EAAA;QAJP,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,GAAG,GAAH,GAAG;QACH,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACd;IAEG,IAAI,GAAA;QACT,OAAO;AACL,YAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACxB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,CAAC,cAAoC,EAAE,UAAmB,EAAA;AACtE,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AAC3F,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,SAAA,CAAW,CAAC;QAC7D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,UAAU,~~IAAI~~,CAAC,~~YAAYC~~,~~wBAAe~~,EAAE;AAC3D,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,OAAA,CAAS,CAAC;;AAEzD,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,CAC5B,cAAoC,EACpC,UAAmB,EACnB,MAAmB,EAAA;QAEnB,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,MAAMC,oBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AAC/B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,CAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EACnB,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;YACtB,IAAI,gBAAgB,GAAG,OAAO;YAC9B,IAAI,UAAU,EAAE;gBACd,MAAM,MAAM,GAAGC,qBAAS,CAAC,KAAK,CAACC,eAAI,CAAC,YAAY,EAAE,CAAC;gBACnD,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAC5D;AAEA,YAAA,MAAMC,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;gBACzE,MAAMC,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,gBAAA,MAAM,KAAK,GAAGC,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;AAClD,YAAA,CAAC,CAAC;AAEF,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;AACpB,gBAAA,GAAG,EAAEC,iBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAEC,kBAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE;QAEF,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;IAClB;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAMJ,cAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAGL,eAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAMK,cAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAMA,cAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;;;;;;"}
1	+ {"version":3,"file":"task.cjs","sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type {\n MiLogger,\n Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from '@milaboratories/ts-helpers';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport type { RemoteFileDownloader } from '../../helpers/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport type { UrlResult } from './driver';\nimport { newBlockUIURL } from '../urls/url';\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly url: URL,\n readonly signer: Signer,\n readonly saveDir: string,\n ) { }\n\n public info() {\n return {\n url: this.url.toString(),\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.url} finished`);\n } catch (e: unknown) {\n if (e instanceof URLAborted \|\| isDownloadNetworkError400(e)) {\n this.setError(e);\n this.change.markChanged(`download of ${this.url} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async downloadAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n const size = await clientDownload.withContent(\n this.url.toString(), \n {}, \n { signal },\n async (content, size) => {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n\n return size;\n }\n );\n\n return size;\n }\n\n getUrl(): UrlResult \| undefined {\n if (this.done) return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Throws when a downloading aborts. /\nexport class URLAborted extends Error {\n name = 'URLAborted';\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"names":["CallersCounter","ChangeSource","isDownloadNetworkError400","ensureDirExists","path","fileExists","Transform","zlib","createPathAtomically","fsp","Writable","tar","newBlockUIURL","notEmpty"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuBA;MACa,iBAAiB,CAAA;AAST,IAAA,MAAA;AACR,IAAA,IAAA;AACA,IAAA,GAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;AAZF,IAAA,OAAO,GAAG,IAAIA,wBAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAIC,uBAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;IAER,WAAA,CACmB,MAAgB,EACxB,IAAY,EACZ,GAAQ,EACR,MAAc,EACd,OAAe,EAAA;QAJP,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,GAAG,GAAH,GAAG;QACH,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACd;IAEG,IAAI,GAAA;QACT,OAAO;AACL,YAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACxB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,CAAC,cAAoC,EAAE,UAAmB,EAAA;AACtE,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AAC3F,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,SAAA,CAAW,CAAC;QAC7D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,UAAU,IAAIC,yCAAyB,CAAC,CAAC,CAAC,EAAE;AAC3D,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,OAAA,CAAS,CAAC;;AAEzD,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,CAC5B,cAAoC,EACpC,UAAmB,EACnB,MAAmB,EAAA;QAEnB,MAAMC,yBAAe,CAACC,eAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,MAAMC,oBAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AAC/B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,CAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EACnB,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;YACtB,IAAI,gBAAgB,GAAG,OAAO;YAC9B,IAAI,UAAU,EAAE;gBACd,MAAM,MAAM,GAAGC,qBAAS,CAAC,KAAK,CAACC,eAAI,CAAC,YAAY,EAAE,CAAC;gBACnD,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAC5D;AAEA,YAAA,MAAMC,8BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;gBACzE,MAAMC,cAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,gBAAA,MAAM,KAAK,GAAGC,oBAAQ,CAAC,KAAK,CAACC,cAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;AAClD,YAAA,CAAC,CAAC;AAEF,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;AACpB,gBAAA,GAAG,EAAEC,iBAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAEC,kBAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE;QAEF,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;IAClB;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAMJ,cAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAGL,eAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAMK,cAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAMA,cAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;;;;;;"}

package/dist/drivers/download_url/task.js CHANGED Viewed

@@ -5,7 +5,7 @@ import * as path from 'node:path';
 import { Transform, Writable } from 'node:stream';
 import * as zlib from 'node:zlib';
 import * as tar from 'tar-fs';
-import { NetworkError400 } from '../../helpers/download.js';
+import { isDownloadNetworkError400 } from '../../helpers/download_errors.js';
 import { newBlockUIURL } from '../urls/url.js';
 /** Downloads and extracts an archive to a directory. */
@@ -49,7 +49,7 @@ class DownloadByUrlTask {
             this.change.markChanged(`download of ${this.url} finished`);
         }
         catch (e) {
-            if (e instanceof URLAborted || e instanceof NetworkError400) {
+            if (e instanceof URLAborted || isDownloadNetworkError400(e)) {
                 this.setError(e);
                 this.change.markChanged(`download of ${this.url} failed`);
                 // Just in case we were half-way extracting an archive.

package/dist/drivers/download_url/task.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"task.js","sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type {\n MiLogger,\n Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from '@milaboratories/ts-helpers';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport type { RemoteFileDownloader } from '../../helpers/download';\nimport { ~~NetworkError400~~ } from '../../helpers/~~download~~';\nimport type { UrlResult } from './driver';\nimport { newBlockUIURL } from '../urls/url';\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly url: URL,\n readonly signer: Signer,\n readonly saveDir: string,\n ) { }\n\n public info() {\n return {\n url: this.url.toString(),\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.url} finished`);\n } catch (e: unknown) {\n if (e instanceof URLAborted \|\| e ~~instanceof NetworkError400~~) {\n this.setError(e);\n this.change.markChanged(`download of ${this.url} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async downloadAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n const size = await clientDownload.withContent(\n this.url.toString(), \n {}, \n { signal },\n async (content, size) => {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n\n return size;\n }\n );\n\n return size;\n }\n\n getUrl(): UrlResult \| undefined {\n if (this.done) return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Throws when a downloading aborts. /\nexport class URLAborted extends Error {\n name = 'URLAborted';\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"names":[],"mappings":";;;;;;;;;;AAuBA;MACa,iBAAiB,CAAA;AAST,IAAA,MAAA;AACR,IAAA,IAAA;AACA,IAAA,GAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;AAZF,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;IAER,WAAA,CACmB,MAAgB,EACxB,IAAY,EACZ,GAAQ,EACR,MAAc,EACd,OAAe,EAAA;QAJP,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,GAAG,GAAH,GAAG;QACH,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACd;IAEG,IAAI,GAAA;QACT,OAAO;AACL,YAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACxB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,CAAC,cAAoC,EAAE,UAAmB,EAAA;AACtE,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AAC3F,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,SAAA,CAAW,CAAC;QAC7D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,UAAU,IAAI,CAAC,~~YAAY~~,~~eAAe~~,EAAE;AAC3D,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,OAAA,CAAS,CAAC;;AAEzD,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,CAC5B,cAAoC,EACpC,UAAmB,EACnB,MAAmB,EAAA;QAEnB,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AAC/B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,CAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EACnB,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;YACtB,IAAI,gBAAgB,GAAG,OAAO;YAC9B,IAAI,UAAU,EAAE;gBACd,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnD,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAC5D;AAEA,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;gBACzE,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,gBAAA,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;AAClD,YAAA,CAAC,CAAC;AAEF,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;AACpB,gBAAA,GAAG,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE;QAEF,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;IAClB;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;;;;"}
1	+ {"version":3,"file":"task.js","sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from '@milaboratories/computable';\nimport { ChangeSource } from '@milaboratories/computable';\nimport type {\n MiLogger,\n Signer,\n} from '@milaboratories/ts-helpers';\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from '@milaboratories/ts-helpers';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Transform, Writable } from 'node:stream';\nimport * as zlib from 'node:zlib';\nimport * as tar from 'tar-fs';\nimport type { RemoteFileDownloader } from '../../helpers/download';\nimport { isDownloadNetworkError400 } from '../../helpers/download_errors';\nimport type { UrlResult } from './driver';\nimport { newBlockUIURL } from '../urls/url';\n\n/** Downloads and extracts an archive to a directory. /\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string \| undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly url: URL,\n readonly signer: Signer,\n readonly saveDir: string,\n ) { }\n\n public info() {\n return {\n url: this.url.toString(),\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.url} finished`);\n } catch (e: unknown) {\n if (e instanceof URLAborted \|\| isDownloadNetworkError400(e)) {\n this.setError(e);\n this.change.markChanged(`download of ${this.url} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async downloadAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n const size = await clientDownload.withContent(\n this.url.toString(), \n {}, \n { signal },\n async (content, size) => {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n\n return size;\n }\n );\n\n return size;\n }\n\n getUrl(): UrlResult \| undefined {\n if (this.done) return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/* Throws when a downloading aborts. /\nexport class URLAborted extends Error {\n name = 'URLAborted';\n}\n\n/* Gets a directory size by calculating sizes recursively. /\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/* Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"names":[],"mappings":";;;;;;;;;;AAuBA;MACa,iBAAiB,CAAA;AAST,IAAA,MAAA;AACR,IAAA,IAAA;AACA,IAAA,GAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;AAZF,IAAA,OAAO,GAAG,IAAI,cAAc,EAAE;AAC9B,IAAA,MAAM,GAAG,IAAI,YAAY,EAAE;AACnB,IAAA,SAAS,GAAG,IAAI,eAAe,EAAE;AAClD,IAAA,KAAK;IACL,IAAI,GAAG,KAAK;IACZ,IAAI,GAAG,CAAC;IAER,WAAA,CACmB,MAAgB,EACxB,IAAY,EACZ,GAAQ,EACR,MAAc,EACd,OAAe,EAAA;QAJP,IAAA,CAAA,MAAM,GAAN,MAAM;QACd,IAAA,CAAA,IAAI,GAAJ,IAAI;QACJ,IAAA,CAAA,GAAG,GAAH,GAAG;QACH,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,OAAO,GAAP,OAAO;IACd;IAEG,IAAI,GAAA;QACT,OAAO;AACL,YAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE;YACxB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,IAAI;AAAE,YAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC9C;AAEA,IAAA,MAAM,QAAQ,CAAC,cAAoC,EAAE,UAAmB,EAAA;AACtE,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AAC3F,YAAA,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,SAAA,CAAW,CAAC;QAC7D;QAAE,OAAO,CAAU,EAAE;YACnB,IAAI,CAAC,YAAY,UAAU,IAAI,yBAAyB,CAAC,CAAC,CAAC,EAAE;AAC3D,gBAAA,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA,YAAA,EAAe,IAAI,CAAC,GAAG,CAAA,OAAA,CAAS,CAAC;;AAEzD,gBAAA,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB;YACF;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,gBAAgB,CAC5B,cAAoC,EACpC,UAAmB,EACnB,MAAmB,EAAA;QAEnB,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;AAC/B,YAAA,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC;QAEA,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,WAAW,CAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EACnB,EAAE,EACF,EAAE,MAAM,EAAE,EACV,OAAO,OAAO,EAAE,IAAI,KAAI;YACtB,IAAI,gBAAgB,GAAG,OAAO;YAC9B,IAAI,UAAU,EAAE;gBACd,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnD,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC;YAC5D;AAEA,YAAA,MAAM,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,KAAa,KAAI;gBACzE,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACvB,gBAAA,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC;AAClD,YAAA,CAAC,CAAC;AAEF,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CACF;AAED,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,GAAA;QACJ,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;AACpB,gBAAA,GAAG,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;aACjE;QAEF,IAAI,IAAI,CAAC,KAAK;AAAE,YAAA,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;AAE5C,QAAA,OAAO,SAAS;IAClB;AAEQ,IAAA,OAAO,CAAC,IAAY,EAAA;AAC1B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;AAChB,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI;IAClB;AAEQ,IAAA,QAAQ,CAAC,CAAM,EAAA;AACrB,QAAA,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACxB;AAEA,IAAA,KAAK,CAAC,MAAc,EAAA;QAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C;AACD;AAED;AACM,MAAO,UAAW,SAAQ,KAAK,CAAA;IACnC,IAAI,GAAG,YAAY;AACpB;AAED;AACA,eAAe,OAAO,CAAC,GAAW,EAAA;AAChC,IAAA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC7D,IAAA,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,OAAO,IAAS,KAAI;AAC5B,QAAA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAEvC,IAAI,IAAI,CAAC,WAAW,EAAE;AAAE,YAAA,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,IAAI,CAAC,IAAI;IAClB,CAAC,CAAC,CACH;AAED,IAAA,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,IAAS,KAAK,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;AAC7D;AAEA;AACO,eAAe,OAAO,CAAC,IAAY,EAAA;AACxC,IAAA,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtD;;;;"}

package/dist/drivers/logs_stream.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"logs_stream.cjs","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n ComputableCtx,\n Watcher,\n} from '@milaboratories/computable';\nimport {\n ChangeSource,\n Computable,\n PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n ResourceId,\n ResourceType } from '@milaboratories/pl-client';\nimport {\n resourceIdToString,\n stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states /\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /* Holds a map of StreamManager Resource Id to all logs of this stream. /\n private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to the last log line of this stream. /\n private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to log id smart object. /\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo \| PlTreeEntry, lines: number): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx\n ): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n 'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getLastLogs', rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string\n ): Computable<string \| undefined>;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx\n ): string \| undefined;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n 'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getProgressLog', rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> \| sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType('getLogHandle', rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: ResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: ResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. /\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/* A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string \| undefined;\n private error: any \| undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n e as RpcError;\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n // No resource\n this.logs = '';\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith('StreamWorkdir')) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, `\n + `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"names":["PollingComputableHooks","Computable","treeEntryToResourceInfo","randomUUID","resourceIdToString","newLogHandle","getResourceInfoFromLogHandle","isLiveLogHandle","asyncPool","scheduler","ChangeSource","CallersCounter","stringifyWithResourceId","WrongResourceTypeError"],"mappings":";;;;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAIA,iCAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAYE,2BAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAOC,wBAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvBC,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtBA,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAACC,2BAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAMC,mBAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAMC,kBAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAIC,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAYP,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwBQ,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAA,CAAE,CACtF;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAIC,8BAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}
1	+ {"version":3,"file":"logs_stream.cjs","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n ComputableCtx,\n Watcher,\n} from '@milaboratories/computable';\nimport {\n ChangeSource,\n Computable,\n PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n ResourceId,\n ResourceType } from '@milaboratories/pl-client';\nimport {\n resourceIdToString,\n stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states /\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /* Holds a map of StreamManager Resource Id to all logs of this stream. /\n private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to the last log line of this stream. /\n private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to log id smart object. /\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo \| PlTreeEntry, lines: number): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx\n ): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n 'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getLastLogs', rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string\n ): Computable<string \| undefined>;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx\n ): string \| undefined;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n 'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getProgressLog', rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> \| sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType('getLogHandle', rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: ResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: ResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. /\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/* A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string \| undefined;\n private error: any \| undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n e as RpcError;\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n // No resource\n this.logs = '';\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith('StreamWorkdir')) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, `\n + `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"names":["PollingComputableHooks","Computable","treeEntryToResourceInfo","randomUUID","resourceIdToString","newLogHandle","getResourceInfoFromLogHandle","isLiveLogHandle","asyncPool","scheduler","ChangeSource","CallersCounter","stringifyWithResourceId","WrongResourceTypeError"],"mappings":";;;;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAIA,iCAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAOC,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAGC,sBAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAOF,qBAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAGC,8BAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAYE,2BAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAOC,wBAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvBC,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtBA,wCAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAACC,2BAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAMC,mBAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAMC,kBAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAIC,uBAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAIC,wBAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAYP,2BAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwBQ,gCAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAA,CAAE,CACtF;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAIC,8BAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}

package/dist/drivers/logs_stream.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"logs_stream.js","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n ComputableCtx,\n Watcher,\n} from '@milaboratories/computable';\nimport {\n ChangeSource,\n Computable,\n PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n ResourceId,\n ResourceType } from '@milaboratories/pl-client';\nimport {\n resourceIdToString,\n stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states /\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /* Holds a map of StreamManager Resource Id to all logs of this stream. /\n private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to the last log line of this stream. /\n private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to log id smart object. /\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo \| PlTreeEntry, lines: number): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx\n ): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n 'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getLastLogs', rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string\n ): Computable<string \| undefined>;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx\n ): string \| undefined;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n 'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getProgressLog', rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> \| sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType('getLogHandle', rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: ResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: ResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. /\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/* A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string \| undefined;\n private error: any \| undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n e as RpcError;\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n // No resource\n this.logs = '';\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith('StreamWorkdir')) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, `\n + `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"names":[],"mappings":";;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAI,sBAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAO,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAI,YAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAI,cAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwB,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAA,CAAE,CACtF;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAI,sBAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}
1	+ {"version":3,"file":"logs_stream.js","sources":["../../src/drivers/logs_stream.ts"],"sourcesContent":["import type {\n ComputableCtx,\n Watcher,\n} from '@milaboratories/computable';\nimport {\n ChangeSource,\n Computable,\n PollingComputableHooks,\n} from '@milaboratories/computable';\nimport type {\n ResourceId,\n ResourceType } from '@milaboratories/pl-client';\nimport {\n resourceIdToString,\n stringifyWithResourceId,\n} from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';\nimport type { ClientLogs } from '../clients/logs';\nimport { randomUUID } from 'node:crypto';\nimport type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';\nimport { treeEntryToResourceInfo } from '@milaboratories/pl-tree';\nimport { scheduler } from 'node:timers/promises';\nimport type { StreamingAPI_Response } from '../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type * as sdk from '@milaboratories/pl-model-common';\nimport type { PollingOps } from './helpers/polling_ops';\nimport type { RpcError } from '@protobuf-ts/runtime-rpc';\nimport { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';\nimport { WrongResourceTypeError } from './helpers/helpers';\n\nexport type LogsStreamDriverOps = PollingOps & {\n /** Max number of concurrent requests to log streaming backend while calculating computable states /\n nConcurrentGetLogs: number;\n};\n\nexport class LogsStreamDriver implements sdk.LogsDriver {\n /* Holds a map of StreamManager Resource Id to all logs of this stream. /\n private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to the last log line of this stream. /\n private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();\n\n /* Holds a map of StreamManager Resource Id to log id smart object. /\n private readonly hooks: PollingComputableHooks;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly opts: LogsStreamDriverOps = {\n nConcurrentGetLogs: 10,\n pollingInterval: 1000,\n stopPollingDelay: 1000,\n },\n ) {\n this.hooks = new PollingComputableHooks(\n () => this.startUpdating(),\n () => this.stopUpdating(),\n { stopDebounce: opts.stopPollingDelay },\n (resolve, reject) => this.scheduleOnNextState(resolve, reject),\n );\n }\n\n getLastLogs(res: ResourceInfo \| PlTreeEntry, lines: number): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx\n ): Computable<string \| undefined>;\n getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);\n ctx.markUnstable(\n 'The logs are from stream, so we consider them unstable. Final values will be got from blobs.',\n );\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getLastLogs', rInfo.type);\n\n let logGetter = this.idToLastLines.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, lines);\n this.idToLastLines.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error != undefined) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string\n ): Computable<string \| undefined>;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx\n ): string \| undefined;\n getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.attacheHooks(this.hooks);\n ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));\n\n const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);\n ctx.markUnstable(\n 'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.',\n );\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceInfo,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateResourceType('getProgressLog', rInfo.type);\n\n let logGetter = this.idToProgressLog.get(rInfo.id);\n\n if (logGetter == undefined) {\n const newLogGetter = new LogGetter(this.logger, this.clientLogs, rInfo, 1, patternToSearch);\n this.idToProgressLog.set(rInfo.id, newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n logGetter.attach(w, callerId);\n const result = logGetter.getLog();\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<sdk.AnyLogHandle>;\n getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;\n getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<sdk.AnyLogHandle> \| sdk.AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n const result = this.getLogHandleNoCtx(r);\n\n // All logs from streams should be considered unstable,\n // final value will be got from blobs.\n ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);\n\n return result;\n }\n\n private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {\n validateResourceType('getLogHandle', rInfo.type);\n\n return newLogHandle(true, rInfo);\n }\n\n async lastLines(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n async readText(\n handle: sdk.AnyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string \| undefined,\n ) {\n return await this.tryWithNotFound(handle, () =>\n this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n ),\n );\n }\n\n private async tryWithNotFound(\n handle: sdk.AnyLogHandle,\n method: () => Promise<StreamingAPI_Response>,\n ): Promise<sdk.StreamingApiResponse> {\n if (!isLiveLogHandle(handle))\n throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);\n\n try {\n const resp = await method();\n return {\n live: true,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n } catch (e: any) {\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n return { shouldUpdateHandle: true };\n }\n\n throw e;\n }\n }\n\n private async releaseLastLogs(rId: ResourceId, callerId: string) {\n const deleted = this.idToLastLines.get(rId)?.release(callerId);\n if (deleted) this.idToLastLines.delete(rId);\n }\n\n private async releaseProgressLog(rId: ResourceId, callerId: string) {\n const deleted = this.idToProgressLog.get(rId)?.release(callerId);\n if (deleted) this.idToProgressLog.delete(rId);\n }\n\n async releaseAll() {}\n\n private scheduledOnNextState: ScheduledRefresh[] = [];\n\n private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {\n this.scheduledOnNextState.push({ resolve, reject });\n }\n\n /* Called from observer /\n private startUpdating(): void {\n this.keepRunning = true;\n if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();\n }\n\n /* Called from observer /\n private stopUpdating(): void {\n this.keepRunning = false;\n }\n\n /* If true, main loop will continue polling pl state. /\n private keepRunning = false;\n /* Actual state of main loop. /\n private currentLoop: Promise<void> \| undefined = undefined;\n\n private async mainLoop() {\n while (this.keepRunning) {\n const toNotify = this.scheduledOnNextState;\n this.scheduledOnNextState = [];\n\n try {\n const logs = this.getAllLogs();\n await asyncPool(\n this.opts.nConcurrentGetLogs,\n logs.map((getter) => async () => await getter.update()),\n );\n\n toNotify.forEach((n) => n.resolve());\n } catch (e: any) {\n console.error(e);\n toNotify.forEach((n) => n.reject(e));\n }\n\n if (!this.keepRunning) break;\n await scheduler.wait(this.opts.pollingInterval);\n }\n\n this.currentLoop = undefined;\n }\n\n private getAllLogs(): Array<LogGetter> {\n return Array.from(this.idToLastLines.entries())\n .concat(Array.from(this.idToProgressLog.entries()))\n .map(([_, getter]) => getter);\n }\n}\n\n/* A job that gets last lines from a StreamWorkdir resource. */\nclass LogGetter {\n private logs: string \| undefined;\n private error: any \| undefined = undefined;\n\n private readonly change: ChangeSource = new ChangeSource();\n private readonly counter: CallersCounter = new CallersCounter();\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientLogs: ClientLogs,\n private readonly rInfo: ResourceInfo,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {}\n\n getLog(): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n return {\n log: this.logs,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.change.attachWatcher(w);\n this.counter.inc(callerId);\n }\n\n release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n\n async update() {\n try {\n const resp = await this.clientLogs.lastLines(\n this.rInfo,\n this.lines,\n 0n,\n this.patternToSearch,\n );\n\n const newLogs = new TextDecoder().decode(resp.data);\n if (this.logs != newLogs) this.change.markChanged(`logs for ${resourceIdToString(this.rInfo.id)} updated`);\n this.logs = newLogs;\n this.error = undefined;\n\n return;\n } catch (e: any) {\n e as RpcError;\n if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {\n // No resource\n this.logs = '';\n this.error = e;\n this.change.markChanged();\n return;\n }\n\n this.logger.error(\n `Stream log lines for ${stringifyWithResourceId(this.rInfo.id)} failed, reason: ${e}`,\n );\n throw e;\n }\n }\n}\n\ntype ScheduledRefresh = {\n resolve: () => void;\n reject: (err: any) => void;\n};\n\nfunction validateResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith('StreamWorkdir')) {\n throw new WrongResourceTypeError(\n `${methodName}: wrong resource type: ${rType.name}, `\n + `expected: a resource of type 'StreamWorkdir'.`,\n );\n }\n}\n"],"names":[],"mappings":";;;;;;;;;MAmCa,gBAAgB,CAAA;AAWR,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,IAAA;;AAXF,IAAA,aAAa,GAA+B,IAAI,GAAG,EAAE;;AAGrD,IAAA,eAAe,GAA+B,IAAI,GAAG,EAAE;;AAGvD,IAAA,KAAK;AAEtB,IAAA,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,IAAA,GAA4B;AAC3C,QAAA,kBAAkB,EAAE,EAAE;AACtB,QAAA,eAAe,EAAE,IAAI;AACrB,QAAA,gBAAgB,EAAE,IAAI;AACvB,KAAA,EAAA;QANgB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,IAAI,GAAJ,IAAI;QAMrB,IAAI,CAAC,KAAK,GAAG,IAAI,sBAAsB,CACrC,MAAM,IAAI,CAAC,aAAa,EAAE,EAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,EACzB,EAAE,YAAY,EAAE,IAAI,CAAC,gBAAgB,EAAE,EACvC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAC/D;IACH;AAQA,IAAA,WAAW,CACT,GAA+B,EAC/B,KAAa,EACb,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAAE,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAExF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE5D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC;AACrE,QAAA,GAAG,CAAC,YAAY,CACd,8FAA8F,CAC/F;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,gBAAgB,CACtB,CAAU,EACV,KAAmB,EACnB,KAAa,EACb,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC;AAE/C,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAEhD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;YAC9E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAE9C,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;AACjC,QAAA,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK;QAEjD,OAAO,MAAM,CAAC,GAAG;IACnB;AAaA,IAAA,cAAc,CACZ,GAA+B,EAC/B,eAAuB,EACvB,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;YAClB,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAEjF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;AAC3C,QAAA,MAAM,QAAQ,GAAG,UAAU,EAAE;AAC7B,QAAA,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAC5B,QAAA,GAAG,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE/D,QAAA,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC;AAClF,QAAA,GAAG,CAAC,YAAY,CACd,sGAAsG,CACvG;AAED,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,mBAAmB,CACzB,CAAU,EACV,KAAmB,EACnB,eAAuB,EACvB,QAAgB,EAAA;AAEhB,QAAA,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC;AAElD,QAAA,IAAI,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAElD,QAAA,IAAI,SAAS,IAAI,SAAS,EAAE;AAC1B,YAAA,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;YAC3F,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,CAAC;YAEhD,SAAS,GAAG,YAAY;QAC1B;AAEA,QAAA,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC;AAC7B,QAAA,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE;QACjC,IAAI,MAAM,CAAC,KAAK;YAAE,MAAM,MAAM,CAAC,KAAK;QAEpC,OAAO,MAAM,CAAC,GAAG;IACnB;IAIA,YAAY,CACV,GAA+B,EAC/B,GAAmB,EAAA;QAEnB,IAAI,GAAG,IAAI,SAAS;AAAE,YAAA,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElF,MAAM,CAAC,GAAG,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;AAIxC,QAAA,GAAG,CAAC,YAAY,CAAC,CAAA,SAAA,EAAY,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA,CAAE,CAAC;AAExD,QAAA,OAAO,MAAM;IACf;AAEQ,IAAA,iBAAiB,CAAC,KAAmB,EAAA;AAC3C,QAAA,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC;AAEhD,QAAA,OAAO,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC;IAClC;IAEA,MAAM,SAAS,CACb,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CACvB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;IAEA,MAAM,QAAQ,CACZ,MAAwB,EACxB,SAAiB,EACjB,WAAoB,EACpB,SAA8B,EAAA;AAE9B,QAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CACtB,4BAA4B,CAAC,MAAM,CAAC,EACpC,SAAS,EACT,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EACxB,SAAS,CACV,CACF;IACH;AAEQ,IAAA,MAAM,eAAe,CAC3B,MAAwB,EACxB,MAA4C,EAAA;AAE5C,QAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;AAC1B,YAAA,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAA,CAAE,CAAC;AAEzF,QAAA,IAAI;AACF,YAAA,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE;YAC3B,OAAO;AACL,gBAAA,IAAI,EAAE,IAAI;AACV,gBAAA,kBAAkB,EAAE,KAAK;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAI;AACf,gBAAA,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB,gBAAA,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;aAClC;QACH;QAAE,OAAO,CAAM,EAAE;AACf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;AACjD,gBAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE;YACrC;AAEA,YAAA,MAAM,CAAC;QACT;IACF;AAEQ,IAAA,MAAM,eAAe,CAAC,GAAe,EAAE,QAAgB,EAAA;AAC7D,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC9D,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;IAC7C;AAEQ,IAAA,MAAM,kBAAkB,CAAC,GAAe,EAAE,QAAgB,EAAA;AAChE,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC;AAChE,QAAA,IAAI,OAAO;AAAE,YAAA,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C;IAEA,MAAM,UAAU,GAAA,EAAI;IAEZ,oBAAoB,GAAuB,EAAE;IAE7C,mBAAmB,CAAC,OAAmB,EAAE,MAA0B,EAAA;QACzE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD;;IAGQ,aAAa,GAAA;AACnB,QAAA,IAAI,CAAC,WAAW,GAAG,IAAI;AACvB,QAAA,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;AAAE,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE;IACxE;;IAGQ,YAAY,GAAA;AAClB,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;IAC1B;;IAGQ,WAAW,GAAG,KAAK;;IAEnB,WAAW,GAA8B,SAAS;AAElD,IAAA,MAAM,QAAQ,GAAA;AACpB,QAAA,OAAO,IAAI,CAAC,WAAW,EAAE;AACvB,YAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB;AAC1C,YAAA,IAAI,CAAC,oBAAoB,GAAG,EAAE;AAE9B,YAAA,IAAI;AACF,gBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE;AAC9B,gBAAA,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,YAAY,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CACxD;AAED,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;YACtC;YAAE,OAAO,CAAM,EAAE;AACf,gBAAA,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AAChB,gBAAA,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC;YAEA,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE;YACvB,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;QACjD;AAEA,QAAA,IAAI,CAAC,WAAW,GAAG,SAAS;IAC9B;IAEQ,UAAU,GAAA;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;AAC3C,aAAA,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;AACjD,aAAA,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;IACjC;AACD;AAED;AACA,MAAM,SAAS,CAAA;AAQM,IAAA,MAAA;AACA,IAAA,UAAA;AACA,IAAA,KAAA;AACA,IAAA,KAAA;AACA,IAAA,eAAA;AAXX,IAAA,IAAI;IACJ,KAAK,GAAoB,SAAS;AAEzB,IAAA,MAAM,GAAiB,IAAI,YAAY,EAAE;AACzC,IAAA,OAAO,GAAmB,IAAI,cAAc,EAAE;IAE/D,WAAA,CACmB,MAAgB,EAChB,UAAsB,EACtB,KAAmB,EACnB,KAAa,EACb,eAAwB,EAAA;QAJxB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,KAAK,GAAL,KAAK;QACL,IAAA,CAAA,eAAe,GAAf,eAAe;IAC/B;IAEH,MAAM,GAAA;QAIJ,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,IAAI;YACd,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB;IACH;IAEA,MAAM,CAAC,CAAU,EAAE,QAAgB,EAAA;AACjC,QAAA,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5B,QAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC5B;AAEA,IAAA,OAAO,CAAC,QAAgB,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnC;AAEA,IAAA,MAAM,MAAM,GAAA;AACV,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC1C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,EAAE,EACF,IAAI,CAAC,eAAe,CACrB;AAED,YAAA,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;AACnD,YAAA,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;AAAE,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA,QAAA,CAAU,CAAC;AAC1G,YAAA,IAAI,CAAC,IAAI,GAAG,OAAO;AACnB,YAAA,IAAI,CAAC,KAAK,GAAG,SAAS;YAEtB;QACF;QAAE,OAAO,CAAM,EAAE;AAEf,YAAA,IAAI,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE;;AAEjD,gBAAA,IAAI,CAAC,IAAI,GAAG,EAAE;AACd,gBAAA,IAAI,CAAC,KAAK,GAAG,CAAC;AACd,gBAAA,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACzB;YACF;AAEA,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,qBAAA,EAAwB,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAA,CAAE,CACtF;AACD,YAAA,MAAM,CAAC;QACT;IACF;AACD;AAOD,SAAS,oBAAoB,CAAC,UAAkB,EAAE,KAAmB,EAAA;IACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QAC3C,MAAM,IAAI,sBAAsB,CAC9B,CAAA,EAAG,UAAU,CAAA,uBAAA,EAA0B,KAAK,CAAC,IAAI,CAAA,EAAA;AAC/C,cAAA,CAAA,6CAAA,CAA+C,CAClD;IACH;AACF;;;;"}

package/dist/helpers/download.cjs CHANGED Viewed

@@ -4,22 +4,8 @@ var undici = require('undici');
 var node_stream = require('node:stream');
 var web = require('node:stream/web');
 var consumers = require('node:stream/consumers');
+var download_errors = require('./download_errors.cjs');
-/** Throws when a status code of the downloading URL was in range [400, 500). */
-class NetworkError400 extends Error {
-    name = 'NetworkError400';
-}
-/**
- * There are backend versions that return 1 less byte than requested in range.
- * For such cases, this error will be thrown, so client can retry the request.
- * Dowloader will retry the request with one more byte in range.
- */
-class OffByOneError extends Error {
-    name = 'OffByOneError';
-}
-function isOffByOneError(error) {
-    return error instanceof Error && error.name === 'OffByOneError';
-}
 class RemoteFileDownloader {
     httpClient;
     offByOneServers = [];
@@ -76,7 +62,7 @@ class RemoteFileDownloader {
                                 // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.
                                 if (sizeBytes === 1) {
                                     recordOffByOne();
-                                    controller.error(new OffByOneError());
+                                    controller.error(new download_errors.OffByOneError());
                                 }
                             },
                         });
@@ -104,16 +90,11 @@ class RemoteFileDownloader {
 async function checkStatusCodeOk(statusCode, webBody, url) {
     if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {
         const beginning = (await consumers.text(webBody)).substring(0, 1000);
-        if (400 <= statusCode && statusCode < 500) {
-            throw new NetworkError400(`Http error: statusCode: ${statusCode} `
-                + `url: ${url.toString()}, beginning of body: ${beginning}`);
-        }
-        throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);
+        if (400 <= statusCode && statusCode < 500)
+            throw new download_errors.DownloadNetworkError400(statusCode, url, beginning);
+        throw new download_errors.DownloadNetworkError(statusCode, url, beginning);
     }
 }
-exports.NetworkError400 = NetworkError400;
-exports.OffByOneError = OffByOneError;
 exports.RemoteFileDownloader = RemoteFileDownloader;
-exports.isOffByOneError = isOffByOneError;
 //# sourceMappingURL=download.cjs.map

package/dist/helpers/download.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins /\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\~~n/* Throws when a status code of the downloading URL was in range [400, 500). /\~~nexport class NetworkError400 extends Error {\n name = 'NetworkError400';\n}\n\n/\n There are backend versions that return 1 less byte than requested in range.\n * For such cases, this error will be thrown, so client can retry the request.\n * Dowloader will retry the request with one more byte in range.\n /\nexport class OffByOneError extends Error {\n name = 'OffByOneError';\n}\n\nexport function isOffByOneError(error: unknown): error is OffByOneError {\n return error instanceof Error && error.name === 'OffByOneError';\n}\n\nexport class RemoteFileDownloader {\n private readonly offByOneServers: string[] = [];\n\n constructor(public readonly httpClient: Dispatcher) {}\n\n async withContent<T>(\n url: string,\n reqHeaders: Record<string, string>,\n ops: GetContentOptions,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const headers = { ...reqHeaders };\n const urlOrigin = new URL(url).origin;\n\n // Add range header if specified\n if (ops.range) {\n const offByOne = this.offByOneServers.includes(urlOrigin);\n headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n }\n\n const { statusCode, body, headers: responseHeaders } = await request(url, {\n dispatcher: this.httpClient,\n // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n headers: Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])),\n signal: ops.signal,\n highWaterMark: 1 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n });\n ops.signal?.throwIfAborted();\n\n const webBody = Readable.toWeb(body);\n let handlerSuccess = false;\n\n try {\n await checkStatusCodeOk(statusCode, webBody, url);\n ops.signal?.throwIfAborted();\n\n let result: T \| undefined = undefined;\n\n const contentLength = Number(responseHeaders['content-length']);\n if (Number.isNaN(contentLength) \|\| contentLength === 0) {\n // Some backend versions have a bug that they are not returning content-length header.\n // In this case `content-length` header is returned as 0.\n // We should not clip the result stream to 0 bytes in such case.\n result = await handler(webBody, 0);\n } else {\n // Some backend versions have a bug where they return more data than requested in range.\n // So we have to manually normalize the stream to the expected size.\n const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n constructor(sizeBytes: number, recordOffByOne: () => void) {\n super({\n transform(chunk: Uint8Array, controller) {\n const truncatedChunk = chunk.slice(0, sizeBytes);\n controller.enqueue(truncatedChunk);\n sizeBytes -= truncatedChunk.length;\n if (!sizeBytes) controller.terminate();\n },\n flush(controller) {\n // Some backend versions have a bug where they return 1 less byte than requested in range.\n // We cannot always request one more byte because if this end byte is the last byte of the file,\n // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n if (sizeBytes === 1) {\n recordOffByOne();\n controller.error(new OffByOneError());\n }\n },\n });\n }\n })(size, () => this.offByOneServers.push(urlOrigin)));\n result = await handler(normalizedStream, size);\n }\n\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && !webBody.locked) {\n try {\n await webBody.cancel();\n } catch {\n // Ignore cleanup errors\n }\n }\n throw error;\n }\n }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n const beginning = (await text(webBody)).substring(0, 1000);\n\n if (400 <= statusCode && statusCode < 500) {\n throw new ~~NetworkError400~~(~~\n `Http error:~~ statusCode: ~~${statusCode} `\n + `~~url~~: ${url.toString()}~~, beginning ~~of body: ${beginning}`~~);\n ~~}\n\n~~ throw new ~~Error~~(~~`Http error:~~ statusCode: ~~${statusCode}~~ url: ~~${url.toString(~~)~~}`)~~;\n }\n}\n"],"names":["request","Readable","TransformStream","text"],"mappings":"~~;;;;;;;AAYA;AACM~~,MAAO,eAAgB,SAAQ,KAAK,CAAA;IACxC,IAAI,GAAG,iBAAiB;AACzB;AAED;;;;AAIG;AACG,MAAO,aAAc,SAAQ,KAAK,CAAA;IACtC,IAAI,GAAG,eAAe;AACvB;AAEK,SAAU,eAAe,CAAC,KAAc,EAAA;IAC5C,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe;AACjE;MAEa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAMA,cAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;;;AAG3B,YAAA,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;YACtG,MAAM,EAAE,GAAG,CAAC,MAAM;AAClB,YAAA,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;AAC/B,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAGC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAcC,mBAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,~~IAAI~~,~~aAAa~~,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAMC,cAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;~~QAE1D~~,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG~~,EAAE~~;~~AACzC~~,~~YAAA,~~MAAM,~~IAAI~~,~~eAAe~~,~~CACvB~~,~~CAAA,wBAAA,EAA2B,~~UAAU,~~CAAA~~,~~CAAA;kBACnC,CAAA,KAAA,EAAQ,~~GAAG,~~CAAC,QAAQ,~~EAAE,~~wBAAwB,~~SAAS,~~CAAA,CAAE,~~CAAC;~~QAChE;AAEA~~,~~QAAA,~~MAAM,~~IAAI~~,~~KAAK~~,CAAC,~~CAAA,wBAAA,EAA2B,~~UAAU,~~CAAA~~,~~MAAA,EAAS,~~GAAG,~~CAAC,QAAQ,~~EAAE,~~CAAA~~,~~CAAE,~~CAAC;~~IACjF~~;AACF~~;;;;;;;~~"}
1	+ {"version":3,"file":"download.cjs","sources":["../../src/helpers/download.ts"],"sourcesContent":["// @TODO Gleb Zakharov\n/* eslint-disable n/no-unsupported-features/node-builtins /\nimport type { Dispatcher } from 'undici';\nimport { request } from 'undici';\nimport { Readable } from 'node:stream';\nimport type { ReadableStream } from 'node:stream/web';\nimport { TransformStream } from 'node:stream/web';\nimport { text } from 'node:stream/consumers';\nimport type { GetContentOptions } from '@milaboratories/pl-model-common';\nimport { OffByOneError, DownloadNetworkError400, DownloadNetworkError } from './download_errors';\n\nexport type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;\n\nexport class RemoteFileDownloader {\n private readonly offByOneServers: string[] = [];\n\n constructor(public readonly httpClient: Dispatcher) {}\n\n async withContent<T>(\n url: string,\n reqHeaders: Record<string, string>,\n ops: GetContentOptions,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const headers = { ...reqHeaders };\n const urlOrigin = new URL(url).origin;\n\n // Add range header if specified\n if (ops.range) {\n const offByOne = this.offByOneServers.includes(urlOrigin);\n headers['Range'] = `bytes=${ops.range.from}-${ops.range.to - (offByOne ? 0 : 1)}`;\n }\n\n const { statusCode, body, headers: responseHeaders } = await request(url, {\n dispatcher: this.httpClient,\n // Undici automatically sets certain headers, so we need to lowercase user-provided headers\n // to prevent automatic headers from being set and avoid \"duplicated headers\" error.\n headers: Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])),\n signal: ops.signal,\n highWaterMark: 1 1024 * 1024, // 1MB chunks instead of 64KB, tested to be optimal for Human Aging dataset\n });\n ops.signal?.throwIfAborted();\n\n const webBody = Readable.toWeb(body);\n let handlerSuccess = false;\n\n try {\n await checkStatusCodeOk(statusCode, webBody, url);\n ops.signal?.throwIfAborted();\n\n let result: T \| undefined = undefined;\n\n const contentLength = Number(responseHeaders['content-length']);\n if (Number.isNaN(contentLength) \|\| contentLength === 0) {\n // Some backend versions have a bug that they are not returning content-length header.\n // In this case `content-length` header is returned as 0.\n // We should not clip the result stream to 0 bytes in such case.\n result = await handler(webBody, 0);\n } else {\n // Some backend versions have a bug where they return more data than requested in range.\n // So we have to manually normalize the stream to the expected size.\n const size = ops.range ? ops.range.to - ops.range.from : contentLength;\n const normalizedStream = webBody.pipeThrough(new (class extends TransformStream {\n constructor(sizeBytes: number, recordOffByOne: () => void) {\n super({\n transform(chunk: Uint8Array, controller) {\n const truncatedChunk = chunk.slice(0, sizeBytes);\n controller.enqueue(truncatedChunk);\n sizeBytes -= truncatedChunk.length;\n if (!sizeBytes) controller.terminate();\n },\n flush(controller) {\n // Some backend versions have a bug where they return 1 less byte than requested in range.\n // We cannot always request one more byte because if this end byte is the last byte of the file,\n // the backend will return 416 (Range Not Satisfiable). So error is thrown to force client to retry the request.\n if (sizeBytes === 1) {\n recordOffByOne();\n controller.error(new OffByOneError());\n }\n },\n });\n }\n })(size, () => this.offByOneServers.push(urlOrigin)));\n result = await handler(normalizedStream, size);\n }\n\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && !webBody.locked) {\n try {\n await webBody.cancel();\n } catch {\n // Ignore cleanup errors\n }\n }\n throw error;\n }\n }\n}\n\nasync function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {\n if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {\n const beginning = (await text(webBody)).substring(0, 1000);\n if (400 <= statusCode && statusCode < 500)\n throw new DownloadNetworkError400(statusCode, url, beginning);\n throw new DownloadNetworkError(statusCode, url, beginning);\n }\n}\n"],"names":["request","Readable","TransformStream","OffByOneError","text","DownloadNetworkError400","DownloadNetworkError"],"mappings":";;;;;;;;MAaa,oBAAoB,CAAA;AAGH,IAAA,UAAA;IAFX,eAAe,GAAa,EAAE;AAE/C,IAAA,WAAA,CAA4B,UAAsB,EAAA;QAAtB,IAAA,CAAA,UAAU,GAAV,UAAU;IAAe;IAErD,MAAM,WAAW,CACf,GAAW,EACX,UAAkC,EAClC,GAAsB,EACtB,OAA0B,EAAA;AAE1B,QAAA,MAAM,OAAO,GAAG,EAAE,GAAG,UAAU,EAAE;QACjC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;;AAGrC,QAAA,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;AACzD,YAAA,OAAO,CAAC,OAAO,CAAC,GAAG,CAAA,MAAA,EAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAA,CAAA,EAAI,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE;QACnF;AAEA,QAAA,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,MAAMA,cAAO,CAAC,GAAG,EAAE;YACxE,UAAU,EAAE,IAAI,CAAC,UAAU;;;AAG3B,YAAA,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;YACtG,MAAM,EAAE,GAAG,CAAC,MAAM;AAClB,YAAA,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;AAC/B,SAAA,CAAC;AACF,QAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QAE5B,MAAM,OAAO,GAAGC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpC,IAAI,cAAc,GAAG,KAAK;AAE1B,QAAA,IAAI;YACF,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC;AACjD,YAAA,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;YAE5B,IAAI,MAAM,GAAkB,SAAS;YAErC,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE;;;;gBAItD,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACpC;iBAAO;;;gBAGL,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa;gBACtE,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,cAAcC,mBAAe,CAAA;oBAC7E,WAAA,CAAY,SAAiB,EAAE,cAA0B,EAAA;AACvD,wBAAA,KAAK,CAAC;4BACJ,SAAS,CAAC,KAAiB,EAAE,UAAU,EAAA;gCACrC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;AAChD,gCAAA,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC;AAClC,gCAAA,SAAS,IAAI,cAAc,CAAC,MAAM;AAClC,gCAAA,IAAI,CAAC,SAAS;oCAAE,UAAU,CAAC,SAAS,EAAE;4BACxC,CAAC;AACD,4BAAA,KAAK,CAAC,UAAU,EAAA;;;;AAId,gCAAA,IAAI,SAAS,KAAK,CAAC,EAAE;AACnB,oCAAA,cAAc,EAAE;AAChB,oCAAA,UAAU,CAAC,KAAK,CAAC,IAAIC,6BAAa,EAAE,CAAC;gCACvC;4BACF,CAAC;AACF,yBAAA,CAAC;oBACJ;AACD,iBAAA,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrD,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChD;YAEA,cAAc,GAAG,IAAI;AACrB,YAAA,OAAO,MAAM;QACf;QAAE,OAAO,KAAK,EAAE;;YAEd,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;AACtC,gBAAA,IAAI;AACF,oBAAA,MAAM,OAAO,CAAC,MAAM,EAAE;gBACxB;AAAE,gBAAA,MAAM;;gBAER;YACF;AACA,YAAA,MAAM,KAAK;QACb;IACF;AACD;AAED,eAAe,iBAAiB,CAAC,UAAkB,EAAE,OAAuB,EAAE,GAAW,EAAA;IACvF,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,IAAI,GAAG,2CAA2C;AACnF,QAAA,MAAM,SAAS,GAAG,CAAC,MAAMC,cAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;AAC1D,QAAA,IAAI,GAAG,IAAI,UAAU,IAAI,UAAU,GAAG,GAAG;YACvC,MAAM,IAAIC,uCAAuB,CAAC,UAAU,EAAE,GAAG,EAAE,SAAS,CAAC;QAC/D,MAAM,IAAIC,oCAAoB,CAAC,UAAU,EAAE,GAAG,EAAE,SAAS,CAAC;IAC5D;AACF;;;;"}