npm - @milaboratories/pl-drivers - Versions diffs - 1.16.0 → 1.16.2 - Mend

@milaboratories/pl-drivers 1.16.0 → 1.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (243) hide show

package/dist/drivers/download_blob/download_blob.cjs CHANGED Viewed

@@ -12,27 +12,33 @@ const require_files_cache = require("../helpers/files_cache.cjs");
 const require_cache = require("./sparse_cache/cache.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let node_fs_promises = require("node:fs/promises");
-node_fs_promises = require_runtime.__toESM(node_fs_promises);
+node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_path = require("node:path");
-node_path = require_runtime.__toESM(node_path);
+node_path = require_runtime.__toESM(node_path, 1);
 let node_stream_consumers = require("node:stream/consumers");
 let _milaboratories_computable = require("@milaboratories/computable");
 let _milaboratories_pl_model_common = require("@milaboratories/pl-model-common");
 let _milaboratories_pl_tree = require("@milaboratories/pl-tree");
 let denque = require("denque");
-denque = require_runtime.__toESM(denque);
+denque = require_runtime.__toESM(denque, 1);
 let fs = require("fs");
-fs = require_runtime.__toESM(fs);
+fs = require_runtime.__toESM(fs, 1);
 let node_crypto = require("node:crypto");
 let node_os = require("node:os");
-node_os = require_runtime.__toESM(node_os);
+node_os = require_runtime.__toESM(node_os, 1);
 let node_readline_promises = require("node:readline/promises");
-node_readline_promises = require_runtime.__toESM(node_readline_promises);
+node_readline_promises = require_runtime.__toESM(node_readline_promises, 1);
 //#region src/drivers/download_blob/download_blob.ts
 /** DownloadDriver holds a queue of downloading tasks,
 * and notifies every watcher when a file were downloaded. */
 var DownloadDriver = class DownloadDriver {
+	logger;
+	clientDownload;
+	clientLogs;
+	rangesCacheDir;
+	signer;
+	ops;
 	/** Represents a unique key to the path of a blob as a map. */
 	keyToDownload = /* @__PURE__ */ new Map();
 	/** Writes and removes files to a hard drive and holds a counter for every
@@ -45,6 +51,9 @@ var DownloadDriver = class DownloadDriver {
 	idToLastLines = /* @__PURE__ */ new Map();
 	idToProgressLog = /* @__PURE__ */ new Map();
 	saveDir;
+	/** Downloads that bypassed the ranges cache; counted when issued. */
+	uncachedRequests = 0;
+	uncachedRequestBytes = 0;
 	constructor(logger, clientDownload, clientLogs, saveDir, rangesCacheDir, signer, ops) {
 		this.logger = logger;
 		this.clientDownload = clientDownload;
@@ -147,6 +156,17 @@ var DownloadDriver = class DownloadDriver {
 			bypassRangesCache: true
 		});
 	}
+	/**
+	* Operational metrics for a monitoring panel. Serv cache metrics are reported separately
+	* (different owner) — the panel composes both.
+	*/
+	getMetrics() {
+		return {
+			uncachedRequests: this.uncachedRequests,
+			uncachedRequestBytes: this.uncachedRequestBytes,
+			...this.clientDownload.metrics()
+		};
+	}
 	async getContentImpl({ handle, options, bypassRangesCache = false }) {
 		const request = () => this.withContent(handle, {
 			...options,
@@ -190,8 +210,12 @@ var DownloadDriver = class DownloadDriver {
 				signal,
 				handler
 			});
+			if (bypassRangesCache) this.uncachedRequests++;
 			return await this.clientDownload.withBlobContent(result.info, { signal }, options, async (content, size) => {
-				if (bypassRangesCache) return await handler(content, size);
+				if (bypassRangesCache) return await handler(content.pipeThrough(new TransformStream({ transform: (chunk, controller) => {
+					this.uncachedRequestBytes += chunk.byteLength;
+					controller.enqueue(chunk);
+				} })), size);
 				const [handlerStream, cacheStream] = content.tee();
 				const handlerPromise = handler(handlerStream, size);
 				(0, node_stream_consumers.buffer)(cacheStream).then((data) => this.rangesCache.set(key, range ?? {
@@ -327,6 +351,8 @@ var DownloadDriver = class DownloadDriver {
 };
 /** Keeps a counter to the on demand handle. */
 var OnDemandBlobHolder = class {
+	size;
+	handle;
 	counter = new _milaboratories_ts_helpers.CallersCounter();
 	constructor(size, handle) {
 		this.size = size;
@@ -346,6 +372,9 @@ var OnDemandBlobHolder = class {
 	}
 };
 var LastLinesGetter = class {
+	path;
+	lines;
+	patternToSearch;
 	updater;
 	log;
 	change = new _milaboratories_computable.ChangeSource();

package/dist/drivers/download_blob/download_blob.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"download_blob.cjs","names":["FilesCache","SparseCacheFsRanges","SparseCacheFsFile","SparseCache","TaskProcessor","path","Computable","blobKey","DownloadBlobTask","newLocalHandle","nonRecoverableError","OnDemandBlobResourceSnapshot","getSize","newRemoteHandle","parseLocalHandle","isOffByOneError","isLocalBlobHandle","withFileContent","isRemoteBlobHandle","parseRemoteHandle","newLogHandle","getResourceInfoFromLogHandle","fsp","pathToKey","CallersCounter","ChangeSource","Updater","readline","Denque","os","WrongResourceTypeError"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n AnyLogHandle,\n BlobDriver,\n ContentHandler,\n GetContentOptions,\n LocalBlobHandle,\n LocalBlobHandleAndSize,\n ReadyLogHandle,\n RemoteBlobHandle,\n RemoteBlobHandleAndSize,\n StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n makeResourceSnapshot,\n treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n isLocalBlobHandle,\n newLocalHandle,\n parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n isRemoteBlobHandle,\n newRemoteHandle,\n parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n /*\n A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /\n A hard limit of the amount of sparse cache, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one.\n \n The sparse cache is used to store ranges of blobs.\n * /\n rangesCacheMaxSizeBytes: number;\n\n /\n Max number of concurrent downloads while calculating computable states\n * derived from this driver\n * /\n nConcurrentDownloads: number;\n};\n\n/* DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. /\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n /* Represents a unique key to the path of a blob as a map. /\n private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadBlobTask>;\n private rangesCache: SparseCache;\n\n /* Downloads files and writes them to the local dir. /\n private downloadQueue: TaskProcessor;\n\n private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n private idToLastLines: Map<string, LastLinesGetter> = new Map();\n private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n private readonly saveDir: string;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientDownload: ClientDownload,\n private readonly clientLogs: ClientLogs,\n saveDir: string,\n private readonly rangesCacheDir: string,\n private readonly signer: Signer,\n private readonly ops: DownloadDriverOps,\n ) {\n this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n this.rangesCache = new SparseCache(\n this.logger,\n this.ops.rangesCacheMaxSizeBytes,\n fsRanges,\n fsStorage,\n );\n\n this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n this.saveDir = path.resolve(saveDir);\n }\n\n static async init(\n logger: MiLogger,\n clientDownload: ClientDownload,\n clientLogs: ClientLogs,\n saveDir: string,\n rangesCacheDir: string,\n signer: Signer,\n ops: DownloadDriverOps,\n ): Promise<DownloadDriver> {\n const driver = new DownloadDriver(\n logger,\n clientDownload,\n clientLogs,\n saveDir,\n rangesCacheDir,\n signer,\n ops,\n );\n await driver.rangesCache.reset();\n\n return driver;\n }\n\n /* Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. /\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx: ComputableCtx,\n ): LocalBlobHandleAndSize \| undefined;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ): ComputableStableDefined<LocalBlobHandleAndSize>;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<LocalBlobHandleAndSize \| undefined> \| LocalBlobHandleAndSize \| undefined {\n if (ctx === undefined) {\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n }\n\n const rInfo = treeEntryToResourceInfo(res, ctx);\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n if (result == undefined) {\n ctx.markUnstable(\"download blob is still undefined\");\n }\n\n return result;\n }\n\n private getDownloadedBlobNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n callerId: string,\n ): LocalBlobHandleAndSize \| undefined {\n validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n // We don't need to request files with wider limits,\n // PFrame's engine does it disk-optimally by itself.\n\n const task = this.getOrSetNewTask(rInfo, callerId);\n task.attach(w, callerId);\n\n const result = task.getBlob();\n if (!result.done) {\n return undefined;\n }\n if (result.result.ok) {\n return result.result.value;\n }\n throw result.result.error;\n }\n\n private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n const key = blobKey(rInfo.id);\n\n const inMemoryTask = this.keyToDownload.get(key);\n if (inMemoryTask) {\n return inMemoryTask;\n }\n\n // schedule the blob downloading, then it'll be added to the cache.\n const fPath = path.resolve(this.saveDir, key);\n\n const newTask = new DownloadBlobTask(\n this.logger,\n this.clientDownload,\n rInfo,\n newLocalHandle(fPath, this.signer),\n fPath,\n );\n this.keyToDownload.set(key, newTask);\n\n this.downloadQueue.push({\n fn: () => this.downloadBlob(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask;\n }\n\n private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n await task.download();\n const blob = task.getBlob();\n if (blob.done && blob.result.ok) {\n this.cache.addCache(task, callerId);\n }\n }\n\n /* Gets on demand blob. /\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: undefined,\n fromBytes?: number,\n toBytes?: number,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n fromBytes?: number,\n toBytes?: number,\n ): RemoteBlobHandleAndSize;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): ComputableStableDefined<RemoteBlobHandleAndSize> \| RemoteBlobHandleAndSize \| undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n : res;\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n // note that the watcher is not needed,\n // the handler never changes.\n const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n return result;\n }\n\n private getOnDemandBlobNoCtx(\n info: OnDemandBlobResourceSnapshot,\n callerId: string,\n ): RemoteBlobHandleAndSize {\n validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n if (blob === undefined) {\n blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n this.keyToOnDemand.set(blobKey(info.id), blob);\n }\n\n blob.attach(callerId);\n\n return blob.getHandle();\n }\n\n /* Gets a path from a handle. /\n public getLocalPath(handle: LocalBlobHandle): string {\n const { path } = parseLocalHandle(handle, this.signer);\n return path;\n }\n\n /* Gets a content of a blob by a handle. /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array>;\n /* @deprecated Use {@link getContent} with {@link GetContentOptions} instead /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n range?: RangeBytes,\n ): Promise<Uint8Array>;\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n optionsOrRange?: GetContentOptions \| RangeBytes,\n ): Promise<Uint8Array> {\n let options: GetContentOptions = {};\n if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n if (\"range\" in optionsOrRange) {\n options = optionsOrRange;\n } else {\n const range = optionsOrRange as RangeBytes;\n validateRangeBytes(range, `getContent`);\n options = { range };\n }\n }\n\n return await this.getContentImpl({ handle, options });\n }\n\n /\n Same as {@link getContent}, but bypasses the ranges cache entirely (no read, no write).\n * For local handles this is identical to {@link getContent}, since local content never\n * uses the ranges cache.\n /\n public async getContentDirect(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array> {\n return await this.getContentImpl({\n handle,\n options: options ?? {},\n bypassRangesCache: true,\n });\n }\n\n private async getContentImpl({\n handle,\n options,\n bypassRangesCache = false,\n }: {\n handle: LocalBlobHandle \| RemoteBlobHandle;\n options: GetContentOptions;\n bypassRangesCache?: boolean;\n }): Promise<Uint8Array> {\n const request = () =>\n this.withContent(handle, {\n ...options,\n bypassRangesCache,\n handler: async (content) => {\n const chunks: Uint8Array[] = [];\n for await (const chunk of content) {\n options.signal?.throwIfAborted();\n chunks.push(chunk);\n }\n return Buffer.concat(chunks);\n },\n });\n\n try {\n return await request();\n } catch (error) {\n if (isOffByOneError(error)) {\n return await request();\n }\n throw error;\n }\n }\n\n /* Gets a content stream of a blob by a handle and calls handler with it. /\n public async withContent<T>(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options: GetContentOptions & {\n handler: ContentHandler<T>;\n bypassRangesCache?: boolean;\n },\n ): Promise<T> {\n const { range, signal, handler, bypassRangesCache } = options;\n\n if (isLocalBlobHandle(handle)) {\n return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n }\n\n if (isRemoteBlobHandle(handle)) {\n const result = parseRemoteHandle(handle, this.signer);\n\n const key = blobKey(result.info.id);\n const filePath = bypassRangesCache\n ? undefined\n : await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n signal?.throwIfAborted();\n\n if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n return await this.clientDownload.withBlobContent(\n result.info,\n { signal },\n options,\n async (content, size) => {\n if (bypassRangesCache) return await handler(content, size);\n\n const [handlerStream, cacheStream] = content.tee();\n\n const handlerPromise = handler(handlerStream, size);\n const _cachePromise = buffer(cacheStream)\n .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n .catch(() => {\n // Ignore cache errors - they shouldn't affect the main handler result\n // This prevents unhandled promise rejections when the stream fails\n });\n\n return await handlerPromise;\n },\n );\n }\n\n throw new Error(\"Malformed remote handle\");\n }\n\n /\n Creates computable that will return blob content once it is downloaded.\n * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n /\n public getComputableContent(\n res: ResourceInfo \| PlTreeEntry,\n range?: RangeBytes,\n ): ComputableStableDefined<Uint8Array> {\n if (range) {\n validateRangeBytes(range, `getComputableContent`);\n }\n\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n }).withStableType();\n }\n\n /* Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. /\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n if (result == undefined)\n ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, lines);\n this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ): Computable<string \| undefined>;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string \| undefined;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getProgressLogNoCtx(\n ctx.watcher,\n r as ResourceSnapshot,\n patternToSearch,\n callerId,\n );\n if (result === undefined)\n ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns an Id of a smart object, that can read logs directly from\n * the platform. /\n public getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<AnyLogHandle>;\n public getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n public getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<AnyLogHandle> \| AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n return this.getLogHandleNoCtx(r as ResourceSnapshot);\n }\n\n private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n return newLogHandle(false, rInfo);\n }\n\n public async lastLines(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number, // if 0n, then start from the end.\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n public async readText(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n const task = this.keyToDownload.get(blobKey(rInfo.id));\n if (task == undefined) {\n return;\n }\n\n if (this.cache.existsFile(blobKey(rInfo.id))) {\n const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n await Promise.all(\n toDelete.map(async (cachedFile) => {\n await fsp.rm(cachedFile.path);\n\n this.cache.removeCache(cachedFile);\n\n this.removeTask(\n mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted) {\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n }\n\n private removeTask(task: DownloadBlobTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n this.keyToDownload.delete(pathToKey(task.path));\n this.idToLastLines.delete(blobKey(task.rInfo.id));\n this.idToProgressLog.delete(blobKey(task.rInfo.id));\n }\n\n private async releaseOnDemandBlob(blobId: SignedResourceId, callerId: string) {\n const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n this.keyToDownload.forEach((task, key) => {\n this.keyToDownload.delete(key);\n task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n });\n }\n\n async dispose(): Promise<void> {\n await this.rangesCache.dispose();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.dispose();\n }\n}\n\n/* Keeps a counter to the on demand handle. /\nclass OnDemandBlobHolder {\n private readonly counter = new CallersCounter();\n\n constructor(\n private readonly size: number,\n private readonly handle: RemoteBlobHandle,\n ) {}\n\n public getHandle(): RemoteBlobHandleAndSize {\n return { handle: this.handle, size: this.size };\n }\n\n public attach(callerId: string) {\n this.counter.inc(callerId);\n }\n\n public release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n}\n\nclass LastLinesGetter {\n private updater: Updater;\n private log: string \| undefined;\n private readonly change: ChangeSource = new ChangeSource();\n private error: any \| undefined = undefined;\n\n constructor(\n private readonly path: string,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {\n this.updater = new Updater(async () => this.update());\n }\n\n getOrSchedule(w: Watcher): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n this.change.attachWatcher(w);\n\n this.updater.schedule();\n\n return {\n log: this.log,\n error: this.error,\n };\n }\n\n async update(): Promise<void> {\n try {\n const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n this.log = newLogs;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.log = \"\";\n this.error = e;\n this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n return;\n }\n\n throw e;\n }\n }\n}\n\n/* Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n fPath: string,\n nLines: number,\n patternToSearch?: string,\n): Promise<string> {\n let inStream: fs.ReadStream \| undefined;\n let rl: readline.Interface \| undefined;\n\n try {\n inStream = fs.createReadStream(fPath);\n rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n const lines = new Denque();\n\n for await (const line of rl) {\n if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n lines.push(line);\n if (lines.length > nLines) {\n lines.shift();\n }\n }\n\n // last EOL is for keeping backward compat with platforma implementation.\n return lines.toArray().join(os.EOL) + os.EOL;\n } finally {\n // Cleanup resources in finally block to ensure they're always cleaned up\n try {\n if (rl) {\n rl.close();\n }\n } catch (cleanupError) {\n console.error(\"Error closing readline interface:\", cleanupError);\n }\n\n try {\n if (inStream && !inStream.destroyed) {\n inStream.destroy();\n }\n } catch (cleanupError) {\n console.error(\"Error destroying read stream:\", cleanupError);\n }\n }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"Blob/\")) {\n let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n if (rType.name == \"Blob\")\n message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n throw new WrongResourceTypeError(message);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,gCAAuD,IAAI,KAAK;;;CAIhE;CACA;;CAGA;CAEA,gCAAyD,IAAI,KAAK;CAElE,gCAAsD,IAAI,KAAK;CAC/D,kCAAwD,IAAI,KAAK;CAEjE;CAEA,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;AAPiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,aAAA;AAEA,OAAA,iBAAA;AACA,OAAA,SAAA;AACA,OAAA,MAAA;AAEjB,OAAK,QAAQ,IAAIA,oBAAAA,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAIC,cAAAA,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAIC,cAAAA,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAIC,cAAAA,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAIC,2BAAAA,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUC,UAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,kBACE,KACA,KACqF;AACrF,MAAI,QAAQ,KAAA,EACV,QAAOC,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,SAAA,GAAA,wBAAA,yBAAgC,KAAK,IAAI;EAE/C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,uBACE,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAMC,iBAAAA,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQF,UAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAIG,2BAAAA,iBAClB,KAAK,QACL,KAAK,gBACL,OACAC,8BAAAA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,gBACE,KACA,KACwF;AACxF,MAAI,QAAQ,KAAA,EAAW,QAAOJ,2BAAAA,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,SAAA,GAAA,wBAAA,eAAoD,IAAI,IAAA,GAAA,wBAAA,sBACrC,KAAKK,cAAAA,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,qBACE,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAIJ,iBAAAA,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,KAAA,GAAW;AACtB,UAAO,IAAI,mBAAmBK,cAAAA,QAAQ,KAAK,EAAEC,+BAAAA,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAIN,iBAAAA,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAASO,8BAAAA,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,IAAA,GAAA,gCAAA,oBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;AAIvB,SAAO,MAAM,KAAK,eAAe;GAAE;GAAQ;GAAS,CAAC;;;;;;;CAQvD,MAAa,iBACX,QACA,SACqB;AACrB,SAAO,MAAM,KAAK,eAAe;GAC/B;GACA,SAAS,WAAW,EAAE;GACtB,mBAAmB;GACpB,CAAC;;CAGJ,MAAc,eAAe,EAC3B,QACA,SACA,oBAAoB,SAKE;EACtB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH;GACA,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAIC,wBAAAA,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAIY;EACZ,MAAM,EAAE,OAAO,QAAQ,SAAS,sBAAsB;AAEtD,MAAIC,8BAAAA,kBAAkB,OAAO,CAC3B,QAAO,MAAMC,kBAAAA,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAIC,+BAAAA,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAASC,+BAAAA,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAMZ,iBAAAA,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,oBACb,KAAA,IACA,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AAC1E,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAMU,kBAAAA,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;AACvB,QAAI,kBAAmB,QAAO,MAAM,QAAQ,SAAS,KAAK;IAE1D,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,KAAA,GAAA,sBAAA,QAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,qBACE,KACA,OACqC;AACrC,MAAI,MACF,EAAA,GAAA,gCAAA,oBAAmB,OAAO,uBAAuB;AAGnD,SAAOX,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAA,GACtE,CAAC,CAAC,gBAAgB;;CAcrB,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAE9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAIP,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAIA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;EAC3C,MAAM,YAAA,GAAA,YAAA,aAAuB;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,KAAA,EACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAC9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAIP,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAIA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,aACE,KACA,KACyC;AACzC,MAAI,OAAO,KAAA,EAAW,QAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,KAAA,GAAA,wBAAA,yBAA4B,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,kBAA0B,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAOc,oBAAAA,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjCC,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjCA,oBAAAA,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAId,iBAAAA,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,KAAA,EACV;AAGF,MAAI,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAMe,iBAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,YAAA,GAAA,2BAAA,QACI,KAAK,eAAeC,iBAAAA,UAAU,WAAW,KAAK,CAAC,EACtD,aAAA,GAAA,0BAAA,yBAAoC,WAAW,CAAC,qCAAA,GAAA,0BAAA,yBACG,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,aAAA,GAAA,0BAAA,yBAAoC,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,WAAmB,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAOA,iBAAAA,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAOhB,iBAAAA,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAOA,iBAAAA,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAA0B,UAAkB;AAE5E,MADgB,KAAK,cAAc,IAAIA,iBAAAA,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAOA,iBAAAA,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,SAAA,GAAA,0BAAA,oBAA2B,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,UAA2B,IAAIiB,2BAAAA,gBAAgB;CAE/C,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,YAA4C;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,OAAc,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAe,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB;CACA;CACA,SAAwC,IAAIC,2BAAAA,cAAc;CAC1D,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;AAHiB,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;AAEjB,OAAK,UAAU,IAAIC,gBAAAA,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,QAAA,GAAA,0BAAA,iBAAoB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAW,GAAG,iBAAiB,MAAM;AACrC,OAAKC,uBAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAIC,OAAAA,SAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAKC,QAAG,IAAI,GAAGA,QAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAIC,gBAAAA,uBAAuB,QAAQ"}
1	+ {"version":3,"file":"download_blob.cjs","names":["FilesCache","SparseCacheFsRanges","SparseCacheFsFile","SparseCache","TaskProcessor","path","Computable","blobKey","DownloadBlobTask","newLocalHandle","nonRecoverableError","OnDemandBlobResourceSnapshot","getSize","newRemoteHandle","parseLocalHandle","isOffByOneError","isLocalBlobHandle","withFileContent","isRemoteBlobHandle","parseRemoteHandle","newLogHandle","getResourceInfoFromLogHandle","fsp","pathToKey","CallersCounter","ChangeSource","Updater","readline","Denque","os","WrongResourceTypeError"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n AnyLogHandle,\n BlobDriver,\n BlobDriverMetrics,\n ContentHandler,\n GetContentOptions,\n LocalBlobHandle,\n LocalBlobHandleAndSize,\n ReadyLogHandle,\n RemoteBlobHandle,\n RemoteBlobHandleAndSize,\n StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n makeResourceSnapshot,\n treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n isLocalBlobHandle,\n newLocalHandle,\n parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n isRemoteBlobHandle,\n newRemoteHandle,\n parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n /*\n A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /\n A hard limit of the amount of sparse cache, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one.\n \n The sparse cache is used to store ranges of blobs.\n * /\n rangesCacheMaxSizeBytes: number;\n\n /\n Max number of concurrent downloads while calculating computable states\n * derived from this driver\n * /\n nConcurrentDownloads: number;\n};\n\n/* DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. /\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n /* Represents a unique key to the path of a blob as a map. /\n private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadBlobTask>;\n private rangesCache: SparseCache;\n\n /* Downloads files and writes them to the local dir. /\n private downloadQueue: TaskProcessor;\n\n private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n private idToLastLines: Map<string, LastLinesGetter> = new Map();\n private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n private readonly saveDir: string;\n\n /* Downloads that bypassed the ranges cache; counted when issued. /\n private uncachedRequests = 0;\n private uncachedRequestBytes = 0;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientDownload: ClientDownload,\n private readonly clientLogs: ClientLogs,\n saveDir: string,\n private readonly rangesCacheDir: string,\n private readonly signer: Signer,\n private readonly ops: DownloadDriverOps,\n ) {\n this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n this.rangesCache = new SparseCache(\n this.logger,\n this.ops.rangesCacheMaxSizeBytes,\n fsRanges,\n fsStorage,\n );\n\n this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n this.saveDir = path.resolve(saveDir);\n }\n\n static async init(\n logger: MiLogger,\n clientDownload: ClientDownload,\n clientLogs: ClientLogs,\n saveDir: string,\n rangesCacheDir: string,\n signer: Signer,\n ops: DownloadDriverOps,\n ): Promise<DownloadDriver> {\n const driver = new DownloadDriver(\n logger,\n clientDownload,\n clientLogs,\n saveDir,\n rangesCacheDir,\n signer,\n ops,\n );\n await driver.rangesCache.reset();\n\n return driver;\n }\n\n /* Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. /\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx: ComputableCtx,\n ): LocalBlobHandleAndSize \| undefined;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ): ComputableStableDefined<LocalBlobHandleAndSize>;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<LocalBlobHandleAndSize \| undefined> \| LocalBlobHandleAndSize \| undefined {\n if (ctx === undefined) {\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n }\n\n const rInfo = treeEntryToResourceInfo(res, ctx);\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n if (result == undefined) {\n ctx.markUnstable(\"download blob is still undefined\");\n }\n\n return result;\n }\n\n private getDownloadedBlobNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n callerId: string,\n ): LocalBlobHandleAndSize \| undefined {\n validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n // We don't need to request files with wider limits,\n // PFrame's engine does it disk-optimally by itself.\n\n const task = this.getOrSetNewTask(rInfo, callerId);\n task.attach(w, callerId);\n\n const result = task.getBlob();\n if (!result.done) {\n return undefined;\n }\n if (result.result.ok) {\n return result.result.value;\n }\n throw result.result.error;\n }\n\n private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n const key = blobKey(rInfo.id);\n\n const inMemoryTask = this.keyToDownload.get(key);\n if (inMemoryTask) {\n return inMemoryTask;\n }\n\n // schedule the blob downloading, then it'll be added to the cache.\n const fPath = path.resolve(this.saveDir, key);\n\n const newTask = new DownloadBlobTask(\n this.logger,\n this.clientDownload,\n rInfo,\n newLocalHandle(fPath, this.signer),\n fPath,\n );\n this.keyToDownload.set(key, newTask);\n\n this.downloadQueue.push({\n fn: () => this.downloadBlob(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask;\n }\n\n private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n await task.download();\n const blob = task.getBlob();\n if (blob.done && blob.result.ok) {\n this.cache.addCache(task, callerId);\n }\n }\n\n /* Gets on demand blob. /\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: undefined,\n fromBytes?: number,\n toBytes?: number,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n fromBytes?: number,\n toBytes?: number,\n ): RemoteBlobHandleAndSize;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): ComputableStableDefined<RemoteBlobHandleAndSize> \| RemoteBlobHandleAndSize \| undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n : res;\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n // note that the watcher is not needed,\n // the handler never changes.\n const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n return result;\n }\n\n private getOnDemandBlobNoCtx(\n info: OnDemandBlobResourceSnapshot,\n callerId: string,\n ): RemoteBlobHandleAndSize {\n validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n if (blob === undefined) {\n blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n this.keyToOnDemand.set(blobKey(info.id), blob);\n }\n\n blob.attach(callerId);\n\n return blob.getHandle();\n }\n\n /* Gets a path from a handle. /\n public getLocalPath(handle: LocalBlobHandle): string {\n const { path } = parseLocalHandle(handle, this.signer);\n return path;\n }\n\n /* Gets a content of a blob by a handle. /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array>;\n /* @deprecated Use {@link getContent} with {@link GetContentOptions} instead /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n range?: RangeBytes,\n ): Promise<Uint8Array>;\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n optionsOrRange?: GetContentOptions \| RangeBytes,\n ): Promise<Uint8Array> {\n let options: GetContentOptions = {};\n if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n if (\"range\" in optionsOrRange) {\n options = optionsOrRange;\n } else {\n const range = optionsOrRange as RangeBytes;\n validateRangeBytes(range, `getContent`);\n options = { range };\n }\n }\n\n return await this.getContentImpl({ handle, options });\n }\n\n /\n Same as {@link getContent}, but bypasses the ranges cache entirely (no read, no write).\n * For local handles this is identical to {@link getContent}, since local content never\n * uses the ranges cache.\n /\n public async getContentDirect(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array> {\n return await this.getContentImpl({\n handle,\n options: options ?? {},\n bypassRangesCache: true,\n });\n }\n\n /\n Operational metrics for a monitoring panel. Serv cache metrics are reported separately\n * (different owner) — the panel composes both.\n /\n public getMetrics(): BlobDriverMetrics {\n return {\n uncachedRequests: this.uncachedRequests,\n uncachedRequestBytes: this.uncachedRequestBytes,\n ...this.clientDownload.metrics(),\n };\n }\n\n private async getContentImpl({\n handle,\n options,\n bypassRangesCache = false,\n }: {\n handle: LocalBlobHandle \| RemoteBlobHandle;\n options: GetContentOptions;\n bypassRangesCache?: boolean;\n }): Promise<Uint8Array> {\n const request = () =>\n this.withContent(handle, {\n ...options,\n bypassRangesCache,\n handler: async (content) => {\n const chunks: Uint8Array[] = [];\n for await (const chunk of content) {\n options.signal?.throwIfAborted();\n chunks.push(chunk);\n }\n return Buffer.concat(chunks);\n },\n });\n\n try {\n return await request();\n } catch (error) {\n if (isOffByOneError(error)) {\n return await request();\n }\n throw error;\n }\n }\n\n /* Gets a content stream of a blob by a handle and calls handler with it. /\n public async withContent<T>(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options: GetContentOptions & {\n handler: ContentHandler<T>;\n bypassRangesCache?: boolean;\n },\n ): Promise<T> {\n const { range, signal, handler, bypassRangesCache } = options;\n\n if (isLocalBlobHandle(handle)) {\n return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n }\n\n if (isRemoteBlobHandle(handle)) {\n const result = parseRemoteHandle(handle, this.signer);\n\n const key = blobKey(result.info.id);\n const filePath = bypassRangesCache\n ? undefined\n : await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n signal?.throwIfAborted();\n\n if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n if (bypassRangesCache) this.uncachedRequests++;\n\n return await this.clientDownload.withBlobContent(\n result.info,\n { signal },\n options,\n async (content, size) => {\n if (bypassRangesCache) {\n const counted = content.pipeThrough(\n new TransformStream<Uint8Array, Uint8Array>({\n transform: (chunk, controller) => {\n this.uncachedRequestBytes += chunk.byteLength;\n controller.enqueue(chunk);\n },\n }),\n );\n return await handler(counted, size);\n }\n\n const [handlerStream, cacheStream] = content.tee();\n\n const handlerPromise = handler(handlerStream, size);\n const _cachePromise = buffer(cacheStream)\n .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n .catch(() => {\n // Ignore cache errors - they shouldn't affect the main handler result\n // This prevents unhandled promise rejections when the stream fails\n });\n\n return await handlerPromise;\n },\n );\n }\n\n throw new Error(\"Malformed remote handle\");\n }\n\n /\n Creates computable that will return blob content once it is downloaded.\n * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n /\n public getComputableContent(\n res: ResourceInfo \| PlTreeEntry,\n range?: RangeBytes,\n ): ComputableStableDefined<Uint8Array> {\n if (range) {\n validateRangeBytes(range, `getComputableContent`);\n }\n\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n }).withStableType();\n }\n\n /* Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. /\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n if (result == undefined)\n ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, lines);\n this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ): Computable<string \| undefined>;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string \| undefined;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getProgressLogNoCtx(\n ctx.watcher,\n r as ResourceSnapshot,\n patternToSearch,\n callerId,\n );\n if (result === undefined)\n ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns an Id of a smart object, that can read logs directly from\n * the platform. /\n public getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<AnyLogHandle>;\n public getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n public getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<AnyLogHandle> \| AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n return this.getLogHandleNoCtx(r as ResourceSnapshot);\n }\n\n private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n return newLogHandle(false, rInfo);\n }\n\n public async lastLines(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number, // if 0n, then start from the end.\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n public async readText(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n const task = this.keyToDownload.get(blobKey(rInfo.id));\n if (task == undefined) {\n return;\n }\n\n if (this.cache.existsFile(blobKey(rInfo.id))) {\n const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n await Promise.all(\n toDelete.map(async (cachedFile) => {\n await fsp.rm(cachedFile.path);\n\n this.cache.removeCache(cachedFile);\n\n this.removeTask(\n mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted) {\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n }\n\n private removeTask(task: DownloadBlobTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n this.keyToDownload.delete(pathToKey(task.path));\n this.idToLastLines.delete(blobKey(task.rInfo.id));\n this.idToProgressLog.delete(blobKey(task.rInfo.id));\n }\n\n private async releaseOnDemandBlob(blobId: SignedResourceId, callerId: string) {\n const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n this.keyToDownload.forEach((task, key) => {\n this.keyToDownload.delete(key);\n task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n });\n }\n\n async dispose(): Promise<void> {\n await this.rangesCache.dispose();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.dispose();\n }\n}\n\n/* Keeps a counter to the on demand handle. /\nclass OnDemandBlobHolder {\n private readonly counter = new CallersCounter();\n\n constructor(\n private readonly size: number,\n private readonly handle: RemoteBlobHandle,\n ) {}\n\n public getHandle(): RemoteBlobHandleAndSize {\n return { handle: this.handle, size: this.size };\n }\n\n public attach(callerId: string) {\n this.counter.inc(callerId);\n }\n\n public release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n}\n\nclass LastLinesGetter {\n private updater: Updater;\n private log: string \| undefined;\n private readonly change: ChangeSource = new ChangeSource();\n private error: any \| undefined = undefined;\n\n constructor(\n private readonly path: string,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {\n this.updater = new Updater(async () => this.update());\n }\n\n getOrSchedule(w: Watcher): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n this.change.attachWatcher(w);\n\n this.updater.schedule();\n\n return {\n log: this.log,\n error: this.error,\n };\n }\n\n async update(): Promise<void> {\n try {\n const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n this.log = newLogs;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.log = \"\";\n this.error = e;\n this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n return;\n }\n\n throw e;\n }\n }\n}\n\n/* Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n fPath: string,\n nLines: number,\n patternToSearch?: string,\n): Promise<string> {\n let inStream: fs.ReadStream \| undefined;\n let rl: readline.Interface \| undefined;\n\n try {\n inStream = fs.createReadStream(fPath);\n rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n const lines = new Denque();\n\n for await (const line of rl) {\n if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n lines.push(line);\n if (lines.length > nLines) {\n lines.shift();\n }\n }\n\n // last EOL is for keeping backward compat with platforma implementation.\n return lines.toArray().join(os.EOL) + os.EOL;\n } finally {\n // Cleanup resources in finally block to ensure they're always cleaned up\n try {\n if (rl) {\n rl.close();\n }\n } catch (cleanupError) {\n console.error(\"Error closing readline interface:\", cleanupError);\n }\n\n try {\n if (inStream && !inStream.destroyed) {\n inStream.destroy();\n }\n } catch (cleanupError) {\n console.error(\"Error destroying read stream:\", cleanupError);\n }\n }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"Blob/\")) {\n let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n if (rType.name == \"Blob\")\n message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n throw new WrongResourceTypeError(message);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqFA,IAAa,iBAAb,MAAa,eAAsD;CAwB9C;CACA;CACA;CAEA;CACA;CACA;;CA5BnB,gCAAuD,IAAI,IAAI;;;CAI/D;CACA;;CAGA;CAEA,gCAAyD,IAAI,IAAI;CAEjE,gCAAsD,IAAI,IAAI;CAC9D,kCAAwD,IAAI,IAAI;CAEhE;;CAGA,mBAA2B;CAC3B,uBAA+B;CAE/B,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;EAPiB,KAAA,SAAA;EACA,KAAA,iBAAA;EACA,KAAA,aAAA;EAEA,KAAA,iBAAA;EACA,KAAA,SAAA;EACA,KAAA,MAAA;EAEjB,KAAK,QAAQ,IAAIA,oBAAAA,WAAW,KAAK,IAAI,kBAAkB;EAEvD,MAAM,WAAW,IAAIC,cAAAA,oBAAoB,KAAK,QAAQ,KAAK,cAAc;EACzE,MAAM,YAAY,IAAIC,cAAAA,kBAAkB,KAAK,QAAQ,KAAK,cAAc;EACxE,KAAK,cAAc,IAAIC,cAAAA,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,SACF;EAEA,KAAK,gBAAgB,IAAIC,2BAAAA,cAAc,KAAK,QAAQ,IAAI,oBAAoB;EAE5E,KAAK,UAAUC,UAAK,QAAQ,OAAO;CACrC;CAEA,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,GACF;EACA,MAAM,OAAO,YAAY,MAAM;EAE/B,OAAO;CACT;CAUA,kBACE,KACA,KACqF;EACrF,IAAI,QAAQ,KAAA,GACV,OAAOC,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,GAAG,CAAC;EAGlE,MAAM,SAAA,GAAA,wBAAA,wBAAA,CAAgC,KAAK,GAAG;EAE9C,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAC5B,IAAI,mBAAmB,KAAK,YAAY,OAAO,QAAQ,CAAC;EAExD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,QAAQ;EAC3F,IAAI,UAAU,KAAA,GACZ,IAAI,aAAa,kCAAkC;EAGrD,OAAO;CACT;CAEA,uBACE,GACA,OACA,UACoC;EACpC,iCAAiC,qBAAqB,MAAM,IAAI;EAKhE,MAAM,OAAO,KAAK,gBAAgB,OAAO,QAAQ;EACjD,KAAK,OAAO,GAAG,QAAQ;EAEvB,MAAM,SAAS,KAAK,QAAQ;EAC5B,IAAI,CAAC,OAAO,MACV;EAEF,IAAI,OAAO,OAAO,IAChB,OAAO,OAAO,OAAO;EAEvB,MAAM,OAAO,OAAO;CACtB;CAEA,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAMC,iBAAAA,QAAQ,MAAM,EAAE;EAE5B,MAAM,eAAe,KAAK,cAAc,IAAI,GAAG;EAC/C,IAAI,cACF,OAAO;EAIT,MAAM,QAAQF,UAAK,QAAQ,KAAK,SAAS,GAAG;EAE5C,MAAM,UAAU,IAAIG,2BAAAA,iBAClB,KAAK,QACL,KAAK,gBACL,OACAC,8BAAAA,eAAe,OAAO,KAAK,MAAM,GACjC,KACF;EACA,KAAK,cAAc,IAAI,KAAK,OAAO;EAEnC,KAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,QAAQ;GAC7C,4BAA4B,MAAM,CAACC,2BAAAA,oBAAoB,CAAC;EAC1D,CAAC;EAED,OAAO;CACT;CAEA,MAAc,aAAa,MAAwB,UAAkB;EACnE,MAAM,KAAK,SAAS;EACpB,MAAM,OAAO,KAAK,QAAQ;EAC1B,IAAI,KAAK,QAAQ,KAAK,OAAO,IAC3B,KAAK,MAAM,SAAS,MAAM,QAAQ;CAEtC;CAkBA,gBACE,KACA,KACwF;EACxF,IAAI,QAAQ,KAAA,GAAW,OAAOJ,2BAAAA,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,GAAG,CAAC;EAErF,MAAM,SAAA,GAAA,wBAAA,cAAA,CAAoD,GAAG,KAAA,GAAA,wBAAA,qBAAA,CACpC,KAAKK,cAAAA,8BAA8B,GAAG,IAC3D;EAEJ,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAC5B,IAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,QAAQ,CAAC;EAMnE,OAFe,KAAK,qBAAqB,OAAO,QAEpC;CACd;CAEA,qBACE,MACA,UACyB;EACzB,iCAAiC,mBAAmB,KAAK,IAAI;EAE7D,IAAI,OAAO,KAAK,cAAc,IAAIJ,iBAAAA,QAAQ,KAAK,EAAE,CAAC;EAElD,IAAI,SAAS,KAAA,GAAW;GACtB,OAAO,IAAI,mBAAmBK,cAAAA,QAAQ,IAAI,GAAGC,+BAAAA,gBAAgB,MAAM,KAAK,MAAM,CAAC;GAC/E,KAAK,cAAc,IAAIN,iBAAAA,QAAQ,KAAK,EAAE,GAAG,IAAI;EAC/C;EAEA,KAAK,OAAO,QAAQ;EAEpB,OAAO,KAAK,UAAU;CACxB;;CAGA,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAASO,8BAAAA,iBAAiB,QAAQ,KAAK,MAAM;EACrD,OAAO;CACT;CAYA,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,CAAC;EAClC,IAAI,OAAO,mBAAmB,YAAY,mBAAmB,MAC3D,IAAI,WAAW,gBACb,UAAU;OACL;GACL,MAAM,QAAQ;GACd,CAAA,GAAA,gCAAA,mBAAA,CAAmB,OAAO,YAAY;GACtC,UAAU,EAAE,MAAM;EACpB;EAGF,OAAO,MAAM,KAAK,eAAe;GAAE;GAAQ;EAAQ,CAAC;CACtD;;;;;;CAOA,MAAa,iBACX,QACA,SACqB;EACrB,OAAO,MAAM,KAAK,eAAe;GAC/B;GACA,SAAS,WAAW,CAAC;GACrB,mBAAmB;EACrB,CAAC;CACH;;;;;CAMA,aAAuC;EACrC,OAAO;GACL,kBAAkB,KAAK;GACvB,sBAAsB,KAAK;GAC3B,GAAG,KAAK,eAAe,QAAQ;EACjC;CACF;CAEA,MAAc,eAAe,EAC3B,QACA,SACA,oBAAoB,SAKE;EACtB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH;GACA,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,CAAC;IAC9B,WAAW,MAAM,SAAS,SAAS;KACjC,QAAQ,QAAQ,eAAe;KAC/B,OAAO,KAAK,KAAK;IACnB;IACA,OAAO,OAAO,OAAO,MAAM;GAC7B;EACF,CAAC;EAEH,IAAI;GACF,OAAO,MAAM,QAAQ;EACvB,SAAS,OAAO;GACd,IAAIC,wBAAAA,gBAAgB,KAAK,GACvB,OAAO,MAAM,QAAQ;GAEvB,MAAM;EACR;CACF;;CAGA,MAAa,YACX,QACA,SAIY;EACZ,MAAM,EAAE,OAAO,QAAQ,SAAS,sBAAsB;EAEtD,IAAIC,8BAAAA,kBAAkB,MAAM,GAC1B,OAAO,MAAMC,kBAAAA,gBAAgB;GAAE,MAAM,KAAK,aAAa,MAAM;GAAG;GAAO;GAAQ;EAAQ,CAAC;EAG1F,IAAIC,+BAAAA,mBAAmB,MAAM,GAAG;GAC9B,MAAM,SAASC,+BAAAA,kBAAkB,QAAQ,KAAK,MAAM;GAEpD,MAAM,MAAMZ,iBAAAA,QAAQ,OAAO,KAAK,EAAE;GAClC,MAAM,WAAW,oBACb,KAAA,IACA,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;GAAK,CAAC;GACzE,QAAQ,eAAe;GAEvB,IAAI,UAAU,OAAO,MAAMU,kBAAAA,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;GAAQ,CAAC;GAErF,IAAI,mBAAmB,KAAK;GAE5B,OAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,OAAO,GACT,SACA,OAAO,SAAS,SAAS;IACvB,IAAI,mBASF,OAAO,MAAM,QARG,QAAQ,YACtB,IAAI,gBAAwC,EAC1C,YAAY,OAAO,eAAe;KAChC,KAAK,wBAAwB,MAAM;KACnC,WAAW,QAAQ,KAAK;IAC1B,EACF,CAAC,CAEwB,GAAG,IAAI;IAGpC,MAAM,CAAC,eAAe,eAAe,QAAQ,IAAI;IAEjD,MAAM,iBAAiB,QAAQ,eAAe,IAAI;IAC5B,CAAA,GAAA,sBAAA,OAAA,CAAO,WAAW,CAAC,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;IAAK,GAAG,IAAI,CAAC,CAAC,CACtF,YAAY,CAGb,CAAC;IAEH,OAAO,MAAM;GACf,CACF;EACF;EAEA,MAAM,IAAI,MAAM,yBAAyB;CAC3C;;;;;CAMA,qBACE,KACA,OACqC;EACrC,IAAI,OACF,CAAA,GAAA,gCAAA,mBAAA,CAAmB,OAAO,sBAAsB;EAGlD,OAAOX,2BAAAA,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,GAAG,GAAG,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,KAAA,EACvE,CAAC,CAAC,CAAC,eAAe;CACpB;CAaA,YACE,KACA,OACA,KACqD;EACrD,IAAI,OAAO,KAAA,GAAW,OAAOA,2BAAAA,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,GAAG,CAAC;EAEvF,MAAM,KAAA,GAAA,wBAAA,wBAAA,CAA4B,KAAK,GAAG;EAC1C,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAC5B,IAAI,mBAAmB,KAAK,YAAY,GAAG,QAAQ,CAAC;EAEpD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,QAAQ;EACxF,IAAI,UAAU,KAAA,GACZ,IAAI,aAAa,uDAAuD;EAE1E,OAAO;CACT;CAEA,iBACE,GACA,OACA,OACA,UACoB;EACpB,iCAAiC,eAAe,MAAM,IAAI;EAC1D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,QAAQ;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAE9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,MAAM;EAE1D,IAAI,YAAY,KAAK,cAAc,IAAIP,iBAAAA,QAAQ,MAAM,EAAE,CAAC;EAExD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,KAAK;GACpD,KAAK,cAAc,IAAIA,iBAAAA,QAAQ,MAAM,EAAE,GAAG,YAAY;GACtD,YAAY;EACd;EAEA,MAAM,SAAS,UAAU,cAAc,CAAC;EACxC,IAAI,OAAO,OAAO,MAAM,OAAO;EAE/B,OAAO,OAAO;CAChB;CAaA,eACE,KACA,iBACA,KACqD;EACrD,IAAI,OAAO,KAAA,GACT,OAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,GAAG,CAAC;EAEhF,MAAM,KAAA,GAAA,wBAAA,wBAAA,CAA4B,KAAK,GAAG;EAC1C,MAAM,YAAA,GAAA,YAAA,WAAA,CAAsB;EAC5B,IAAI,mBAAmB,KAAK,YAAY,GAAG,QAAQ,CAAC;EAEpD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,QACF;EACA,IAAI,WAAW,KAAA,GACb,IAAI,aAAa,iEAAiE;EAEpF,OAAO;CACT;CAEA,oBACE,GACA,OACA,iBACA,UACoB;EACpB,iCAAiC,kBAAkB,MAAM,IAAI;EAE7D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,QAAQ;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,MAAM,EAAE,SAASQ,8BAAAA,iBAAiB,KAAK,QAAQ,KAAK,MAAM;EAE1D,IAAI,YAAY,KAAK,gBAAgB,IAAIP,iBAAAA,QAAQ,MAAM,EAAE,CAAC;EAE1D,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,eAAe;GACjE,KAAK,gBAAgB,IAAIA,iBAAAA,QAAQ,MAAM,EAAE,GAAG,YAAY;GAExD,YAAY;EACd;EAEA,MAAM,SAAS,UAAU,cAAc,CAAC;EACxC,IAAI,OAAO,OAAO,MAAM,OAAO;EAE/B,OAAO,OAAO;CAChB;CAMA,aACE,KACA,KACyC;EACzC,IAAI,OAAO,KAAA,GAAW,OAAOD,2BAAAA,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,GAAG,CAAC;EAEjF,MAAM,KAAA,GAAA,wBAAA,wBAAA,CAA4B,KAAK,GAAG;EAE1C,OAAO,KAAK,kBAAkB,CAAqB;CACrD;CAEA,kBAA0B,OAAuC;EAC/D,iCAAiC,gBAAgB,MAAM,IAAI;EAC3D,OAAOc,oBAAAA,aAAa,OAAO,KAAK;CAClC;CAEA,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjCC,oBAAAA,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF;EAEA,OAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,IAAI;GACtB,WAAW,OAAO,KAAK,SAAS;EAClC;CACF;CAEA,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjCA,oBAAAA,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF;EAEA,OAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,IAAI;GACtB,WAAW,OAAO,KAAK,SAAS;EAClC;CACF;CAEA,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAId,iBAAAA,QAAQ,MAAM,EAAE,CAAC;EACrD,IAAI,QAAQ,KAAA,GACV;EAGF,IAAI,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,EAAE,CAAC,GAAG;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAWA,iBAAAA,QAAQ,MAAM,EAAE,GAAG,QAAQ;GAElE,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;IACjC,MAAMe,iBAAI,GAAG,WAAW,IAAI;IAE5B,KAAK,MAAM,YAAY,UAAU;IAEjC,KAAK,YAAA,GAAA,2BAAA,OAAA,CACI,KAAK,eAAeC,iBAAAA,UAAU,WAAW,IAAI,CAAC,GACrD,aAAA,GAAA,0BAAA,wBAAA,CAAoC,UAAU,EAAE,qCAAA,GAAA,0BAAA,wBAAA,CACG,SAAS,KAAK,MAAM,EAAE,IAAI,CAAC,GAChF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,aAAA,GAAA,0BAAA,wBAAA,CAAoC,KAAK,KAAK,CAAC,EAAE,wBACnD;CAGN;CAEA,WAAmB,MAAwB,QAAgB;EACzD,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,QAAQ;EAC3E,KAAK,cAAc,OAAOA,iBAAAA,UAAU,KAAK,IAAI,CAAC;EAC9C,KAAK,cAAc,OAAOhB,iBAAAA,QAAQ,KAAK,MAAM,EAAE,CAAC;EAChD,KAAK,gBAAgB,OAAOA,iBAAAA,QAAQ,KAAK,MAAM,EAAE,CAAC;CACpD;CAEA,MAAc,oBAAoB,QAA0B,UAAkB;EAE5E,IADgB,KAAK,cAAc,IAAIA,iBAAAA,QAAQ,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,KAAK,OACjE,KAAK,cAAc,OAAOA,iBAAAA,QAAQ,MAAM,CAAC;CACxD;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,KAAK,cAAc,SAAS,MAAM,QAAQ;GACxC,KAAK,cAAc,OAAO,GAAG;GAC7B,KAAK,OAAO,YAAY,SAAA,GAAA,0BAAA,mBAAA,CAA2B,KAAK,MAAM,EAAE,EAAE,UAAU;EAC9E,CAAC;CACH;CAEA,MAAM,UAAyB;EAC7B,MAAM,KAAK,YAAY,QAAQ;CACjC;CAEA,OAAO,OAAO,gBAA+B;EAC3C,MAAM,KAAK,QAAQ;CACrB;AACF;;AAGA,IAAM,qBAAN,MAAyB;CAIJ;CACA;CAJnB,UAA2B,IAAIiB,2BAAAA,eAAe;CAE9C,YACE,MACA,QACA;EAFiB,KAAA,OAAA;EACA,KAAA,SAAA;CAChB;CAEH,YAA4C;EAC1C,OAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;EAAK;CAChD;CAEA,OAAc,UAAkB;EAC9B,KAAK,QAAQ,IAAI,QAAQ;CAC3B;CAEA,QAAe,UAA2B;EACxC,OAAO,KAAK,QAAQ,IAAI,QAAQ;CAClC;AACF;AAEA,IAAM,kBAAN,MAAsB;CAOD;CACA;CACA;CARnB;CACA;CACA,SAAwC,IAAIC,2BAAAA,aAAa;CACzD,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;EAHiB,KAAA,OAAA;EACA,KAAA,QAAA;EACA,KAAA,kBAAA;EAEjB,KAAK,UAAU,IAAIC,gBAAAA,QAAQ,YAAY,KAAK,OAAO,CAAC;CACtD;CAEA,cAAc,GAGZ;EACA,KAAK,OAAO,cAAc,CAAC;EAE3B,KAAK,QAAQ,SAAS;EAEtB,OAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;EACd;CACF;CAEA,MAAM,SAAwB;EAC5B,IAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,eAAe;GAE9E,IAAI,KAAK,OAAO,SAAS,KAAK,OAAO,YAAY,YAAY,KAAK,KAAK,SAAS;GAChF,KAAK,MAAM;EACb,SAAS,GAAQ;GACf,KAAA,GAAA,0BAAA,gBAAA,CAAoB,CAAC,GAAG;IAEtB,KAAK,MAAM;IACX,KAAK,QAAQ;IACb,KAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,4BAA4B;IAChF;GACF;GAEA,MAAM;EACR;CACF;AACF;;;AAIA,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;CAEJ,IAAI;EACF,WAAW,GAAG,iBAAiB,KAAK;EACpC,KAAKC,uBAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;EAAS,CAAC;EAEtE,MAAM,QAAQ,IAAIC,OAAAA,QAAO;EAEzB,WAAW,MAAM,QAAQ,IAAI;GAC3B,IAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,eAAe,GAAG;GAErE,MAAM,KAAK,IAAI;GACf,IAAI,MAAM,SAAS,QACjB,MAAM,MAAM;EAEhB;EAGA,OAAO,MAAM,QAAQ,CAAC,CAAC,KAAKC,QAAG,GAAG,IAAIA,QAAG;CAC3C,UAAU;EAER,IAAI;GACF,IAAI,IACF,GAAG,MAAM;EAEb,SAAS,cAAc;GACrB,QAAQ,MAAM,qCAAqC,YAAY;EACjE;EAEA,IAAI;GACF,IAAI,YAAY,CAAC,SAAS,WACxB,SAAS,QAAQ;EAErB,SAAS,cAAc;GACrB,QAAQ,MAAM,iCAAiC,YAAY;EAC7D;CACF;AACF;AAEA,SAAS,iCAAiC,YAAoB,OAAqB;CACjF,IAAI,CAAC,MAAM,KAAK,WAAW,OAAO,GAAG;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;EAChE,IAAI,MAAM,QAAQ,QAChB,WAAW;EAEb,MAAM,IAAIC,gBAAAA,uBAAuB,OAAO;CAC1C;AACF"}

package/dist/drivers/download_blob/download_blob.d.ts CHANGED Viewed

@@ -3,7 +3,7 @@ import { ClientDownload } from "../../clients/download.js";
 import { ClientLogs } from "../../clients/logs.js";
 import { MiLogger, Signer } from "@milaboratories/ts-helpers";
 import { Computable, ComputableCtx, ComputableStableDefined } from "@milaboratories/computable";
-import { AnyLogHandle, BlobDriver, ContentHandler, GetContentOptions, LocalBlobHandle, LocalBlobHandleAndSize, RangeBytes, ReadyLogHandle, RemoteBlobHandle, RemoteBlobHandleAndSize, StreamingApiResponse } from "@milaboratories/pl-model-common";
+import { AnyLogHandle, BlobDriver, BlobDriverMetrics, ContentHandler, GetContentOptions, LocalBlobHandle, LocalBlobHandleAndSize, RangeBytes, ReadyLogHandle, RemoteBlobHandle, RemoteBlobHandleAndSize, StreamingApiResponse } from "@milaboratories/pl-model-common";
 import { PlTreeEntry, ResourceInfo } from "@milaboratories/pl-tree";
 //#region src/drivers/download_blob/download_blob.d.ts
@@ -48,6 +48,9 @@ declare class DownloadDriver implements BlobDriver, AsyncDisposable {
   private idToLastLines;
   private idToProgressLog;
   private readonly saveDir;
+  /** Downloads that bypassed the ranges cache; counted when issued. */
+  private uncachedRequests;
+  private uncachedRequestBytes;
   constructor(logger: MiLogger, clientDownload: ClientDownload, clientLogs: ClientLogs, saveDir: string, rangesCacheDir: string, signer: Signer, ops: DownloadDriverOps);
   static init(logger: MiLogger, clientDownload: ClientDownload, clientLogs: ClientLogs, saveDir: string, rangesCacheDir: string, signer: Signer, ops: DownloadDriverOps): Promise<DownloadDriver>;
   /** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */
@@ -73,6 +76,11 @@ declare class DownloadDriver implements BlobDriver, AsyncDisposable {
    * uses the ranges cache.
    */
   getContentDirect(handle: LocalBlobHandle | RemoteBlobHandle, options?: GetContentOptions): Promise<Uint8Array>;
+  /**
+   * Operational metrics for a monitoring panel. Serv cache metrics are reported separately
+   * (different owner) — the panel composes both.
+   */
+  getMetrics(): BlobDriverMetrics;
   private getContentImpl;
   /** Gets a content stream of a blob by a handle and calls handler with it. */
   withContent<T>(handle: LocalBlobHandle | RemoteBlobHandle, options: GetContentOptions & {

package/dist/drivers/download_blob/download_blob.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"download_blob.d.ts","names":[],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"mappings":";;;;;;;;;~~KA2DY~~,iBAAA;EAAiB;;;;;EAM3B,kBAAA;EAcoB;;AAKtB;;;;EAXE,uBAAA;~~EAiC~~+B;;;;~~EA3B~~/B,oBAAA;AAAA;;;cAKW,cAAA,YAA0B,UAAA,EAAY,eAAA;EAAA,~~iBAoB9B~~,MAAA;EAAA,iBACA,cAAA;EAAA,iBACA,UAAA;EAAA,iBAEA,cAAA;EAAA,iBACA,MAAA;EAAA,iBACA,GAAA;EA+CG;EAAA,~~QAvEd~~,aAAA;~~EAwEL~~;;EAAA,~~QApEK~~,KAAA;EAAA,QACA,WAAA;~~EAuJL~~;EAAA,~~QApJK~~,aAAA;EAAA,QAEA,aAAA;EAAA,QAEA,aAAA;EAAA,QACA,eAAA;EAAA,iBAES,OAAA;~~cAGE~~,MAAA,EAAQ,QAAA,EACR,cAAA,EAAgB,cAAA,EAChB,UAAA,EAAY,UAAA,EAC7B,OAAA,UACiB,cAAA,UACA,MAAA,EAAQ,MAAA,EACR,GAAA,EAAK,iBAAA;EAAA,OAkBX,IAAA,CACX,MAAA,EAAQ,QAAA,EACR,cAAA,EAAgB,cAAA,EAChB,UAAA,EAAY,UAAA,EACZ,OAAA,UACA,cAAA,UACA,MAAA,EAAQ,MAAA,EACR,GAAA,EAAK,iBAAA,GACJ,OAAA,CAAQ,cAAA;~~EAsHR~~;~~EAtGI~~,iBAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,GAAA,EAAK,aAAA,GACJ,sBAAA;EACI,iBAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,GACnB,uBAAA,CAAwB,sBAAA;EAAA,QAsBnB,sBAAA;EAAA,QAuBA,eAAA;EAAA,QA4BM,YAAA;~~EAwEX~~;~~EA/DI~~,eAAA,CACL,GAAA,EAAK,4BAAA,GAA+B,WAAA,GACnC,UAAA,CAAW,uBAAA;EACP,eAAA,CACL,GAAA,EAAK,4BAAA,GAA+B,WAAA,EACpC,GAAA,cACA,SAAA,WACA,OAAA,YACC,UAAA,CAAW,uBAAA;EACP,eAAA,CACL,GAAA,EAAK,4BAAA,GAA+B,WAAA,EACpC,GAAA,EAAK,aAAA,EACL,SAAA,WACA,OAAA,YACC,uBAAA;EAAA,QAqBK,oBAAA;~~EAiCL~~;~~EAdI~~,YAAA,CAAa,MAAA,EAAQ,eAAA;~~EAuCA~~;~~EAjCf~~,UAAA,CACX,MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,OAAA,GAAU,iBAAA,GACT,OAAA,CAAQ,UAAA;~~EAgCA~~;~~EA9BE~~,UAAA,CACX,MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,KAAA,GAAQ,UAAA,GACP,OAAA,CAAQ,UAAA;~~EAsED~~;;;;;~~EA9CG~~,gBAAA,CACX,MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,OAAA,GAAU,iBAAA,GACT,OAAA,CAAQ,UAAA;EAAA,~~QAQG~~,cAAA;~~EAwFP~~;~~EAtDM~~,WAAA,~~GAAA~~,~~CACX,~~MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,OAAA,EAAS,iBAAA;IACP,OAAA,EAAS,cAAA,CAAe,CAAA;IACxB,iBAAA;EAAA,IAED,OAAA,CAAQ,CAAA;~~EA+DJ~~;;;;~~EAhBA~~,oBAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,KAAA,GAAQ,UAAA,GACP,uBAAA,CAAwB,UAAA;~~EAmBpB~~;;~~EAPA~~,WAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,KAAA,WACC,UAAA;EACI,WAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,KAAA,UACA,GAAA,EAAK,aAAA,GACJ,UAAA;EAAA,QAmBK,gBAAA;~~EAiCD~~;;~~EALA~~,cAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,eAAA,WACC,UAAA;EACI,cAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,eAAA,UACA,GAAA,EAAK,aAAA;EAAA,QA0BC,mBAAA;~~EA6ByD~~;;~~EAA1D~~,YAAA,CAAa,GAAA,EAAK,YAAA,GAAe,WAAA,GAAc,UAAA,CAAW,YAAA;EAC1D,YAAA,CAAa,GAAA,EAAK,YAAA,GAAe,WAAA,EAAa,GAAA,EAAK,aAAA,GAAgB,YAAA;EAAA,QAYlE,iBAAA;EAKK,SAAA,CACX,MAAA,EAAQ,cAAA,EACR,SAAA,UACA,WAAA;EACA,SAAA,YACC,OAAA,CAAQ,oBAAA;EAiBE,QAAA,CACX,MAAA,EAAQ,cAAA,EACR,SAAA,UACA,WAAA,WACA,SAAA,YACC,OAAA,CAAQ,oBAAA;EAAA,QAiBG,WAAA;EAAA,QAkCN,UAAA;EAAA,QAQM,mBAAA;~~EA3DX~~;~~EAiEG~~,UAAA,~~CAAA~~,~~GAAU,~~OAAA;EASV,OAAA,~~CAAA~~,~~GAAW,~~OAAA;EAAA,CAIV,MAAA,CAAO,YAAA,KAAiB,OAAA;AAAA"}
1	+ {"version":3,"file":"download_blob.d.ts","names":[],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"mappings":";;;;;;;;;KA4DY,iBAAA;EAAiB;;;;;EAM3B,kBAAA;EAcoB;AAAA;AAKtB;;;;EAXE,uBAAA;EAqC+B;;;;EA/B/B,oBAAA;AAAA;;;cAKW,cAAA,YAA0B,UAAA,EAAY,eAAA;EAAA,iBAwB9B,MAAA;EAAA,iBACA,cAAA;EAAA,iBACA,UAAA;EAAA,iBAEA,cAAA;EAAA,iBACA,MAAA;EAAA,iBACA,GAAA;EA+CG;EAAA,QA3Ed,aAAA;EA4EL;;EAAA,QAxEK,KAAA;EAAA,QACA,WAAA;EA2JL;EAAA,QAxJK,aAAA;EAAA,QAEA,aAAA;EAAA,QAEA,aAAA;EAAA,QACA,eAAA;EAAA,iBAES,OAAA;EAyJqB;EAAA,QAtJ9B,gBAAA;EAAA,QACA,oBAAA;cAGW,MAAA,EAAQ,QAAA,EACR,cAAA,EAAgB,cAAA,EAChB,UAAA,EAAY,UAAA,EAC7B,OAAA,UACiB,cAAA,UACA,MAAA,EAAQ,MAAA,EACR,GAAA,EAAK,iBAAA;EAAA,OAkBX,IAAA,CACX,MAAA,EAAQ,QAAA,EACR,cAAA,EAAgB,cAAA,EAChB,UAAA,EAAY,UAAA,EACZ,OAAA,UACA,cAAA,UACA,MAAA,EAAQ,MAAA,EACR,GAAA,EAAK,iBAAA,GACJ,OAAA,CAAQ,cAAA;EAqKiB;EArJrB,iBAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,GAAA,EAAK,aAAA,GACJ,sBAAA;EACI,iBAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,GACnB,uBAAA,CAAwB,sBAAA;EAAA,QAsBnB,sBAAA;EAAA,QAuBA,eAAA;EAAA,QA4BM,YAAA;EA4EJ;EAnEH,eAAA,CACL,GAAA,EAAK,4BAAA,GAA+B,WAAA,GACnC,UAAA,CAAW,uBAAA;EACP,eAAA,CACL,GAAA,EAAK,4BAAA,GAA+B,WAAA,EACpC,GAAA,cACA,SAAA,WACA,OAAA,YACC,UAAA,CAAW,uBAAA;EACP,eAAA,CACL,GAAA,EAAK,4BAAA,GAA+B,WAAA,EACpC,GAAA,EAAK,aAAA,EACL,SAAA,WACA,OAAA,YACC,uBAAA;EAAA,QAqBK,oBAAA;EA2DI;EAxCL,YAAA,CAAa,MAAA,EAAQ,eAAA;EAyCzB;EAnCU,UAAA,CACX,MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,OAAA,GAAU,iBAAA,GACT,OAAA,CAAQ,UAAA;EAuFD;EArFG,UAAA,CACX,MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,KAAA,GAAQ,UAAA,GACP,OAAA,CAAQ,UAAA;EAmFA;;;;;EA3DE,gBAAA,CACX,MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,OAAA,GAAU,iBAAA,GACT,OAAA,CAAQ,UAAA;EAwHW;;;;EA5Gf,UAAA,IAAc,iBAAA;EAAA,QAQP,cAAA;EAqHX;EAnFU,WAAA,IACX,MAAA,EAAQ,eAAA,GAAkB,gBAAA,EAC1B,OAAA,EAAS,iBAAA;IACP,OAAA,EAAS,cAAA,CAAe,CAAA;IACxB,iBAAA;EAAA,IAED,OAAA,CAAQ,CAAA;EAkIJ;;;;EAvEA,oBAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,KAAA,GAAQ,UAAA,GACP,uBAAA,CAAwB,UAAA;EA0EpB;;EA9DA,WAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,KAAA,WACC,UAAA;EACI,WAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,KAAA,UACA,GAAA,EAAK,aAAA,GACJ,UAAA;EAAA,QAmBK,gBAAA;EA2FiB;;EA/DlB,cAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,eAAA,WACC,UAAA;EACI,cAAA,CACL,GAAA,EAAK,YAAA,GAAe,WAAA,EACpB,eAAA,UACA,GAAA,EAAK,aAAA;EAAA,QA0BC,mBAAA;EAoDG;;EAvBJ,YAAA,CAAa,GAAA,EAAK,YAAA,GAAe,WAAA,GAAc,UAAA,CAAW,YAAA;EAC1D,YAAA,CAAa,GAAA,EAAK,YAAA,GAAe,WAAA,EAAa,GAAA,EAAK,aAAA,GAAgB,YAAA;EAAA,QAYlE,iBAAA;EAKK,SAAA,CACX,MAAA,EAAQ,cAAA,EACR,SAAA,UACA,WAAA;EACA,SAAA,YACC,OAAA,CAAQ,oBAAA;EAiBE,QAAA,CACX,MAAA,EAAQ,cAAA,EACR,SAAA,UACA,WAAA,WACA,SAAA,YACC,OAAA,CAAQ,oBAAA;EAAA,QAiBG,WAAA;EAAA,QAkCN,UAAA;EAAA,QAQM,mBAAA;EAlmBkD;EAwmB1D,UAAA,IAAU,OAAA;EASV,OAAA,IAAW,OAAA;EAAA,CAIV,MAAA,CAAO,YAAA,KAAiB,OAAA;AAAA"}

package/dist/drivers/download_blob/download_blob.js CHANGED Viewed

@@ -26,6 +26,12 @@ import * as readline from "node:readline/promises";
 /** DownloadDriver holds a queue of downloading tasks,
 * and notifies every watcher when a file were downloaded. */
 var DownloadDriver = class DownloadDriver {
+	logger;
+	clientDownload;
+	clientLogs;
+	rangesCacheDir;
+	signer;
+	ops;
 	/** Represents a unique key to the path of a blob as a map. */
 	keyToDownload = /* @__PURE__ */ new Map();
 	/** Writes and removes files to a hard drive and holds a counter for every
@@ -38,6 +44,9 @@ var DownloadDriver = class DownloadDriver {
 	idToLastLines = /* @__PURE__ */ new Map();
 	idToProgressLog = /* @__PURE__ */ new Map();
 	saveDir;
+	/** Downloads that bypassed the ranges cache; counted when issued. */
+	uncachedRequests = 0;
+	uncachedRequestBytes = 0;
 	constructor(logger, clientDownload, clientLogs, saveDir, rangesCacheDir, signer, ops) {
 		this.logger = logger;
 		this.clientDownload = clientDownload;
@@ -140,6 +149,17 @@ var DownloadDriver = class DownloadDriver {
 			bypassRangesCache: true
 		});
 	}
+	/**
+	* Operational metrics for a monitoring panel. Serv cache metrics are reported separately
+	* (different owner) — the panel composes both.
+	*/
+	getMetrics() {
+		return {
+			uncachedRequests: this.uncachedRequests,
+			uncachedRequestBytes: this.uncachedRequestBytes,
+			...this.clientDownload.metrics()
+		};
+	}
 	async getContentImpl({ handle, options, bypassRangesCache = false }) {
 		const request = () => this.withContent(handle, {
 			...options,
@@ -183,8 +203,12 @@ var DownloadDriver = class DownloadDriver {
 				signal,
 				handler
 			});
+			if (bypassRangesCache) this.uncachedRequests++;
 			return await this.clientDownload.withBlobContent(result.info, { signal }, options, async (content, size) => {
-				if (bypassRangesCache) return await handler(content, size);
+				if (bypassRangesCache) return await handler(content.pipeThrough(new TransformStream({ transform: (chunk, controller) => {
+					this.uncachedRequestBytes += chunk.byteLength;
+					controller.enqueue(chunk);
+				} })), size);
 				const [handlerStream, cacheStream] = content.tee();
 				const handlerPromise = handler(handlerStream, size);
 				buffer(cacheStream).then((data) => this.rangesCache.set(key, range ?? {
@@ -320,6 +344,8 @@ var DownloadDriver = class DownloadDriver {
 };
 /** Keeps a counter to the on demand handle. */
 var OnDemandBlobHolder = class {
+	size;
+	handle;
 	counter = new CallersCounter();
 	constructor(size, handle) {
 		this.size = size;
@@ -339,6 +365,9 @@ var OnDemandBlobHolder = class {
 	}
 };
 var LastLinesGetter = class {
+	path;
+	lines;
+	patternToSearch;
 	updater;
 	log;
 	change = new ChangeSource();

package/dist/drivers/download_blob/download_blob.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"download_blob.js","names":["path","fs"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n AnyLogHandle,\n BlobDriver,\n ContentHandler,\n GetContentOptions,\n LocalBlobHandle,\n LocalBlobHandleAndSize,\n ReadyLogHandle,\n RemoteBlobHandle,\n RemoteBlobHandleAndSize,\n StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n makeResourceSnapshot,\n treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n isLocalBlobHandle,\n newLocalHandle,\n parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n isRemoteBlobHandle,\n newRemoteHandle,\n parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n /*\n A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /\n A hard limit of the amount of sparse cache, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one.\n \n The sparse cache is used to store ranges of blobs.\n * /\n rangesCacheMaxSizeBytes: number;\n\n /\n Max number of concurrent downloads while calculating computable states\n * derived from this driver\n * /\n nConcurrentDownloads: number;\n};\n\n/* DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. /\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n /* Represents a unique key to the path of a blob as a map. /\n private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadBlobTask>;\n private rangesCache: SparseCache;\n\n /* Downloads files and writes them to the local dir. /\n private downloadQueue: TaskProcessor;\n\n private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n private idToLastLines: Map<string, LastLinesGetter> = new Map();\n private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n private readonly saveDir: string;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientDownload: ClientDownload,\n private readonly clientLogs: ClientLogs,\n saveDir: string,\n private readonly rangesCacheDir: string,\n private readonly signer: Signer,\n private readonly ops: DownloadDriverOps,\n ) {\n this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n this.rangesCache = new SparseCache(\n this.logger,\n this.ops.rangesCacheMaxSizeBytes,\n fsRanges,\n fsStorage,\n );\n\n this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n this.saveDir = path.resolve(saveDir);\n }\n\n static async init(\n logger: MiLogger,\n clientDownload: ClientDownload,\n clientLogs: ClientLogs,\n saveDir: string,\n rangesCacheDir: string,\n signer: Signer,\n ops: DownloadDriverOps,\n ): Promise<DownloadDriver> {\n const driver = new DownloadDriver(\n logger,\n clientDownload,\n clientLogs,\n saveDir,\n rangesCacheDir,\n signer,\n ops,\n );\n await driver.rangesCache.reset();\n\n return driver;\n }\n\n /* Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. /\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx: ComputableCtx,\n ): LocalBlobHandleAndSize \| undefined;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ): ComputableStableDefined<LocalBlobHandleAndSize>;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<LocalBlobHandleAndSize \| undefined> \| LocalBlobHandleAndSize \| undefined {\n if (ctx === undefined) {\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n }\n\n const rInfo = treeEntryToResourceInfo(res, ctx);\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n if (result == undefined) {\n ctx.markUnstable(\"download blob is still undefined\");\n }\n\n return result;\n }\n\n private getDownloadedBlobNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n callerId: string,\n ): LocalBlobHandleAndSize \| undefined {\n validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n // We don't need to request files with wider limits,\n // PFrame's engine does it disk-optimally by itself.\n\n const task = this.getOrSetNewTask(rInfo, callerId);\n task.attach(w, callerId);\n\n const result = task.getBlob();\n if (!result.done) {\n return undefined;\n }\n if (result.result.ok) {\n return result.result.value;\n }\n throw result.result.error;\n }\n\n private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n const key = blobKey(rInfo.id);\n\n const inMemoryTask = this.keyToDownload.get(key);\n if (inMemoryTask) {\n return inMemoryTask;\n }\n\n // schedule the blob downloading, then it'll be added to the cache.\n const fPath = path.resolve(this.saveDir, key);\n\n const newTask = new DownloadBlobTask(\n this.logger,\n this.clientDownload,\n rInfo,\n newLocalHandle(fPath, this.signer),\n fPath,\n );\n this.keyToDownload.set(key, newTask);\n\n this.downloadQueue.push({\n fn: () => this.downloadBlob(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask;\n }\n\n private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n await task.download();\n const blob = task.getBlob();\n if (blob.done && blob.result.ok) {\n this.cache.addCache(task, callerId);\n }\n }\n\n /* Gets on demand blob. /\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: undefined,\n fromBytes?: number,\n toBytes?: number,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n fromBytes?: number,\n toBytes?: number,\n ): RemoteBlobHandleAndSize;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): ComputableStableDefined<RemoteBlobHandleAndSize> \| RemoteBlobHandleAndSize \| undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n : res;\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n // note that the watcher is not needed,\n // the handler never changes.\n const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n return result;\n }\n\n private getOnDemandBlobNoCtx(\n info: OnDemandBlobResourceSnapshot,\n callerId: string,\n ): RemoteBlobHandleAndSize {\n validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n if (blob === undefined) {\n blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n this.keyToOnDemand.set(blobKey(info.id), blob);\n }\n\n blob.attach(callerId);\n\n return blob.getHandle();\n }\n\n /* Gets a path from a handle. /\n public getLocalPath(handle: LocalBlobHandle): string {\n const { path } = parseLocalHandle(handle, this.signer);\n return path;\n }\n\n /* Gets a content of a blob by a handle. /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array>;\n /* @deprecated Use {@link getContent} with {@link GetContentOptions} instead /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n range?: RangeBytes,\n ): Promise<Uint8Array>;\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n optionsOrRange?: GetContentOptions \| RangeBytes,\n ): Promise<Uint8Array> {\n let options: GetContentOptions = {};\n if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n if (\"range\" in optionsOrRange) {\n options = optionsOrRange;\n } else {\n const range = optionsOrRange as RangeBytes;\n validateRangeBytes(range, `getContent`);\n options = { range };\n }\n }\n\n return await this.getContentImpl({ handle, options });\n }\n\n /\n Same as {@link getContent}, but bypasses the ranges cache entirely (no read, no write).\n * For local handles this is identical to {@link getContent}, since local content never\n * uses the ranges cache.\n /\n public async getContentDirect(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array> {\n return await this.getContentImpl({\n handle,\n options: options ?? {},\n bypassRangesCache: true,\n });\n }\n\n private async getContentImpl({\n handle,\n options,\n bypassRangesCache = false,\n }: {\n handle: LocalBlobHandle \| RemoteBlobHandle;\n options: GetContentOptions;\n bypassRangesCache?: boolean;\n }): Promise<Uint8Array> {\n const request = () =>\n this.withContent(handle, {\n ...options,\n bypassRangesCache,\n handler: async (content) => {\n const chunks: Uint8Array[] = [];\n for await (const chunk of content) {\n options.signal?.throwIfAborted();\n chunks.push(chunk);\n }\n return Buffer.concat(chunks);\n },\n });\n\n try {\n return await request();\n } catch (error) {\n if (isOffByOneError(error)) {\n return await request();\n }\n throw error;\n }\n }\n\n /* Gets a content stream of a blob by a handle and calls handler with it. /\n public async withContent<T>(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options: GetContentOptions & {\n handler: ContentHandler<T>;\n bypassRangesCache?: boolean;\n },\n ): Promise<T> {\n const { range, signal, handler, bypassRangesCache } = options;\n\n if (isLocalBlobHandle(handle)) {\n return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n }\n\n if (isRemoteBlobHandle(handle)) {\n const result = parseRemoteHandle(handle, this.signer);\n\n const key = blobKey(result.info.id);\n const filePath = bypassRangesCache\n ? undefined\n : await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n signal?.throwIfAborted();\n\n if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n return await this.clientDownload.withBlobContent(\n result.info,\n { signal },\n options,\n async (content, size) => {\n if (bypassRangesCache) return await handler(content, size);\n\n const [handlerStream, cacheStream] = content.tee();\n\n const handlerPromise = handler(handlerStream, size);\n const _cachePromise = buffer(cacheStream)\n .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n .catch(() => {\n // Ignore cache errors - they shouldn't affect the main handler result\n // This prevents unhandled promise rejections when the stream fails\n });\n\n return await handlerPromise;\n },\n );\n }\n\n throw new Error(\"Malformed remote handle\");\n }\n\n /\n Creates computable that will return blob content once it is downloaded.\n * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n /\n public getComputableContent(\n res: ResourceInfo \| PlTreeEntry,\n range?: RangeBytes,\n ): ComputableStableDefined<Uint8Array> {\n if (range) {\n validateRangeBytes(range, `getComputableContent`);\n }\n\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n }).withStableType();\n }\n\n /* Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. /\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n if (result == undefined)\n ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, lines);\n this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ): Computable<string \| undefined>;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string \| undefined;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getProgressLogNoCtx(\n ctx.watcher,\n r as ResourceSnapshot,\n patternToSearch,\n callerId,\n );\n if (result === undefined)\n ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns an Id of a smart object, that can read logs directly from\n * the platform. /\n public getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<AnyLogHandle>;\n public getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n public getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<AnyLogHandle> \| AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n return this.getLogHandleNoCtx(r as ResourceSnapshot);\n }\n\n private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n return newLogHandle(false, rInfo);\n }\n\n public async lastLines(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number, // if 0n, then start from the end.\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n public async readText(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n const task = this.keyToDownload.get(blobKey(rInfo.id));\n if (task == undefined) {\n return;\n }\n\n if (this.cache.existsFile(blobKey(rInfo.id))) {\n const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n await Promise.all(\n toDelete.map(async (cachedFile) => {\n await fsp.rm(cachedFile.path);\n\n this.cache.removeCache(cachedFile);\n\n this.removeTask(\n mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted) {\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n }\n\n private removeTask(task: DownloadBlobTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n this.keyToDownload.delete(pathToKey(task.path));\n this.idToLastLines.delete(blobKey(task.rInfo.id));\n this.idToProgressLog.delete(blobKey(task.rInfo.id));\n }\n\n private async releaseOnDemandBlob(blobId: SignedResourceId, callerId: string) {\n const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n this.keyToDownload.forEach((task, key) => {\n this.keyToDownload.delete(key);\n task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n });\n }\n\n async dispose(): Promise<void> {\n await this.rangesCache.dispose();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.dispose();\n }\n}\n\n/* Keeps a counter to the on demand handle. /\nclass OnDemandBlobHolder {\n private readonly counter = new CallersCounter();\n\n constructor(\n private readonly size: number,\n private readonly handle: RemoteBlobHandle,\n ) {}\n\n public getHandle(): RemoteBlobHandleAndSize {\n return { handle: this.handle, size: this.size };\n }\n\n public attach(callerId: string) {\n this.counter.inc(callerId);\n }\n\n public release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n}\n\nclass LastLinesGetter {\n private updater: Updater;\n private log: string \| undefined;\n private readonly change: ChangeSource = new ChangeSource();\n private error: any \| undefined = undefined;\n\n constructor(\n private readonly path: string,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {\n this.updater = new Updater(async () => this.update());\n }\n\n getOrSchedule(w: Watcher): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n this.change.attachWatcher(w);\n\n this.updater.schedule();\n\n return {\n log: this.log,\n error: this.error,\n };\n }\n\n async update(): Promise<void> {\n try {\n const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n this.log = newLogs;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.log = \"\";\n this.error = e;\n this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n return;\n }\n\n throw e;\n }\n }\n}\n\n/* Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n fPath: string,\n nLines: number,\n patternToSearch?: string,\n): Promise<string> {\n let inStream: fs.ReadStream \| undefined;\n let rl: readline.Interface \| undefined;\n\n try {\n inStream = fs.createReadStream(fPath);\n rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n const lines = new Denque();\n\n for await (const line of rl) {\n if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n lines.push(line);\n if (lines.length > nLines) {\n lines.shift();\n }\n }\n\n // last EOL is for keeping backward compat with platforma implementation.\n return lines.toArray().join(os.EOL) + os.EOL;\n } finally {\n // Cleanup resources in finally block to ensure they're always cleaned up\n try {\n if (rl) {\n rl.close();\n }\n } catch (cleanupError) {\n console.error(\"Error closing readline interface:\", cleanupError);\n }\n\n try {\n if (inStream && !inStream.destroyed) {\n inStream.destroy();\n }\n } catch (cleanupError) {\n console.error(\"Error destroying read stream:\", cleanupError);\n }\n }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"Blob/\")) {\n let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n if (rType.name == \"Blob\")\n message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n throw new WrongResourceTypeError(message);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAoFA,IAAa,iBAAb,MAAa,eAAsD;;CAEjE,gCAAuD,IAAI,KAAK;;;CAIhE;CACA;;CAGA;CAEA,gCAAyD,IAAI,KAAK;CAElE,gCAAsD,IAAI,KAAK;CAC/D,kCAAwD,IAAI,KAAK;CAEjE;CAEA,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;AAPiB,OAAA,SAAA;AACA,OAAA,iBAAA;AACA,OAAA,aAAA;AAEA,OAAA,iBAAA;AACA,OAAA,SAAA;AACA,OAAA,MAAA;AAEjB,OAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,mBAAmB;EAExD,MAAM,WAAW,IAAI,oBAAoB,KAAK,QAAQ,KAAK,eAAe;EAC1E,MAAM,YAAY,IAAI,kBAAkB,KAAK,QAAQ,KAAK,eAAe;AACzE,OAAK,cAAc,IAAI,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,UACD;AAED,OAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,IAAI,qBAAqB;AAE7E,OAAK,UAAUA,OAAK,QAAQ,QAAQ;;CAGtC,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,IACD;AACD,QAAM,OAAO,YAAY,OAAO;AAEhC,SAAO;;CAWT,kBACE,KACA,KACqF;AACrF,MAAI,QAAQ,KAAA,EACV,QAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,CAAC;EAGnE,MAAM,QAAQ,wBAAwB,KAAK,IAAI;EAE/C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,OAAO,SAAS,CAAC;EAEzD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,SAAS;AAC5F,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,mCAAmC;AAGtD,SAAO;;CAGT,uBACE,GACA,OACA,UACoC;AACpC,mCAAiC,qBAAqB,MAAM,KAAK;EAKjE,MAAM,OAAO,KAAK,gBAAgB,OAAO,SAAS;AAClD,OAAK,OAAO,GAAG,SAAS;EAExB,MAAM,SAAS,KAAK,SAAS;AAC7B,MAAI,CAAC,OAAO,KACV;AAEF,MAAI,OAAO,OAAO,GAChB,QAAO,OAAO,OAAO;AAEvB,QAAM,OAAO,OAAO;;CAGtB,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAM,QAAQ,MAAM,GAAG;EAE7B,MAAM,eAAe,KAAK,cAAc,IAAI,IAAI;AAChD,MAAI,aACF,QAAO;EAIT,MAAM,QAAQA,OAAK,QAAQ,KAAK,SAAS,IAAI;EAE7C,MAAM,UAAU,IAAI,iBAClB,KAAK,QACL,KAAK,gBACL,OACA,eAAe,OAAO,KAAK,OAAO,EAClC,MACD;AACD,OAAK,cAAc,IAAI,KAAK,QAAQ;AAEpC,OAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,SAAS;GAC9C,4BAA4B,MAAM,CAAC,oBAAoB,EAAE;GAC1D,CAAC;AAEF,SAAO;;CAGT,MAAc,aAAa,MAAwB,UAAkB;AACnE,QAAM,KAAK,UAAU;EACrB,MAAM,OAAO,KAAK,SAAS;AAC3B,MAAI,KAAK,QAAQ,KAAK,OAAO,GAC3B,MAAK,MAAM,SAAS,MAAM,SAAS;;CAoBvC,gBACE,KACA,KACwF;AACxF,MAAI,QAAQ,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,IAAI,CAAC;EAEtF,MAAM,QAAsC,cAAc,IAAI,GAC1D,qBAAqB,KAAK,8BAA8B,IAAI,GAC5D;EAEJ,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,SAAS,CAAC;AAMpE,SAFe,KAAK,qBAAqB,OAAO,SAAS;;CAK3D,qBACE,MACA,UACyB;AACzB,mCAAiC,mBAAmB,KAAK,KAAK;EAE9D,IAAI,OAAO,KAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,CAAC;AAEnD,MAAI,SAAS,KAAA,GAAW;AACtB,UAAO,IAAI,mBAAmB,QAAQ,KAAK,EAAE,gBAAgB,MAAM,KAAK,OAAO,CAAC;AAChF,QAAK,cAAc,IAAI,QAAQ,KAAK,GAAG,EAAE,KAAK;;AAGhD,OAAK,OAAO,SAAS;AAErB,SAAO,KAAK,WAAW;;;CAIzB,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAAS,iBAAiB,QAAQ,KAAK,OAAO;AACtD,SAAO;;CAaT,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,EAAE;AACnC,MAAI,OAAO,mBAAmB,YAAY,mBAAmB,KAC3D,KAAI,WAAW,eACb,WAAU;OACL;GACL,MAAM,QAAQ;AACd,sBAAmB,OAAO,aAAa;AACvC,aAAU,EAAE,OAAO;;AAIvB,SAAO,MAAM,KAAK,eAAe;GAAE;GAAQ;GAAS,CAAC;;;;;;;CAQvD,MAAa,iBACX,QACA,SACqB;AACrB,SAAO,MAAM,KAAK,eAAe;GAC/B;GACA,SAAS,WAAW,EAAE;GACtB,mBAAmB;GACpB,CAAC;;CAGJ,MAAc,eAAe,EAC3B,QACA,SACA,oBAAoB,SAKE;EACtB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH;GACA,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,EAAE;AAC/B,eAAW,MAAM,SAAS,SAAS;AACjC,aAAQ,QAAQ,gBAAgB;AAChC,YAAO,KAAK,MAAM;;AAEpB,WAAO,OAAO,OAAO,OAAO;;GAE/B,CAAC;AAEJ,MAAI;AACF,UAAO,MAAM,SAAS;WACf,OAAO;AACd,OAAI,gBAAgB,MAAM,CACxB,QAAO,MAAM,SAAS;AAExB,SAAM;;;;CAKV,MAAa,YACX,QACA,SAIY;EACZ,MAAM,EAAE,OAAO,QAAQ,SAAS,sBAAsB;AAEtD,MAAI,kBAAkB,OAAO,CAC3B,QAAO,MAAM,gBAAgB;GAAE,MAAM,KAAK,aAAa,OAAO;GAAE;GAAO;GAAQ;GAAS,CAAC;AAG3F,MAAI,mBAAmB,OAAO,EAAE;GAC9B,MAAM,SAAS,kBAAkB,QAAQ,KAAK,OAAO;GAErD,MAAM,MAAM,QAAQ,OAAO,KAAK,GAAG;GACnC,MAAM,WAAW,oBACb,KAAA,IACA,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;IAAM,CAAC;AAC1E,WAAQ,gBAAgB;AAExB,OAAI,SAAU,QAAO,MAAM,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;IAAS,CAAC;AAEtF,UAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,QAAQ,EACV,SACA,OAAO,SAAS,SAAS;AACvB,QAAI,kBAAmB,QAAO,MAAM,QAAQ,SAAS,KAAK;IAE1D,MAAM,CAAC,eAAe,eAAe,QAAQ,KAAK;IAElD,MAAM,iBAAiB,QAAQ,eAAe,KAAK;AAC7B,WAAO,YAAY,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;KAAM,EAAE,KAAK,CAAC,CACtF,YAAY,GAGX;AAEJ,WAAO,MAAM;KAEhB;;AAGH,QAAM,IAAI,MAAM,0BAA0B;;;;;;CAO5C,qBACE,KACA,OACqC;AACrC,MAAI,MACF,oBAAmB,OAAO,uBAAuB;AAGnD,SAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,IAAI,EAAE,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAA,GACtE,CAAC,CAAC,gBAAgB;;CAcrB,YACE,KACA,OACA,KACqD;AACrD,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,IAAI,CAAC;EAExF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,SAAS;AACzF,MAAI,UAAU,KAAA,EACZ,KAAI,aAAa,wDAAwD;AAE3E,SAAO;;CAGT,iBACE,GACA,OACA,OACA,UACoB;AACpB,mCAAiC,eAAe,MAAM,KAAK;EAC3D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAE9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AAEzD,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,MAAM;AACrD,QAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AACvD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAchB,eACE,KACA,iBACA,KACqD;AACrD,MAAI,OAAO,KAAA,EACT,QAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,IAAI,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,IAAI;EAC3C,MAAM,WAAW,YAAY;AAC7B,MAAI,mBAAmB,KAAK,YAAY,GAAG,SAAS,CAAC;EAErD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,SACD;AACD,MAAI,WAAW,KAAA,EACb,KAAI,aAAa,kEAAkE;AAErF,SAAO;;CAGT,oBACE,GACA,OACA,iBACA,UACoB;AACpB,mCAAiC,kBAAkB,MAAM,KAAK;EAE9D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,SAAS;AAC5D,MAAI,QAAQ,KAAA,EAAW,QAAO,KAAA;EAC9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,OAAO;EAE3D,IAAI,YAAY,KAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,CAAC;AAE3D,MAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,gBAAgB;AAClE,QAAK,gBAAgB,IAAI,QAAQ,MAAM,GAAG,EAAE,aAAa;AAEzD,eAAY;;EAGd,MAAM,SAAS,UAAU,cAAc,EAAE;AACzC,MAAI,OAAO,MAAO,OAAM,OAAO;AAE/B,SAAO,OAAO;;CAOhB,aACE,KACA,KACyC;AACzC,MAAI,OAAO,KAAA,EAAW,QAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,IAAI,CAAC;EAElF,MAAM,IAAI,wBAAwB,KAAK,IAAI;AAE3C,SAAO,KAAK,kBAAkB,EAAsB;;CAGtD,kBAA0B,OAAuC;AAC/D,mCAAiC,gBAAgB,MAAM,KAAK;AAC5D,SAAO,aAAa,OAAO,MAAM;;CAGnC,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjC,6BAA6B,OAAO,EACpC,WACA,OAAO,eAAe,EAAE,EACxB,UACD;AAED,SAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,KAAK;GACvB,WAAW,OAAO,KAAK,UAAU;GAClC;;CAGH,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAI,QAAQ,MAAM,GAAG,CAAC;AACtD,MAAI,QAAQ,KAAA,EACV;AAGF,MAAI,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,CAAC,EAAE;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAW,QAAQ,MAAM,GAAG,EAAE,SAAS;AAEnE,SAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;AACjC,UAAM,IAAI,GAAG,WAAW,KAAK;AAE7B,SAAK,MAAM,YAAY,WAAW;AAElC,SAAK,WACH,OAAO,KAAK,eAAe,UAAU,WAAW,KAAK,CAAC,EACtD,YAAY,wBAAwB,WAAW,CAAC,oCACrB,wBAAwB,SAAS,KAAK,MAAM,EAAE,KAAK,CAAC,GAChF;KACD,CACH;aAGe,KAAK,QAAQ,IAAI,SAAS,CAExC,MAAK,WACH,MACA,YAAY,wBAAwB,KAAK,MAAM,CAAC,CAAC,yBAClD;;CAKP,WAAmB,MAAwB,QAAgB;AACzD,OAAK,MAAM,OAAO;AAClB,OAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,SAAS;AAC5E,OAAK,cAAc,OAAO,UAAU,KAAK,KAAK,CAAC;AAC/C,OAAK,cAAc,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;AACjD,OAAK,gBAAgB,OAAO,QAAQ,KAAK,MAAM,GAAG,CAAC;;CAGrD,MAAc,oBAAoB,QAA0B,UAAkB;AAE5E,MADgB,KAAK,cAAc,IAAI,QAAQ,OAAO,CAAC,EAAE,QAAQ,SAAS,IAAI,MACjE,MAAK,cAAc,OAAO,QAAQ,OAAO,CAAC;;;CAIzD,MAAM,aAAa;AACjB,OAAK,cAAc,MAAM;AAEzB,OAAK,cAAc,SAAS,MAAM,QAAQ;AACxC,QAAK,cAAc,OAAO,IAAI;AAC9B,QAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,GAAG,CAAC,WAAW;IAC7E;;CAGJ,MAAM,UAAyB;AAC7B,QAAM,KAAK,YAAY,SAAS;;CAGlC,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,SAAS;;;;AAKxB,IAAM,qBAAN,MAAyB;CACvB,UAA2B,IAAI,gBAAgB;CAE/C,YACE,MACA,QACA;AAFiB,OAAA,OAAA;AACA,OAAA,SAAA;;CAGnB,YAA4C;AAC1C,SAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM;;CAGjD,OAAc,UAAkB;AAC9B,OAAK,QAAQ,IAAI,SAAS;;CAG5B,QAAe,UAA2B;AACxC,SAAO,KAAK,QAAQ,IAAI,SAAS;;;AAIrC,IAAM,kBAAN,MAAsB;CACpB;CACA;CACA,SAAwC,IAAI,cAAc;CAC1D,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;AAHiB,OAAA,OAAA;AACA,OAAA,QAAA;AACA,OAAA,kBAAA;AAEjB,OAAK,UAAU,IAAI,QAAQ,YAAY,KAAK,QAAQ,CAAC;;CAGvD,cAAc,GAGZ;AACA,OAAK,OAAO,cAAc,EAAE;AAE5B,OAAK,QAAQ,UAAU;AAEvB,SAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;GACb;;CAGH,MAAM,SAAwB;AAC5B,MAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,gBAAgB;AAE/E,OAAI,KAAK,OAAO,QAAS,MAAK,OAAO,YAAY,YAAY,KAAK,KAAK,UAAU;AACjF,QAAK,MAAM;WACJ,GAAQ;AACf,OAAI,gBAAgB,EAAE,EAAE;AAEtB,SAAK,MAAM;AACX,SAAK,QAAQ;AACb,SAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,6BAA6B;AACjF;;AAGF,SAAM;;;;;;AAOZ,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;AAEJ,KAAI;AACF,aAAWC,KAAG,iBAAiB,MAAM;AACrC,OAAK,SAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;GAAU,CAAC;EAEvE,MAAM,QAAQ,IAAI,QAAQ;AAE1B,aAAW,MAAM,QAAQ,IAAI;AAC3B,OAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,gBAAgB,CAAE;AAErE,SAAM,KAAK,KAAK;AAChB,OAAI,MAAM,SAAS,OACjB,OAAM,OAAO;;AAKjB,SAAO,MAAM,SAAS,CAAC,KAAK,GAAG,IAAI,GAAG,GAAG;WACjC;AAER,MAAI;AACF,OAAI,GACF,IAAG,OAAO;WAEL,cAAc;AACrB,WAAQ,MAAM,qCAAqC,aAAa;;AAGlE,MAAI;AACF,OAAI,YAAY,CAAC,SAAS,UACxB,UAAS,SAAS;WAEb,cAAc;AACrB,WAAQ,MAAM,iCAAiC,aAAa;;;;AAKlE,SAAS,iCAAiC,YAAoB,OAAqB;AACjF,KAAI,CAAC,MAAM,KAAK,WAAW,QAAQ,EAAE;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;AAChE,MAAI,MAAM,QAAQ,OAChB,YAAW;AAEb,QAAM,IAAI,uBAAuB,QAAQ"}
1	+ {"version":3,"file":"download_blob.js","names":["path","fs"],"sources":["../../../src/drivers/download_blob/download_blob.ts"],"sourcesContent":["import type { ComputableCtx, ComputableStableDefined, Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource, Computable } from \"@milaboratories/computable\";\nimport type { SignedResourceId, ResourceType } from \"@milaboratories/pl-client\";\nimport {\n isNotFoundError,\n resourceIdToString,\n stringifyWithResourceId,\n} from \"@milaboratories/pl-client\";\nimport type {\n AnyLogHandle,\n BlobDriver,\n BlobDriverMetrics,\n ContentHandler,\n GetContentOptions,\n LocalBlobHandle,\n LocalBlobHandleAndSize,\n ReadyLogHandle,\n RemoteBlobHandle,\n RemoteBlobHandleAndSize,\n StreamingApiResponse,\n} from \"@milaboratories/pl-model-common\";\nimport { type RangeBytes, validateRangeBytes } from \"@milaboratories/pl-model-common\";\nimport type { PlTreeEntry, ResourceInfo, ResourceSnapshot } from \"@milaboratories/pl-tree\";\nimport {\n isPlTreeEntry,\n makeResourceSnapshot,\n treeEntryToResourceInfo,\n} from \"@milaboratories/pl-tree\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport { CallersCounter, mapGet, TaskProcessor } from \"@milaboratories/ts-helpers\";\nimport Denque from \"denque\";\nimport * as fs from \"fs\";\nimport { randomUUID } from \"node:crypto\";\nimport * as fsp from \"node:fs/promises\";\nimport * as os from \"node:os\";\nimport * as path from \"node:path\";\nimport * as readline from \"node:readline/promises\";\nimport { buffer } from \"node:stream/consumers\";\nimport type { ClientDownload } from \"../../clients/download\";\nimport type { ClientLogs } from \"../../clients/logs\";\nimport { withFileContent } from \"../helpers/read_file\";\nimport {\n isLocalBlobHandle,\n newLocalHandle,\n parseLocalHandle,\n} from \"../helpers/download_local_handle\";\nimport {\n isRemoteBlobHandle,\n newRemoteHandle,\n parseRemoteHandle,\n} from \"../helpers/download_remote_handle\";\nimport { Updater, WrongResourceTypeError } from \"../helpers/helpers\";\nimport { getResourceInfoFromLogHandle, newLogHandle } from \"../helpers/logs_handle\";\nimport { getSize, OnDemandBlobResourceSnapshot } from \"../types\";\nimport { blobKey, pathToKey } from \"./blob_key\";\nimport { DownloadBlobTask, nonRecoverableError } from \"./download_blob_task\";\nimport { FilesCache } from \"../helpers/files_cache\";\nimport { SparseCache, SparseCacheFsFile, SparseCacheFsRanges } from \"./sparse_cache/cache\";\nimport { isOffByOneError } from \"../../helpers/download_errors\";\n\nexport type DownloadDriverOps = {\n /*\n A soft limit of the amount of blob storage, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one\n * when they become unneeded.\n * /\n cacheSoftSizeBytes: number;\n\n /\n A hard limit of the amount of sparse cache, in bytes.\n * Once exceeded, the download driver will start deleting blobs one by one.\n \n The sparse cache is used to store ranges of blobs.\n * /\n rangesCacheMaxSizeBytes: number;\n\n /\n Max number of concurrent downloads while calculating computable states\n * derived from this driver\n * /\n nConcurrentDownloads: number;\n};\n\n/* DownloadDriver holds a queue of downloading tasks,\n * and notifies every watcher when a file were downloaded. /\nexport class DownloadDriver implements BlobDriver, AsyncDisposable {\n /* Represents a unique key to the path of a blob as a map. /\n private keyToDownload: Map<string, DownloadBlobTask> = new Map();\n\n /* Writes and removes files to a hard drive and holds a counter for every\n * file that should be kept. /\n private cache: FilesCache<DownloadBlobTask>;\n private rangesCache: SparseCache;\n\n /* Downloads files and writes them to the local dir. /\n private downloadQueue: TaskProcessor;\n\n private keyToOnDemand: Map<string, OnDemandBlobHolder> = new Map();\n\n private idToLastLines: Map<string, LastLinesGetter> = new Map();\n private idToProgressLog: Map<string, LastLinesGetter> = new Map();\n\n private readonly saveDir: string;\n\n /* Downloads that bypassed the ranges cache; counted when issued. /\n private uncachedRequests = 0;\n private uncachedRequestBytes = 0;\n\n constructor(\n private readonly logger: MiLogger,\n private readonly clientDownload: ClientDownload,\n private readonly clientLogs: ClientLogs,\n saveDir: string,\n private readonly rangesCacheDir: string,\n private readonly signer: Signer,\n private readonly ops: DownloadDriverOps,\n ) {\n this.cache = new FilesCache(this.ops.cacheSoftSizeBytes);\n\n const fsRanges = new SparseCacheFsRanges(this.logger, this.rangesCacheDir);\n const fsStorage = new SparseCacheFsFile(this.logger, this.rangesCacheDir);\n this.rangesCache = new SparseCache(\n this.logger,\n this.ops.rangesCacheMaxSizeBytes,\n fsRanges,\n fsStorage,\n );\n\n this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);\n\n this.saveDir = path.resolve(saveDir);\n }\n\n static async init(\n logger: MiLogger,\n clientDownload: ClientDownload,\n clientLogs: ClientLogs,\n saveDir: string,\n rangesCacheDir: string,\n signer: Signer,\n ops: DownloadDriverOps,\n ): Promise<DownloadDriver> {\n const driver = new DownloadDriver(\n logger,\n clientDownload,\n clientLogs,\n saveDir,\n rangesCacheDir,\n signer,\n ops,\n );\n await driver.rangesCache.reset();\n\n return driver;\n }\n\n /* Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. /\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx: ComputableCtx,\n ): LocalBlobHandleAndSize \| undefined;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ): ComputableStableDefined<LocalBlobHandleAndSize>;\n public getDownloadedBlob(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<LocalBlobHandleAndSize \| undefined> \| LocalBlobHandleAndSize \| undefined {\n if (ctx === undefined) {\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));\n }\n\n const rInfo = treeEntryToResourceInfo(res, ctx);\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(rInfo, callerId));\n\n const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);\n if (result == undefined) {\n ctx.markUnstable(\"download blob is still undefined\");\n }\n\n return result;\n }\n\n private getDownloadedBlobNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n callerId: string,\n ): LocalBlobHandleAndSize \| undefined {\n validateDownloadableResourceType(\"getDownloadedBlob\", rInfo.type);\n\n // We don't need to request files with wider limits,\n // PFrame's engine does it disk-optimally by itself.\n\n const task = this.getOrSetNewTask(rInfo, callerId);\n task.attach(w, callerId);\n\n const result = task.getBlob();\n if (!result.done) {\n return undefined;\n }\n if (result.result.ok) {\n return result.result.value;\n }\n throw result.result.error;\n }\n\n private getOrSetNewTask(rInfo: ResourceSnapshot, callerId: string): DownloadBlobTask {\n const key = blobKey(rInfo.id);\n\n const inMemoryTask = this.keyToDownload.get(key);\n if (inMemoryTask) {\n return inMemoryTask;\n }\n\n // schedule the blob downloading, then it'll be added to the cache.\n const fPath = path.resolve(this.saveDir, key);\n\n const newTask = new DownloadBlobTask(\n this.logger,\n this.clientDownload,\n rInfo,\n newLocalHandle(fPath, this.signer),\n fPath,\n );\n this.keyToDownload.set(key, newTask);\n\n this.downloadQueue.push({\n fn: () => this.downloadBlob(newTask, callerId),\n recoverableErrorPredicate: (e) => !nonRecoverableError(e),\n });\n\n return newTask;\n }\n\n private async downloadBlob(task: DownloadBlobTask, callerId: string) {\n await task.download();\n const blob = task.getBlob();\n if (blob.done && blob.result.ok) {\n this.cache.addCache(task, callerId);\n }\n }\n\n /* Gets on demand blob. /\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: undefined,\n fromBytes?: number,\n toBytes?: number,\n ): Computable<RemoteBlobHandleAndSize>;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx: ComputableCtx,\n fromBytes?: number,\n toBytes?: number,\n ): RemoteBlobHandleAndSize;\n public getOnDemandBlob(\n res: OnDemandBlobResourceSnapshot \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): ComputableStableDefined<RemoteBlobHandleAndSize> \| RemoteBlobHandleAndSize \| undefined {\n if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));\n\n const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)\n ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)\n : res;\n\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));\n\n // note that the watcher is not needed,\n // the handler never changes.\n const result = this.getOnDemandBlobNoCtx(rInfo, callerId);\n\n return result;\n }\n\n private getOnDemandBlobNoCtx(\n info: OnDemandBlobResourceSnapshot,\n callerId: string,\n ): RemoteBlobHandleAndSize {\n validateDownloadableResourceType(\"getOnDemandBlob\", info.type);\n\n let blob = this.keyToOnDemand.get(blobKey(info.id));\n\n if (blob === undefined) {\n blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));\n this.keyToOnDemand.set(blobKey(info.id), blob);\n }\n\n blob.attach(callerId);\n\n return blob.getHandle();\n }\n\n /* Gets a path from a handle. /\n public getLocalPath(handle: LocalBlobHandle): string {\n const { path } = parseLocalHandle(handle, this.signer);\n return path;\n }\n\n /* Gets a content of a blob by a handle. /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array>;\n /* @deprecated Use {@link getContent} with {@link GetContentOptions} instead /\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n range?: RangeBytes,\n ): Promise<Uint8Array>;\n public async getContent(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n optionsOrRange?: GetContentOptions \| RangeBytes,\n ): Promise<Uint8Array> {\n let options: GetContentOptions = {};\n if (typeof optionsOrRange === \"object\" && optionsOrRange !== null) {\n if (\"range\" in optionsOrRange) {\n options = optionsOrRange;\n } else {\n const range = optionsOrRange as RangeBytes;\n validateRangeBytes(range, `getContent`);\n options = { range };\n }\n }\n\n return await this.getContentImpl({ handle, options });\n }\n\n /\n Same as {@link getContent}, but bypasses the ranges cache entirely (no read, no write).\n * For local handles this is identical to {@link getContent}, since local content never\n * uses the ranges cache.\n /\n public async getContentDirect(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options?: GetContentOptions,\n ): Promise<Uint8Array> {\n return await this.getContentImpl({\n handle,\n options: options ?? {},\n bypassRangesCache: true,\n });\n }\n\n /\n Operational metrics for a monitoring panel. Serv cache metrics are reported separately\n * (different owner) — the panel composes both.\n /\n public getMetrics(): BlobDriverMetrics {\n return {\n uncachedRequests: this.uncachedRequests,\n uncachedRequestBytes: this.uncachedRequestBytes,\n ...this.clientDownload.metrics(),\n };\n }\n\n private async getContentImpl({\n handle,\n options,\n bypassRangesCache = false,\n }: {\n handle: LocalBlobHandle \| RemoteBlobHandle;\n options: GetContentOptions;\n bypassRangesCache?: boolean;\n }): Promise<Uint8Array> {\n const request = () =>\n this.withContent(handle, {\n ...options,\n bypassRangesCache,\n handler: async (content) => {\n const chunks: Uint8Array[] = [];\n for await (const chunk of content) {\n options.signal?.throwIfAborted();\n chunks.push(chunk);\n }\n return Buffer.concat(chunks);\n },\n });\n\n try {\n return await request();\n } catch (error) {\n if (isOffByOneError(error)) {\n return await request();\n }\n throw error;\n }\n }\n\n /* Gets a content stream of a blob by a handle and calls handler with it. /\n public async withContent<T>(\n handle: LocalBlobHandle \| RemoteBlobHandle,\n options: GetContentOptions & {\n handler: ContentHandler<T>;\n bypassRangesCache?: boolean;\n },\n ): Promise<T> {\n const { range, signal, handler, bypassRangesCache } = options;\n\n if (isLocalBlobHandle(handle)) {\n return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });\n }\n\n if (isRemoteBlobHandle(handle)) {\n const result = parseRemoteHandle(handle, this.signer);\n\n const key = blobKey(result.info.id);\n const filePath = bypassRangesCache\n ? undefined\n : await this.rangesCache.get(key, range ?? { from: 0, to: result.size });\n signal?.throwIfAborted();\n\n if (filePath) return await withFileContent({ path: filePath, range, signal, handler });\n\n if (bypassRangesCache) this.uncachedRequests++;\n\n return await this.clientDownload.withBlobContent(\n result.info,\n { signal },\n options,\n async (content, size) => {\n if (bypassRangesCache) {\n const counted = content.pipeThrough(\n new TransformStream<Uint8Array, Uint8Array>({\n transform: (chunk, controller) => {\n this.uncachedRequestBytes += chunk.byteLength;\n controller.enqueue(chunk);\n },\n }),\n );\n return await handler(counted, size);\n }\n\n const [handlerStream, cacheStream] = content.tee();\n\n const handlerPromise = handler(handlerStream, size);\n const _cachePromise = buffer(cacheStream)\n .then((data) => this.rangesCache.set(key, range ?? { from: 0, to: result.size }, data))\n .catch(() => {\n // Ignore cache errors - they shouldn't affect the main handler result\n // This prevents unhandled promise rejections when the stream fails\n });\n\n return await handlerPromise;\n },\n );\n }\n\n throw new Error(\"Malformed remote handle\");\n }\n\n /\n Creates computable that will return blob content once it is downloaded.\n * Uses downloaded blob handle under the hood, so stores corresponding blob in file system.\n /\n public getComputableContent(\n res: ResourceInfo \| PlTreeEntry,\n range?: RangeBytes,\n ): ComputableStableDefined<Uint8Array> {\n if (range) {\n validateRangeBytes(range, `getComputableContent`);\n }\n\n return Computable.make((ctx) => this.getDownloadedBlob(res, ctx), {\n postprocessValue: (v) => (v ? this.getContent(v.handle, { range }) : undefined),\n }).withStableType();\n }\n\n /* Returns all logs and schedules a job that reads remain logs.\n * Notifies when a new portion of the log appeared. /\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx: ComputableCtx,\n ): Computable<string \| undefined>;\n public getLastLogs(\n res: ResourceInfo \| PlTreeEntry,\n lines: number,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);\n if (result == undefined)\n ctx.markUnstable(\"either a file was not downloaded or logs was not read\");\n\n return result;\n }\n\n private getLastLogsNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n lines: number,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getLastLogs\", rInfo.type);\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToLastLines.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, lines);\n this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns a last line that has patternToSearch.\n * Notifies when a new line appeared or EOF reached. /\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ): Computable<string \| undefined>;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx: ComputableCtx,\n ): string \| undefined;\n public getProgressLog(\n res: ResourceInfo \| PlTreeEntry,\n patternToSearch: string,\n ctx?: ComputableCtx,\n ): Computable<string \| undefined> \| string \| undefined {\n if (ctx == undefined)\n return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n const callerId = randomUUID();\n ctx.addOnDestroy(() => this.releaseBlob(r, callerId));\n\n const result = this.getProgressLogNoCtx(\n ctx.watcher,\n r as ResourceSnapshot,\n patternToSearch,\n callerId,\n );\n if (result === undefined)\n ctx.markUnstable(\"either a file was not downloaded or a progress log was not read\");\n\n return result;\n }\n\n private getProgressLogNoCtx(\n w: Watcher,\n rInfo: ResourceSnapshot,\n patternToSearch: string,\n callerId: string,\n ): string \| undefined {\n validateDownloadableResourceType(\"getProgressLog\", rInfo.type);\n\n const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);\n if (blob == undefined) return undefined;\n const { path } = parseLocalHandle(blob.handle, this.signer);\n\n let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));\n\n if (logGetter == undefined) {\n const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);\n this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);\n\n logGetter = newLogGetter;\n }\n\n const result = logGetter.getOrSchedule(w);\n if (result.error) throw result.error;\n\n return result.log;\n }\n\n /* Returns an Id of a smart object, that can read logs directly from\n * the platform. /\n public getLogHandle(res: ResourceInfo \| PlTreeEntry): Computable<AnyLogHandle>;\n public getLogHandle(res: ResourceInfo \| PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;\n public getLogHandle(\n res: ResourceInfo \| PlTreeEntry,\n ctx?: ComputableCtx,\n ): Computable<AnyLogHandle> \| AnyLogHandle {\n if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));\n\n const r = treeEntryToResourceInfo(res, ctx);\n\n return this.getLogHandleNoCtx(r as ResourceSnapshot);\n }\n\n private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {\n validateDownloadableResourceType(\"getLogHandle\", rInfo.type);\n return newLogHandle(false, rInfo);\n }\n\n public async lastLines(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number, // if 0n, then start from the end.\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.lastLines(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n public async readText(\n handle: ReadyLogHandle,\n lineCount: number,\n offsetBytes?: number,\n searchStr?: string,\n ): Promise<StreamingApiResponse> {\n const resp = await this.clientLogs.readText(\n getResourceInfoFromLogHandle(handle),\n lineCount,\n BigInt(offsetBytes ?? 0),\n searchStr,\n );\n\n return {\n live: false,\n shouldUpdateHandle: false,\n data: resp.data,\n size: Number(resp.size),\n newOffset: Number(resp.newOffset),\n };\n }\n\n private async releaseBlob(rInfo: ResourceInfo, callerId: string) {\n const task = this.keyToDownload.get(blobKey(rInfo.id));\n if (task == undefined) {\n return;\n }\n\n if (this.cache.existsFile(blobKey(rInfo.id))) {\n const toDelete = this.cache.removeFile(blobKey(rInfo.id), callerId);\n\n await Promise.all(\n toDelete.map(async (cachedFile) => {\n await fsp.rm(cachedFile.path);\n\n this.cache.removeCache(cachedFile);\n\n this.removeTask(\n mapGet(this.keyToDownload, pathToKey(cachedFile.path)),\n `the task ${stringifyWithResourceId(cachedFile)} was removed` +\n `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.path))}`,\n );\n }),\n );\n } else {\n // The task is still in a downloading queue.\n const deleted = task.counter.dec(callerId);\n if (deleted) {\n this.removeTask(\n task,\n `the task ${stringifyWithResourceId(task.info())} was removed from cache`,\n );\n }\n }\n }\n\n private removeTask(task: DownloadBlobTask, reason: string) {\n task.abort(reason);\n task.change.markChanged(`download task for ${task.path} removed: ${reason}`);\n this.keyToDownload.delete(pathToKey(task.path));\n this.idToLastLines.delete(blobKey(task.rInfo.id));\n this.idToProgressLog.delete(blobKey(task.rInfo.id));\n }\n\n private async releaseOnDemandBlob(blobId: SignedResourceId, callerId: string) {\n const deleted = this.keyToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;\n if (deleted) this.keyToOnDemand.delete(blobKey(blobId));\n }\n\n /* Removes all files from a hard drive. /\n async releaseAll() {\n this.downloadQueue.stop();\n\n this.keyToDownload.forEach((task, key) => {\n this.keyToDownload.delete(key);\n task.change.markChanged(`task ${resourceIdToString(task.rInfo.id)} released`);\n });\n }\n\n async dispose(): Promise<void> {\n await this.rangesCache.dispose();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.dispose();\n }\n}\n\n/* Keeps a counter to the on demand handle. /\nclass OnDemandBlobHolder {\n private readonly counter = new CallersCounter();\n\n constructor(\n private readonly size: number,\n private readonly handle: RemoteBlobHandle,\n ) {}\n\n public getHandle(): RemoteBlobHandleAndSize {\n return { handle: this.handle, size: this.size };\n }\n\n public attach(callerId: string) {\n this.counter.inc(callerId);\n }\n\n public release(callerId: string): boolean {\n return this.counter.dec(callerId);\n }\n}\n\nclass LastLinesGetter {\n private updater: Updater;\n private log: string \| undefined;\n private readonly change: ChangeSource = new ChangeSource();\n private error: any \| undefined = undefined;\n\n constructor(\n private readonly path: string,\n private readonly lines: number,\n private readonly patternToSearch?: string,\n ) {\n this.updater = new Updater(async () => this.update());\n }\n\n getOrSchedule(w: Watcher): {\n log: string \| undefined;\n error?: any \| undefined;\n } {\n this.change.attachWatcher(w);\n\n this.updater.schedule();\n\n return {\n log: this.log,\n error: this.error,\n };\n }\n\n async update(): Promise<void> {\n try {\n const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);\n\n if (this.log != newLogs) this.change.markChanged(`logs for ${this.path} updated`);\n this.log = newLogs;\n } catch (e: any) {\n if (isNotFoundError(e)) {\n // No resource\n this.log = \"\";\n this.error = e;\n this.change.markChanged(`log update for ${this.path} failed, resource not found`);\n return;\n }\n\n throw e;\n }\n }\n}\n\n/* Gets last lines from a file by reading the file from the top and keeping\n * last N lines in a window queue. */\nasync function getLastLines(\n fPath: string,\n nLines: number,\n patternToSearch?: string,\n): Promise<string> {\n let inStream: fs.ReadStream \| undefined;\n let rl: readline.Interface \| undefined;\n\n try {\n inStream = fs.createReadStream(fPath);\n rl = readline.createInterface({ input: inStream, crlfDelay: Infinity });\n\n const lines = new Denque();\n\n for await (const line of rl) {\n if (patternToSearch != undefined && !line.includes(patternToSearch)) continue;\n\n lines.push(line);\n if (lines.length > nLines) {\n lines.shift();\n }\n }\n\n // last EOL is for keeping backward compat with platforma implementation.\n return lines.toArray().join(os.EOL) + os.EOL;\n } finally {\n // Cleanup resources in finally block to ensure they're always cleaned up\n try {\n if (rl) {\n rl.close();\n }\n } catch (cleanupError) {\n console.error(\"Error closing readline interface:\", cleanupError);\n }\n\n try {\n if (inStream && !inStream.destroyed) {\n inStream.destroy();\n }\n } catch (cleanupError) {\n console.error(\"Error destroying read stream:\", cleanupError);\n }\n }\n}\n\nfunction validateDownloadableResourceType(methodName: string, rType: ResourceType) {\n if (!rType.name.startsWith(\"Blob/\")) {\n let message = `${methodName}: wrong resource type: ${rType.name}, expected: a resource of type that starts with 'Blob/'.`;\n if (rType.name == \"Blob\")\n message += ` If it's called from workflow, should a file be exported with 'file.exportFile' function?`;\n\n throw new WrongResourceTypeError(message);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAqFA,IAAa,iBAAb,MAAa,eAAsD;CAwB9C;CACA;CACA;CAEA;CACA;CACA;;CA5BnB,gCAAuD,IAAI,IAAI;;;CAI/D;CACA;;CAGA;CAEA,gCAAyD,IAAI,IAAI;CAEjE,gCAAsD,IAAI,IAAI;CAC9D,kCAAwD,IAAI,IAAI;CAEhE;;CAGA,mBAA2B;CAC3B,uBAA+B;CAE/B,YACE,QACA,gBACA,YACA,SACA,gBACA,QACA,KACA;EAPiB,KAAA,SAAA;EACA,KAAA,iBAAA;EACA,KAAA,aAAA;EAEA,KAAA,iBAAA;EACA,KAAA,SAAA;EACA,KAAA,MAAA;EAEjB,KAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,kBAAkB;EAEvD,MAAM,WAAW,IAAI,oBAAoB,KAAK,QAAQ,KAAK,cAAc;EACzE,MAAM,YAAY,IAAI,kBAAkB,KAAK,QAAQ,KAAK,cAAc;EACxE,KAAK,cAAc,IAAI,YACrB,KAAK,QACL,KAAK,IAAI,yBACT,UACA,SACF;EAEA,KAAK,gBAAgB,IAAI,cAAc,KAAK,QAAQ,IAAI,oBAAoB;EAE5E,KAAK,UAAUA,OAAK,QAAQ,OAAO;CACrC;CAEA,aAAa,KACX,QACA,gBACA,YACA,SACA,gBACA,QACA,KACyB;EACzB,MAAM,SAAS,IAAI,eACjB,QACA,gBACA,YACA,SACA,gBACA,QACA,GACF;EACA,MAAM,OAAO,YAAY,MAAM;EAE/B,OAAO;CACT;CAUA,kBACE,KACA,KACqF;EACrF,IAAI,QAAQ,KAAA,GACV,OAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,GAAG,CAAC;EAGlE,MAAM,QAAQ,wBAAwB,KAAK,GAAG;EAE9C,MAAM,WAAW,WAAW;EAC5B,IAAI,mBAAmB,KAAK,YAAY,OAAO,QAAQ,CAAC;EAExD,MAAM,SAAS,KAAK,uBAAuB,IAAI,SAAS,OAA2B,QAAQ;EAC3F,IAAI,UAAU,KAAA,GACZ,IAAI,aAAa,kCAAkC;EAGrD,OAAO;CACT;CAEA,uBACE,GACA,OACA,UACoC;EACpC,iCAAiC,qBAAqB,MAAM,IAAI;EAKhE,MAAM,OAAO,KAAK,gBAAgB,OAAO,QAAQ;EACjD,KAAK,OAAO,GAAG,QAAQ;EAEvB,MAAM,SAAS,KAAK,QAAQ;EAC5B,IAAI,CAAC,OAAO,MACV;EAEF,IAAI,OAAO,OAAO,IAChB,OAAO,OAAO,OAAO;EAEvB,MAAM,OAAO,OAAO;CACtB;CAEA,gBAAwB,OAAyB,UAAoC;EACnF,MAAM,MAAM,QAAQ,MAAM,EAAE;EAE5B,MAAM,eAAe,KAAK,cAAc,IAAI,GAAG;EAC/C,IAAI,cACF,OAAO;EAIT,MAAM,QAAQA,OAAK,QAAQ,KAAK,SAAS,GAAG;EAE5C,MAAM,UAAU,IAAI,iBAClB,KAAK,QACL,KAAK,gBACL,OACA,eAAe,OAAO,KAAK,MAAM,GACjC,KACF;EACA,KAAK,cAAc,IAAI,KAAK,OAAO;EAEnC,KAAK,cAAc,KAAK;GACtB,UAAU,KAAK,aAAa,SAAS,QAAQ;GAC7C,4BAA4B,MAAM,CAAC,oBAAoB,CAAC;EAC1D,CAAC;EAED,OAAO;CACT;CAEA,MAAc,aAAa,MAAwB,UAAkB;EACnE,MAAM,KAAK,SAAS;EACpB,MAAM,OAAO,KAAK,QAAQ;EAC1B,IAAI,KAAK,QAAQ,KAAK,OAAO,IAC3B,KAAK,MAAM,SAAS,MAAM,QAAQ;CAEtC;CAkBA,gBACE,KACA,KACwF;EACxF,IAAI,QAAQ,KAAA,GAAW,OAAO,WAAW,MAAM,QAAQ,KAAK,gBAAgB,KAAK,GAAG,CAAC;EAErF,MAAM,QAAsC,cAAc,GAAG,IACzD,qBAAqB,KAAK,8BAA8B,GAAG,IAC3D;EAEJ,MAAM,WAAW,WAAW;EAC5B,IAAI,mBAAmB,KAAK,oBAAoB,MAAM,IAAI,QAAQ,CAAC;EAMnE,OAFe,KAAK,qBAAqB,OAAO,QAEpC;CACd;CAEA,qBACE,MACA,UACyB;EACzB,iCAAiC,mBAAmB,KAAK,IAAI;EAE7D,IAAI,OAAO,KAAK,cAAc,IAAI,QAAQ,KAAK,EAAE,CAAC;EAElD,IAAI,SAAS,KAAA,GAAW;GACtB,OAAO,IAAI,mBAAmB,QAAQ,IAAI,GAAG,gBAAgB,MAAM,KAAK,MAAM,CAAC;GAC/E,KAAK,cAAc,IAAI,QAAQ,KAAK,EAAE,GAAG,IAAI;EAC/C;EAEA,KAAK,OAAO,QAAQ;EAEpB,OAAO,KAAK,UAAU;CACxB;;CAGA,aAAoB,QAAiC;EACnD,MAAM,EAAE,SAAS,iBAAiB,QAAQ,KAAK,MAAM;EACrD,OAAO;CACT;CAYA,MAAa,WACX,QACA,gBACqB;EACrB,IAAI,UAA6B,CAAC;EAClC,IAAI,OAAO,mBAAmB,YAAY,mBAAmB,MAC3D,IAAI,WAAW,gBACb,UAAU;OACL;GACL,MAAM,QAAQ;GACd,mBAAmB,OAAO,YAAY;GACtC,UAAU,EAAE,MAAM;EACpB;EAGF,OAAO,MAAM,KAAK,eAAe;GAAE;GAAQ;EAAQ,CAAC;CACtD;;;;;;CAOA,MAAa,iBACX,QACA,SACqB;EACrB,OAAO,MAAM,KAAK,eAAe;GAC/B;GACA,SAAS,WAAW,CAAC;GACrB,mBAAmB;EACrB,CAAC;CACH;;;;;CAMA,aAAuC;EACrC,OAAO;GACL,kBAAkB,KAAK;GACvB,sBAAsB,KAAK;GAC3B,GAAG,KAAK,eAAe,QAAQ;EACjC;CACF;CAEA,MAAc,eAAe,EAC3B,QACA,SACA,oBAAoB,SAKE;EACtB,MAAM,gBACJ,KAAK,YAAY,QAAQ;GACvB,GAAG;GACH;GACA,SAAS,OAAO,YAAY;IAC1B,MAAM,SAAuB,CAAC;IAC9B,WAAW,MAAM,SAAS,SAAS;KACjC,QAAQ,QAAQ,eAAe;KAC/B,OAAO,KAAK,KAAK;IACnB;IACA,OAAO,OAAO,OAAO,MAAM;GAC7B;EACF,CAAC;EAEH,IAAI;GACF,OAAO,MAAM,QAAQ;EACvB,SAAS,OAAO;GACd,IAAI,gBAAgB,KAAK,GACvB,OAAO,MAAM,QAAQ;GAEvB,MAAM;EACR;CACF;;CAGA,MAAa,YACX,QACA,SAIY;EACZ,MAAM,EAAE,OAAO,QAAQ,SAAS,sBAAsB;EAEtD,IAAI,kBAAkB,MAAM,GAC1B,OAAO,MAAM,gBAAgB;GAAE,MAAM,KAAK,aAAa,MAAM;GAAG;GAAO;GAAQ;EAAQ,CAAC;EAG1F,IAAI,mBAAmB,MAAM,GAAG;GAC9B,MAAM,SAAS,kBAAkB,QAAQ,KAAK,MAAM;GAEpD,MAAM,MAAM,QAAQ,OAAO,KAAK,EAAE;GAClC,MAAM,WAAW,oBACb,KAAA,IACA,MAAM,KAAK,YAAY,IAAI,KAAK,SAAS;IAAE,MAAM;IAAG,IAAI,OAAO;GAAK,CAAC;GACzE,QAAQ,eAAe;GAEvB,IAAI,UAAU,OAAO,MAAM,gBAAgB;IAAE,MAAM;IAAU;IAAO;IAAQ;GAAQ,CAAC;GAErF,IAAI,mBAAmB,KAAK;GAE5B,OAAO,MAAM,KAAK,eAAe,gBAC/B,OAAO,MACP,EAAE,OAAO,GACT,SACA,OAAO,SAAS,SAAS;IACvB,IAAI,mBASF,OAAO,MAAM,QARG,QAAQ,YACtB,IAAI,gBAAwC,EAC1C,YAAY,OAAO,eAAe;KAChC,KAAK,wBAAwB,MAAM;KACnC,WAAW,QAAQ,KAAK;IAC1B,EACF,CAAC,CAEwB,GAAG,IAAI;IAGpC,MAAM,CAAC,eAAe,eAAe,QAAQ,IAAI;IAEjD,MAAM,iBAAiB,QAAQ,eAAe,IAAI;IAC5B,OAAO,WAAW,CAAC,CACtC,MAAM,SAAS,KAAK,YAAY,IAAI,KAAK,SAAS;KAAE,MAAM;KAAG,IAAI,OAAO;IAAK,GAAG,IAAI,CAAC,CAAC,CACtF,YAAY,CAGb,CAAC;IAEH,OAAO,MAAM;GACf,CACF;EACF;EAEA,MAAM,IAAI,MAAM,yBAAyB;CAC3C;;;;;CAMA,qBACE,KACA,OACqC;EACrC,IAAI,OACF,mBAAmB,OAAO,sBAAsB;EAGlD,OAAO,WAAW,MAAM,QAAQ,KAAK,kBAAkB,KAAK,GAAG,GAAG,EAChE,mBAAmB,MAAO,IAAI,KAAK,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,KAAA,EACvE,CAAC,CAAC,CAAC,eAAe;CACpB;CAaA,YACE,KACA,OACA,KACqD;EACrD,IAAI,OAAO,KAAA,GAAW,OAAO,WAAW,MAAM,QAAQ,KAAK,YAAY,KAAK,OAAO,GAAG,CAAC;EAEvF,MAAM,IAAI,wBAAwB,KAAK,GAAG;EAC1C,MAAM,WAAW,WAAW;EAC5B,IAAI,mBAAmB,KAAK,YAAY,GAAG,QAAQ,CAAC;EAEpD,MAAM,SAAS,KAAK,iBAAiB,IAAI,SAAS,GAAuB,OAAO,QAAQ;EACxF,IAAI,UAAU,KAAA,GACZ,IAAI,aAAa,uDAAuD;EAE1E,OAAO;CACT;CAEA,iBACE,GACA,OACA,OACA,UACoB;EACpB,iCAAiC,eAAe,MAAM,IAAI;EAC1D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,QAAQ;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAE9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,MAAM;EAE1D,IAAI,YAAY,KAAK,cAAc,IAAI,QAAQ,MAAM,EAAE,CAAC;EAExD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,KAAK;GACpD,KAAK,cAAc,IAAI,QAAQ,MAAM,EAAE,GAAG,YAAY;GACtD,YAAY;EACd;EAEA,MAAM,SAAS,UAAU,cAAc,CAAC;EACxC,IAAI,OAAO,OAAO,MAAM,OAAO;EAE/B,OAAO,OAAO;CAChB;CAaA,eACE,KACA,iBACA,KACqD;EACrD,IAAI,OAAO,KAAA,GACT,OAAO,WAAW,MAAM,QAAQ,KAAK,eAAe,KAAK,iBAAiB,GAAG,CAAC;EAEhF,MAAM,IAAI,wBAAwB,KAAK,GAAG;EAC1C,MAAM,WAAW,WAAW;EAC5B,IAAI,mBAAmB,KAAK,YAAY,GAAG,QAAQ,CAAC;EAEpD,MAAM,SAAS,KAAK,oBAClB,IAAI,SACJ,GACA,iBACA,QACF;EACA,IAAI,WAAW,KAAA,GACb,IAAI,aAAa,iEAAiE;EAEpF,OAAO;CACT;CAEA,oBACE,GACA,OACA,iBACA,UACoB;EACpB,iCAAiC,kBAAkB,MAAM,IAAI;EAE7D,MAAM,OAAO,KAAK,uBAAuB,GAAG,OAAO,QAAQ;EAC3D,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,MAAM,EAAE,SAAS,iBAAiB,KAAK,QAAQ,KAAK,MAAM;EAE1D,IAAI,YAAY,KAAK,gBAAgB,IAAI,QAAQ,MAAM,EAAE,CAAC;EAE1D,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,eAAe,IAAI,gBAAgB,MAAM,GAAG,eAAe;GACjE,KAAK,gBAAgB,IAAI,QAAQ,MAAM,EAAE,GAAG,YAAY;GAExD,YAAY;EACd;EAEA,MAAM,SAAS,UAAU,cAAc,CAAC;EACxC,IAAI,OAAO,OAAO,MAAM,OAAO;EAE/B,OAAO,OAAO;CAChB;CAMA,aACE,KACA,KACyC;EACzC,IAAI,OAAO,KAAA,GAAW,OAAO,WAAW,MAAM,QAAQ,KAAK,aAAa,KAAK,GAAG,CAAC;EAEjF,MAAM,IAAI,wBAAwB,KAAK,GAAG;EAE1C,OAAO,KAAK,kBAAkB,CAAqB;CACrD;CAEA,kBAA0B,OAAuC;EAC/D,iCAAiC,gBAAgB,MAAM,IAAI;EAC3D,OAAO,aAAa,OAAO,KAAK;CAClC;CAEA,MAAa,UACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,UACjC,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF;EAEA,OAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,IAAI;GACtB,WAAW,OAAO,KAAK,SAAS;EAClC;CACF;CAEA,MAAa,SACX,QACA,WACA,aACA,WAC+B;EAC/B,MAAM,OAAO,MAAM,KAAK,WAAW,SACjC,6BAA6B,MAAM,GACnC,WACA,OAAO,eAAe,CAAC,GACvB,SACF;EAEA,OAAO;GACL,MAAM;GACN,oBAAoB;GACpB,MAAM,KAAK;GACX,MAAM,OAAO,KAAK,IAAI;GACtB,WAAW,OAAO,KAAK,SAAS;EAClC;CACF;CAEA,MAAc,YAAY,OAAqB,UAAkB;EAC/D,MAAM,OAAO,KAAK,cAAc,IAAI,QAAQ,MAAM,EAAE,CAAC;EACrD,IAAI,QAAQ,KAAA,GACV;EAGF,IAAI,KAAK,MAAM,WAAW,QAAQ,MAAM,EAAE,CAAC,GAAG;GAC5C,MAAM,WAAW,KAAK,MAAM,WAAW,QAAQ,MAAM,EAAE,GAAG,QAAQ;GAElE,MAAM,QAAQ,IACZ,SAAS,IAAI,OAAO,eAAe;IACjC,MAAM,IAAI,GAAG,WAAW,IAAI;IAE5B,KAAK,MAAM,YAAY,UAAU;IAEjC,KAAK,WACH,OAAO,KAAK,eAAe,UAAU,WAAW,IAAI,CAAC,GACrD,YAAY,wBAAwB,UAAU,EAAE,oCACrB,wBAAwB,SAAS,KAAK,MAAM,EAAE,IAAI,CAAC,GAChF;GACF,CAAC,CACH;EACF,OAGE,IADgB,KAAK,QAAQ,IAAI,QACvB,GACR,KAAK,WACH,MACA,YAAY,wBAAwB,KAAK,KAAK,CAAC,EAAE,wBACnD;CAGN;CAEA,WAAmB,MAAwB,QAAgB;EACzD,KAAK,MAAM,MAAM;EACjB,KAAK,OAAO,YAAY,qBAAqB,KAAK,KAAK,YAAY,QAAQ;EAC3E,KAAK,cAAc,OAAO,UAAU,KAAK,IAAI,CAAC;EAC9C,KAAK,cAAc,OAAO,QAAQ,KAAK,MAAM,EAAE,CAAC;EAChD,KAAK,gBAAgB,OAAO,QAAQ,KAAK,MAAM,EAAE,CAAC;CACpD;CAEA,MAAc,oBAAoB,QAA0B,UAAkB;EAE5E,IADgB,KAAK,cAAc,IAAI,QAAQ,MAAM,CAAC,CAAC,EAAE,QAAQ,QAAQ,KAAK,OACjE,KAAK,cAAc,OAAO,QAAQ,MAAM,CAAC;CACxD;;CAGA,MAAM,aAAa;EACjB,KAAK,cAAc,KAAK;EAExB,KAAK,cAAc,SAAS,MAAM,QAAQ;GACxC,KAAK,cAAc,OAAO,GAAG;GAC7B,KAAK,OAAO,YAAY,QAAQ,mBAAmB,KAAK,MAAM,EAAE,EAAE,UAAU;EAC9E,CAAC;CACH;CAEA,MAAM,UAAyB;EAC7B,MAAM,KAAK,YAAY,QAAQ;CACjC;CAEA,OAAO,OAAO,gBAA+B;EAC3C,MAAM,KAAK,QAAQ;CACrB;AACF;;AAGA,IAAM,qBAAN,MAAyB;CAIJ;CACA;CAJnB,UAA2B,IAAI,eAAe;CAE9C,YACE,MACA,QACA;EAFiB,KAAA,OAAA;EACA,KAAA,SAAA;CAChB;CAEH,YAA4C;EAC1C,OAAO;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;EAAK;CAChD;CAEA,OAAc,UAAkB;EAC9B,KAAK,QAAQ,IAAI,QAAQ;CAC3B;CAEA,QAAe,UAA2B;EACxC,OAAO,KAAK,QAAQ,IAAI,QAAQ;CAClC;AACF;AAEA,IAAM,kBAAN,MAAsB;CAOD;CACA;CACA;CARnB;CACA;CACA,SAAwC,IAAI,aAAa;CACzD,QAAiC,KAAA;CAEjC,YACE,MACA,OACA,iBACA;EAHiB,KAAA,OAAA;EACA,KAAA,QAAA;EACA,KAAA,kBAAA;EAEjB,KAAK,UAAU,IAAI,QAAQ,YAAY,KAAK,OAAO,CAAC;CACtD;CAEA,cAAc,GAGZ;EACA,KAAK,OAAO,cAAc,CAAC;EAE3B,KAAK,QAAQ,SAAS;EAEtB,OAAO;GACL,KAAK,KAAK;GACV,OAAO,KAAK;EACd;CACF;CAEA,MAAM,SAAwB;EAC5B,IAAI;GACF,MAAM,UAAU,MAAM,aAAa,KAAK,MAAM,KAAK,OAAO,KAAK,eAAe;GAE9E,IAAI,KAAK,OAAO,SAAS,KAAK,OAAO,YAAY,YAAY,KAAK,KAAK,SAAS;GAChF,KAAK,MAAM;EACb,SAAS,GAAQ;GACf,IAAI,gBAAgB,CAAC,GAAG;IAEtB,KAAK,MAAM;IACX,KAAK,QAAQ;IACb,KAAK,OAAO,YAAY,kBAAkB,KAAK,KAAK,4BAA4B;IAChF;GACF;GAEA,MAAM;EACR;CACF;AACF;;;AAIA,eAAe,aACb,OACA,QACA,iBACiB;CACjB,IAAI;CACJ,IAAI;CAEJ,IAAI;EACF,WAAWC,KAAG,iBAAiB,KAAK;EACpC,KAAK,SAAS,gBAAgB;GAAE,OAAO;GAAU,WAAW;EAAS,CAAC;EAEtE,MAAM,QAAQ,IAAI,OAAO;EAEzB,WAAW,MAAM,QAAQ,IAAI;GAC3B,IAAI,mBAAmB,KAAA,KAAa,CAAC,KAAK,SAAS,eAAe,GAAG;GAErE,MAAM,KAAK,IAAI;GACf,IAAI,MAAM,SAAS,QACjB,MAAM,MAAM;EAEhB;EAGA,OAAO,MAAM,QAAQ,CAAC,CAAC,KAAK,GAAG,GAAG,IAAI,GAAG;CAC3C,UAAU;EAER,IAAI;GACF,IAAI,IACF,GAAG,MAAM;EAEb,SAAS,cAAc;GACrB,QAAQ,MAAM,qCAAqC,YAAY;EACjE;EAEA,IAAI;GACF,IAAI,YAAY,CAAC,SAAS,WACxB,SAAS,QAAQ;EAErB,SAAS,cAAc;GACrB,QAAQ,MAAM,iCAAiC,YAAY;EAC7D;CACF;AACF;AAEA,SAAS,iCAAiC,YAAoB,OAAqB;CACjF,IAAI,CAAC,MAAM,KAAK,WAAW,OAAO,GAAG;EACnC,IAAI,UAAU,GAAG,WAAW,yBAAyB,MAAM,KAAK;EAChE,IAAI,MAAM,QAAQ,QAChB,WAAW;EAEb,MAAM,IAAI,uBAAuB,OAAO;CAC1C;AACF"}

package/dist/drivers/download_blob/download_blob_task.cjs CHANGED Viewed

@@ -3,17 +3,22 @@ const require_download_errors = require("../../helpers/download_errors.cjs");
 const require_download = require("../../clients/download.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let node_fs_promises = require("node:fs/promises");
-node_fs_promises = require_runtime.__toESM(node_fs_promises);
+node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
 let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
 let node_fs = require("node:fs");
-node_fs = require_runtime.__toESM(node_fs);
+node_fs = require_runtime.__toESM(node_fs, 1);
 let node_path = require("node:path");
-node_path = require_runtime.__toESM(node_path);
+node_path = require_runtime.__toESM(node_path, 1);
 let node_stream = require("node:stream");
 let _milaboratories_computable = require("@milaboratories/computable");
 //#region src/drivers/download_blob/download_blob_task.ts
 /** Downloads a blob and holds callers and watchers for the blob. */
 var DownloadBlobTask = class {
+	logger;
+	clientDownload;
+	rInfo;
+	handle;
+	path;
 	change = new _milaboratories_computable.ChangeSource();
 	signalCtl = new AbortController();
 	counter = new _milaboratories_ts_helpers.CallersCounter();
@@ -132,6 +137,7 @@ function getDownloadedBlobResponse(handle, size, error) {
 }
 //#endregion
 exports.DownloadBlobTask = DownloadBlobTask;
+exports.getDownloadedBlobResponse = getDownloadedBlobResponse;
 exports.nonRecoverableError = nonRecoverableError;
 //# sourceMappingURL=download_blob_task.cjs.map