@milaboratories/pl-drivers 1.15.6 → 1.16.1

package/dist/clients/download.cjs

@@ -1,7 +1,9 @@
 const require_runtime = require("../_virtual/_rolldown/runtime.cjs");
+const require_download_errors = require("../helpers/download_errors.cjs");
 const require_download = require("../helpers/download.cjs");
 const require_validate = require("../helpers/validate.cjs");
 const require_protocol_client = require("../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs");
+const require_download_url_cache = require("./download_url_cache.cjs");
 let _milaboratories_pl_client = require("@milaboratories/pl-client");
 let node_fs_promises = require("node:fs/promises");
 node_fs_promises = require_runtime.__toESM(node_fs_promises);
@@ -22,6 +24,14 @@ var ClientDownload = class {
 	localStorageIdsToRoot;
 	/** Concurrency limiter for local file reads - limit to 32 parallel reads */
 	localFileReadLimiter = new _milaboratories_ts_helpers.ConcurrencyLimitingExecutor(32);
+	/** Caches presigned download URLs by resource id until they (almost) expire. */
+	urlCache;
+	/** Active remote downloads; in-flight gauges are summed over this set on read. */
+	inFlight = /* @__PURE__ */ new Set();
+	presignedUrlCacheHits = 0;
+	presignedUrlCacheMisses = 0;
+	presignedUrlStaleHits = 0;
+	presignedUrlRequestSumLatencyMs = 0;
 	constructor(wireClientProviderFactory, httpClient, logger, localProjections) {
 		this.httpClient = httpClient;
 		this.logger = logger;
@@ -36,6 +46,7 @@ var ClientDownload = class {
 		});
 		this.remoteFileDownloader = new require_download.RemoteFileDownloader(httpClient);
 		this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);
+		this.urlCache = new require_download_url_cache.DownloadUrlCache(logger);
 	}
 	close() {}
 	/**
@@ -45,13 +56,68 @@ var ClientDownload = class {
 	* @param toBytes - to byte excluding this byte
 	*/
 	async withBlobContent(info, options, ops, handler) {
-		const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
-		const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
-		this.logger.info(`blob ${(0, _milaboratories_pl_client.stringifyWithResourceId)(info)} download started, url: ${downloadUrl}, range: ${JSON.stringify(ops.range ?? null)}`);
-		const timer = _milaboratories_helpers.PerfTimer.start();
-		const result = isLocal(downloadUrl) ? await this.withLocalFileContent(downloadUrl, ops, handler) : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
-		this.logger.info(`blob ${(0, _milaboratories_pl_client.stringifyWithResourceId)(info)} download finished, took: ${timer.elapsed()}`);
-		return result;
+		const attempt = async ({ downloadUrl, headers }) => {
+			const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
+			this.logger.info(`blob ${(0, _milaboratories_pl_client.stringifyWithResourceId)(info)} download started, url: ${downloadUrl}, range: ${JSON.stringify(ops.range ?? null)}`);
+			const timer = _milaboratories_helpers.PerfTimer.start();
+			const result = isLocal(downloadUrl) ? await this.withLocalFileContent(downloadUrl, ops, handler) : await this.withTrackedRemoteContent(downloadUrl, remoteHeaders, ops, handler);
+			this.logger.info(`blob ${(0, _milaboratories_pl_client.stringifyWithResourceId)(info)} download finished, took: ${timer.elapsed()}`);
+			return result;
+		};
+		const cached = this.urlCache.get(info.id);
+		if (cached !== void 0) try {
+			const result = await attempt(cached);
+			this.presignedUrlCacheHits++;
+			return result;
+		} catch (error) {
+			if (!require_download_errors.isDownloadNetworkError400(error)) throw error;
+			this.urlCache.delete(info.id);
+			this.presignedUrlStaleHits++;
+			this.logger.info(`cached download URL for blob ${(0, _milaboratories_pl_client.stringifyWithResourceId)(info)} rejected (status ${error.statusCode}), re-fetching`);
+		}
+		else this.presignedUrlCacheMisses++;
+		const urlFetchStartMs = performance.now();
+		const fresh = await this.grpcGetDownloadUrl(info, options, ops.signal);
+		this.presignedUrlRequestSumLatencyMs += performance.now() - urlFetchStartMs;
+		this.urlCache.set(info.id, fresh);
+		return await attempt(fresh);
+	}
+	/** Download-client-owned slice of {@link BlobDriverMetrics}: presigned-URL cache + in-flight downloads. */
+	metrics() {
+		let inFlightBytesTotal = 0;
+		let inFlightBytesReceived = 0;
+		for (const rec of this.inFlight) {
+			inFlightBytesTotal += rec.size;
+			inFlightBytesReceived += rec.received;
+		}
+		return {
+			downloadsInFlight: this.inFlight.size,
+			inFlightBytesTotal,
+			inFlightBytesReceived,
+			presignedUrlCacheHits: this.presignedUrlCacheHits,
+			presignedUrlCacheMisses: this.presignedUrlCacheMisses,
+			presignedUrlStaleHits: this.presignedUrlStaleHits,
+			presignedUrlRequestSumLatencyMs: this.presignedUrlRequestSumLatencyMs
+		};
+	}
+	/** Wraps a remote download so it appears in the in-flight gauges and its received bytes are counted live. */
+	async withTrackedRemoteContent(url, headers, ops, handler) {
+		const rec = {
+			size: 0,
+			received: 0
+		};
+		this.inFlight.add(rec);
+		try {
+			return await this.remoteFileDownloader.withContent(url, headers, ops, async (content, size) => {
+				rec.size = size;
+				return await handler(content.pipeThrough(new TransformStream({ transform: (chunk, controller) => {
+					rec.received += chunk.byteLength;
+					controller.enqueue(chunk);
+				} })), size);
+			});
+		} finally {
+			this.inFlight.delete(rec);
+		}
 	}
 	async withLocalFileContent(url, ops, handler) {
 		const { storageId, relativePath } = parseLocalUrl(url);

package/dist/clients/download.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"download.cjs","names":["ConcurrencyLimitingExecutor","DownloadClient","RestAPI","RemoteFileDownloader","PerfTimer","fsp","fs","Readable","path"],"sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { WireClientProvider, WireClientProviderFactory } from \"@milaboratories/pl-client\";\nimport {\n  addRTypeToMetadata,\n  stringifyWithResourceId,\n  RestAPI,\n  createRTypeRoutingHeader,\n  parseSignedResourceId,\n  signatureToBase64Url,\n} from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { PerfTimer } from \"@milaboratories/helpers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport type { Dispatcher } from \"undici\";\nimport type { LocalStorageProjection } from \"../drivers/types\";\nimport { type ContentHandler, RemoteFileDownloader } from \"../helpers/download\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport type { DownloadAPI_GetDownloadURL_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol\";\nimport { DownloadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client\";\nimport type { DownloadApiPaths, DownloadRestClientType } from \"../proto-rest\";\nimport { type GetContentOptions } from \"@milaboratories/pl-model-common\";\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly wire: WireClientProvider<DownloadRestClientType | DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider((wireConn) => {\n      if (wireConn.type === \"grpc\") {\n        return new DownloadClient(wireConn.Transport);\n      } else {\n        return RestAPI.createClient<DownloadApiPaths>({\n          hostAndPort: wireConn.Config.hostAndPort,\n          ssl: wireConn.Config.ssl,\n          dispatcher: wireConn.Dispatcher,\n          middlewares: wireConn.Middlewares,\n        });\n      }\n    });\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download started, ` +\n        `url: ${downloadUrl}, ` +\n        `range: ${JSON.stringify(ops.range ?? null)}`,\n    );\n\n    const timer = PerfTimer.start();\n    const result = isLocal(downloadUrl)\n      ? await this.withLocalFileContent(downloadUrl, ops, handler)\n      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download finished, ` + `took: ${timer.elapsed()}`,\n    );\n    return result;\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    const { globalId, signature } = parseSignedResourceId(id);\n    const client = this.wire.get();\n    if (client instanceof DownloadClient) {\n      return await client.getDownloadURL(\n        { resourceId: globalId, resourceSignature: signature, isInternalUse: false },\n        addRTypeToMetadata(type, withAbort),\n      ).response;\n    } else {\n      return (\n        await client.POST(\"/v1/get-download-url\", {\n          body: {\n            resourceId: globalId.toString(),\n            resourceSignature: signatureToBase64Url(signature),\n            isInternalUse: false,\n          },\n          headers: { ...createRTypeRoutingHeader(type) },\n        })\n      ).data!;\n    }\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == \"\")\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === \"\") return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = \"storage://\";\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = \"WrongLocalFileUrl\";\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = \"UnknownStorageError\";\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== \"\") {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AA8BA,IAAa,iBAAb,MAA4B;CAC1B;CACA;;CAGA;;CAGA,uBAAwC,IAAIA,2BAAAA,4BAA4B,GAAG;CAE3E,YACE,2BACA,YACA,QAEA,kBACA;AAJgB,OAAA,aAAA;AACA,OAAA,SAAA;AAIhB,OAAK,OAAO,0BAA0B,0BAA0B,aAAa;AAC3E,OAAI,SAAS,SAAS,OACpB,QAAO,IAAIC,wBAAAA,eAAe,SAAS,UAAU;OAE7C,QAAOC,0BAAAA,QAAQ,aAA+B;IAC5C,aAAa,SAAS,OAAO;IAC7B,KAAK,SAAS,OAAO;IACrB,YAAY,SAAS;IACrB,aAAa,SAAS;IACvB,CAAC;IAEJ;AACF,OAAK,uBAAuB,IAAIC,iBAAAA,qBAAqB,WAAW;AAChE,OAAK,wBAAwB,yBAAyB,iBAAiB;;CAGzE,QAAQ;;;;;;;CAQR,MAAM,gBACJ,MACA,SACA,KACA,SACY;EACZ,MAAM,EAAE,aAAa,YAAY,MAAM,KAAK,mBAAmB,MAAM,SAAS,IAAI,OAAO;EAEzF,MAAM,gBAAgB,OAAO,YAAY,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,MAAM,CAAC,CAAC;AACzF,OAAK,OAAO,KACV,SAAA,GAAA,0BAAA,yBAAgC,KAAK,CAAC,0BAC5B,YAAY,WACV,KAAK,UAAU,IAAI,SAAS,KAAK,GAC9C;EAED,MAAM,QAAQC,wBAAAA,UAAU,OAAO;EAC/B,MAAM,SAAS,QAAQ,YAAY,GAC/B,MAAM,KAAK,qBAAqB,aAAa,KAAK,QAAQ,GAC1D,MAAM,KAAK,qBAAqB,YAAY,aAAa,eAAe,KAAK,QAAQ;AAEzF,OAAK,OAAO,KACV,SAAA,GAAA,0BAAA,yBAAgC,KAAK,CAAC,4BAAiC,MAAM,SAAS,GACvF;AACD,SAAO;;CAGT,MAAM,qBACJ,KACA,KACA,SACY;EACZ,MAAM,EAAE,WAAW,iBAAiB,cAAc,IAAI;EACtD,MAAM,WAAW,YAAY,WAAW,KAAK,uBAAuB,aAAa;AAEjF,SAAO,MAAM,KAAK,qBAAqB,IAAI,YAAY;GACrD,MAAM,UAAU;IACd,OAAO,IAAI,OAAO;IAClB,KAAK,IAAI,OAAO,OAAO,KAAA,IAAY,IAAI,MAAM,KAAK,IAAI,KAAA;IACtD,QAAQ,IAAI;IACb;GACD,IAAI;GACJ,IAAI,iBAAiB;AAErB,OAAI;IACF,MAAM,OAAO,MAAMC,iBAAI,KAAK,SAAS;AACrC,aAASC,QAAG,iBAAiB,UAAU,QAAQ;IAG/C,MAAM,SAAS,MAAM,QAFHC,YAAAA,SAAS,MAAM,OAAO,EAEA,KAAK,KAAK;AAClD,qBAAiB;AACjB,WAAO;YACA,OAAO;AAEd,QAAI,CAAC,kBAAkB,UAAU,CAAC,OAAO,UACvC,QAAO,SAAS;AAElB,UAAM;;IAER;;CAGJ,MAAc,mBACZ,EAAE,IAAI,QACN,SACA,QAC8C;EAC9C,MAAM,YAAY,WAAW,EAAE;AAC/B,YAAU,QAAQ;EAElB,MAAM,EAAE,UAAU,eAAA,GAAA,0BAAA,uBAAoC,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAC9B,MAAI,kBAAkBN,wBAAAA,eACpB,QAAO,MAAM,OAAO,eAClB;GAAE,YAAY;GAAU,mBAAmB;GAAW,eAAe;GAAO,GAAA,GAAA,0BAAA,oBACzD,MAAM,UAAU,CACpC,CAAC;MAEF,SACE,MAAM,OAAO,KAAK,wBAAwB;GACxC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,oBAAA,GAAA,0BAAA,sBAAwC,UAAU;IAClD,eAAe;IAChB;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;GAC/C,CAAC,EACF;;;AAKR,SAAgB,cAAc,KAAa;CACzC,MAAM,SAAS,IAAI,IAAI,IAAI;AAC3B,KAAI,OAAO,YAAY,GACrB,OAAM,IAAI,kBAAkB,0BAA0B,IAAI,4BAA4B;AAExF,QAAO;EACL,WAAW,OAAO;EAClB,cAAc,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;EAC3D;;AAGH,SAAgB,YACd,WACA,uBACA,cACA;CACA,MAAM,OAAO,sBAAsB,IAAI,UAAU;AACjD,KAAI,SAAS,KAAA,EAAW,OAAM,IAAI,oBAAoB,6BAA6B,YAAY;AAE/F,KAAI,SAAS,GAAI,QAAO;AAExB,QAAOO,UAAK,KAAK,MAAM,aAAa;;AAGtC,MAAM,kBAAkB;AACxB,SAAS,QAAQ,KAAa;AAC5B,QAAO,IAAI,WAAW,gBAAgB;;;AAIxC,IAAa,oBAAb,cAAuC,MAAM;CAC3C,OAAO;;;AAIT,IAAa,sBAAb,cAAyC,MAAM;CAC7C,OAAO;;AAGT,SAAgB,yBAAyB,aAAuC;CAC9E,MAAM,2BAAgC,IAAI,KAAK;AAC/C,MAAK,MAAM,MAAM,aAAa;AAE5B,MAAI,GAAG,cAAc,GACnB,kBAAA,iBAAiB,GAAG,UAAU;AAEhC,WAAS,IAAI,GAAG,WAAW,GAAG,UAAU;;AAG1C,QAAO"}
+{"version":3,"file":"download.cjs","names":["ConcurrencyLimitingExecutor","DownloadClient","RestAPI","RemoteFileDownloader","DownloadUrlCache","PerfTimer","isDownloadNetworkError400","fsp","fs","Readable","path"],"sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { WireClientProvider, WireClientProviderFactory } from \"@milaboratories/pl-client\";\nimport {\n  addRTypeToMetadata,\n  stringifyWithResourceId,\n  RestAPI,\n  createRTypeRoutingHeader,\n  parseSignedResourceId,\n  signatureToBase64Url,\n} from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { PerfTimer } from \"@milaboratories/helpers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport type { Dispatcher } from \"undici\";\nimport type { LocalStorageProjection } from \"../drivers/types\";\nimport { type ContentHandler, RemoteFileDownloader } from \"../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../helpers/download_errors\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport type { DownloadAPI_GetDownloadURL_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol\";\nimport { DownloadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client\";\nimport type { DownloadApiPaths, DownloadRestClientType } from \"../proto-rest\";\nimport { type GetContentOptions, type BlobDriverMetrics } from \"@milaboratories/pl-model-common\";\nimport { DownloadUrlCache } from \"./download_url_cache\";\n\n/** Subset of {@link BlobDriverMetrics} owned by the download client (presigned cache + in-flight downloads). */\ntype ClientDownloadMetrics = Omit<BlobDriverMetrics, \"uncachedRequests\" | \"uncachedRequestBytes\">;\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly wire: WireClientProvider<DownloadRestClientType | DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  /** Caches presigned download URLs by resource id until they (almost) expire. */\n  private readonly urlCache: DownloadUrlCache;\n\n  /** Active remote downloads; in-flight gauges are summed over this set on read. */\n  private readonly inFlight = new Set<{ size: number; received: number }>();\n  private presignedUrlCacheHits = 0;\n  private presignedUrlCacheMisses = 0;\n  private presignedUrlStaleHits = 0;\n  private presignedUrlRequestSumLatencyMs = 0;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider((wireConn) => {\n      if (wireConn.type === \"grpc\") {\n        return new DownloadClient(wireConn.Transport);\n      } else {\n        return RestAPI.createClient<DownloadApiPaths>({\n          hostAndPort: wireConn.Config.hostAndPort,\n          ssl: wireConn.Config.ssl,\n          dispatcher: wireConn.Dispatcher,\n          middlewares: wireConn.Middlewares,\n        });\n      }\n    });\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n    this.urlCache = new DownloadUrlCache(logger);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const attempt = async ({\n      downloadUrl,\n      headers,\n    }: DownloadAPI_GetDownloadURL_Response): Promise<T> => {\n      const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n      this.logger.info(\n        `blob ${stringifyWithResourceId(info)} download started, ` +\n          `url: ${downloadUrl}, ` +\n          `range: ${JSON.stringify(ops.range ?? null)}`,\n      );\n\n      const timer = PerfTimer.start();\n      const result = isLocal(downloadUrl)\n        ? await this.withLocalFileContent(downloadUrl, ops, handler)\n        : await this.withTrackedRemoteContent(downloadUrl, remoteHeaders, ops, handler);\n\n      this.logger.info(\n        `blob ${stringifyWithResourceId(info)} download finished, ` + `took: ${timer.elapsed()}`,\n      );\n      return result;\n    };\n\n    const cached = this.urlCache.get(info.id);\n    if (cached !== undefined) {\n      try {\n        const result = await attempt(cached);\n        this.presignedUrlCacheHits++;\n        return result;\n      } catch (error) {\n        if (!isDownloadNetworkError400(error)) throw error;\n        this.urlCache.delete(info.id);\n        this.presignedUrlStaleHits++;\n        this.logger.info(\n          `cached download URL for blob ${stringifyWithResourceId(info)} rejected ` +\n            `(status ${error.statusCode}), re-fetching`,\n        );\n      }\n    } else {\n      this.presignedUrlCacheMisses++;\n    }\n\n    const urlFetchStartMs = performance.now();\n    const fresh = await this.grpcGetDownloadUrl(info, options, ops.signal);\n    this.presignedUrlRequestSumLatencyMs += performance.now() - urlFetchStartMs;\n    this.urlCache.set(info.id, fresh);\n    return await attempt(fresh);\n  }\n\n  /** Download-client-owned slice of {@link BlobDriverMetrics}: presigned-URL cache + in-flight downloads. */\n  metrics(): ClientDownloadMetrics {\n    let inFlightBytesTotal = 0;\n    let inFlightBytesReceived = 0;\n    for (const rec of this.inFlight) {\n      inFlightBytesTotal += rec.size;\n      inFlightBytesReceived += rec.received;\n    }\n    return {\n      downloadsInFlight: this.inFlight.size,\n      inFlightBytesTotal,\n      inFlightBytesReceived,\n      presignedUrlCacheHits: this.presignedUrlCacheHits,\n      presignedUrlCacheMisses: this.presignedUrlCacheMisses,\n      presignedUrlStaleHits: this.presignedUrlStaleHits,\n      presignedUrlRequestSumLatencyMs: this.presignedUrlRequestSumLatencyMs,\n    };\n  }\n\n  /** Wraps a remote download so it appears in the in-flight gauges and its received bytes are counted live. */\n  private async withTrackedRemoteContent<T>(\n    url: string,\n    headers: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const rec = { size: 0, received: 0 };\n    this.inFlight.add(rec);\n    try {\n      return await this.remoteFileDownloader.withContent(\n        url,\n        headers,\n        ops,\n        async (content, size) => {\n          rec.size = size;\n          const counted = content.pipeThrough(\n            new TransformStream<Uint8Array, Uint8Array>({\n              transform: (chunk, controller) => {\n                rec.received += chunk.byteLength;\n                controller.enqueue(chunk);\n              },\n            }),\n          );\n          return await handler(counted, size);\n        },\n      );\n    } finally {\n      this.inFlight.delete(rec);\n    }\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    const { globalId, signature } = parseSignedResourceId(id);\n    const client = this.wire.get();\n    if (client instanceof DownloadClient) {\n      return await client.getDownloadURL(\n        { resourceId: globalId, resourceSignature: signature, isInternalUse: false },\n        addRTypeToMetadata(type, withAbort),\n      ).response;\n    } else {\n      return (\n        await client.POST(\"/v1/get-download-url\", {\n          body: {\n            resourceId: globalId.toString(),\n            resourceSignature: signatureToBase64Url(signature),\n            isInternalUse: false,\n          },\n          headers: { ...createRTypeRoutingHeader(type) },\n        })\n      ).data!;\n    }\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == \"\")\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === \"\") return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = \"storage://\";\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = \"WrongLocalFileUrl\";\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = \"UnknownStorageError\";\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== \"\") {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAmCA,IAAa,iBAAb,MAA4B;CAC1B;CACA;;CAGA;;CAGA,uBAAwC,IAAIA,2BAAAA,4BAA4B,GAAG;;CAG3E;;CAGA,2BAA4B,IAAI,KAAyC;CACzE,wBAAgC;CAChC,0BAAkC;CAClC,wBAAgC;CAChC,kCAA0C;CAE1C,YACE,2BACA,YACA,QAEA,kBACA;AAJgB,OAAA,aAAA;AACA,OAAA,SAAA;AAIhB,OAAK,OAAO,0BAA0B,0BAA0B,aAAa;AAC3E,OAAI,SAAS,SAAS,OACpB,QAAO,IAAIC,wBAAAA,eAAe,SAAS,UAAU;OAE7C,QAAOC,0BAAAA,QAAQ,aAA+B;IAC5C,aAAa,SAAS,OAAO;IAC7B,KAAK,SAAS,OAAO;IACrB,YAAY,SAAS;IACrB,aAAa,SAAS;IACvB,CAAC;IAEJ;AACF,OAAK,uBAAuB,IAAIC,iBAAAA,qBAAqB,WAAW;AAChE,OAAK,wBAAwB,yBAAyB,iBAAiB;AACvE,OAAK,WAAW,IAAIC,2BAAAA,iBAAiB,OAAO;;CAG9C,QAAQ;;;;;;;CAQR,MAAM,gBACJ,MACA,SACA,KACA,SACY;EACZ,MAAM,UAAU,OAAO,EACrB,aACA,cACqD;GACrD,MAAM,gBAAgB,OAAO,YAAY,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,MAAM,CAAC,CAAC;AACzF,QAAK,OAAO,KACV,SAAA,GAAA,0BAAA,yBAAgC,KAAK,CAAC,0BAC5B,YAAY,WACV,KAAK,UAAU,IAAI,SAAS,KAAK,GAC9C;GAED,MAAM,QAAQC,wBAAAA,UAAU,OAAO;GAC/B,MAAM,SAAS,QAAQ,YAAY,GAC/B,MAAM,KAAK,qBAAqB,aAAa,KAAK,QAAQ,GAC1D,MAAM,KAAK,yBAAyB,aAAa,eAAe,KAAK,QAAQ;AAEjF,QAAK,OAAO,KACV,SAAA,GAAA,0BAAA,yBAAgC,KAAK,CAAC,4BAAiC,MAAM,SAAS,GACvF;AACD,UAAO;;EAGT,MAAM,SAAS,KAAK,SAAS,IAAI,KAAK,GAAG;AACzC,MAAI,WAAW,KAAA,EACb,KAAI;GACF,MAAM,SAAS,MAAM,QAAQ,OAAO;AACpC,QAAK;AACL,UAAO;WACA,OAAO;AACd,OAAI,CAACC,wBAAAA,0BAA0B,MAAM,CAAE,OAAM;AAC7C,QAAK,SAAS,OAAO,KAAK,GAAG;AAC7B,QAAK;AACL,QAAK,OAAO,KACV,iCAAA,GAAA,0BAAA,yBAAwD,KAAK,CAAC,oBACjD,MAAM,WAAW,gBAC/B;;MAGH,MAAK;EAGP,MAAM,kBAAkB,YAAY,KAAK;EACzC,MAAM,QAAQ,MAAM,KAAK,mBAAmB,MAAM,SAAS,IAAI,OAAO;AACtE,OAAK,mCAAmC,YAAY,KAAK,GAAG;AAC5D,OAAK,SAAS,IAAI,KAAK,IAAI,MAAM;AACjC,SAAO,MAAM,QAAQ,MAAM;;;CAI7B,UAAiC;EAC/B,IAAI,qBAAqB;EACzB,IAAI,wBAAwB;AAC5B,OAAK,MAAM,OAAO,KAAK,UAAU;AAC/B,yBAAsB,IAAI;AAC1B,4BAAyB,IAAI;;AAE/B,SAAO;GACL,mBAAmB,KAAK,SAAS;GACjC;GACA;GACA,uBAAuB,KAAK;GAC5B,yBAAyB,KAAK;GAC9B,uBAAuB,KAAK;GAC5B,iCAAiC,KAAK;GACvC;;;CAIH,MAAc,yBACZ,KACA,SACA,KACA,SACY;EACZ,MAAM,MAAM;GAAE,MAAM;GAAG,UAAU;GAAG;AACpC,OAAK,SAAS,IAAI,IAAI;AACtB,MAAI;AACF,UAAO,MAAM,KAAK,qBAAqB,YACrC,KACA,SACA,KACA,OAAO,SAAS,SAAS;AACvB,QAAI,OAAO;AASX,WAAO,MAAM,QARG,QAAQ,YACtB,IAAI,gBAAwC,EAC1C,YAAY,OAAO,eAAe;AAChC,SAAI,YAAY,MAAM;AACtB,gBAAW,QAAQ,MAAM;OAE5B,CAAC,CACH,EAC6B,KAAK;KAEtC;YACO;AACR,QAAK,SAAS,OAAO,IAAI;;;CAI7B,MAAM,qBACJ,KACA,KACA,SACY;EACZ,MAAM,EAAE,WAAW,iBAAiB,cAAc,IAAI;EACtD,MAAM,WAAW,YAAY,WAAW,KAAK,uBAAuB,aAAa;AAEjF,SAAO,MAAM,KAAK,qBAAqB,IAAI,YAAY;GACrD,MAAM,UAAU;IACd,OAAO,IAAI,OAAO;IAClB,KAAK,IAAI,OAAO,OAAO,KAAA,IAAY,IAAI,MAAM,KAAK,IAAI,KAAA;IACtD,QAAQ,IAAI;IACb;GACD,IAAI;GACJ,IAAI,iBAAiB;AAErB,OAAI;IACF,MAAM,OAAO,MAAMC,iBAAI,KAAK,SAAS;AACrC,aAASC,QAAG,iBAAiB,UAAU,QAAQ;IAG/C,MAAM,SAAS,MAAM,QAFHC,YAAAA,SAAS,MAAM,OAAO,EAEA,KAAK,KAAK;AAClD,qBAAiB;AACjB,WAAO;YACA,OAAO;AAEd,QAAI,CAAC,kBAAkB,UAAU,CAAC,OAAO,UACvC,QAAO,SAAS;AAElB,UAAM;;IAER;;CAGJ,MAAc,mBACZ,EAAE,IAAI,QACN,SACA,QAC8C;EAC9C,MAAM,YAAY,WAAW,EAAE;AAC/B,YAAU,QAAQ;EAElB,MAAM,EAAE,UAAU,eAAA,GAAA,0BAAA,uBAAoC,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAC9B,MAAI,kBAAkBR,wBAAAA,eACpB,QAAO,MAAM,OAAO,eAClB;GAAE,YAAY;GAAU,mBAAmB;GAAW,eAAe;GAAO,GAAA,GAAA,0BAAA,oBACzD,MAAM,UAAU,CACpC,CAAC;MAEF,SACE,MAAM,OAAO,KAAK,wBAAwB;GACxC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,oBAAA,GAAA,0BAAA,sBAAwC,UAAU;IAClD,eAAe;IAChB;GACD,SAAS,EAAE,IAAA,GAAA,0BAAA,0BAA4B,KAAK,EAAE;GAC/C,CAAC,EACF;;;AAKR,SAAgB,cAAc,KAAa;CACzC,MAAM,SAAS,IAAI,IAAI,IAAI;AAC3B,KAAI,OAAO,YAAY,GACrB,OAAM,IAAI,kBAAkB,0BAA0B,IAAI,4BAA4B;AAExF,QAAO;EACL,WAAW,OAAO;EAClB,cAAc,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;EAC3D;;AAGH,SAAgB,YACd,WACA,uBACA,cACA;CACA,MAAM,OAAO,sBAAsB,IAAI,UAAU;AACjD,KAAI,SAAS,KAAA,EAAW,OAAM,IAAI,oBAAoB,6BAA6B,YAAY;AAE/F,KAAI,SAAS,GAAI,QAAO;AAExB,QAAOS,UAAK,KAAK,MAAM,aAAa;;AAGtC,MAAM,kBAAkB;AACxB,SAAS,QAAQ,KAAa;AAC5B,QAAO,IAAI,WAAW,gBAAgB;;;AAIxC,IAAa,oBAAb,cAAuC,MAAM;CAC3C,OAAO;;;AAIT,IAAa,sBAAb,cAAyC,MAAM;CAC7C,OAAO;;AAGT,SAAgB,yBAAyB,aAAuC;CAC9E,MAAM,2BAAgC,IAAI,KAAK;AAC/C,MAAK,MAAM,MAAM,aAAa;AAE5B,MAAI,GAAG,cAAc,GACnB,kBAAA,iBAAiB,GAAG,UAAU;AAEhC,WAAS,IAAI,GAAG,WAAW,GAAG,UAAU;;AAG1C,QAAO"}

package/dist/clients/download.d.ts

@@ -6,10 +6,12 @@ import { WireClientProvider, WireClientProviderFactory } from "@milaboratories/p
 import { Dispatcher } from "undici";
 import { RpcOptions } from "@protobuf-ts/runtime-rpc";
 import { MiLogger } from "@milaboratories/ts-helpers";
-import { GetContentOptions } from "@milaboratories/pl-model-common";
+import { BlobDriverMetrics, GetContentOptions } from "@milaboratories/pl-model-common";
 import { ResourceInfo } from "@milaboratories/pl-tree";
 
 //#region src/clients/download.d.ts
+/** Subset of {@link BlobDriverMetrics} owned by the download client (presigned cache + in-flight downloads). */
+type ClientDownloadMetrics = Omit<BlobDriverMetrics, "uncachedRequests" | "uncachedRequestBytes">;
 /** Gets URLs for downloading from pl-core, parses them and reads or downloads
  * files locally and from the web. */
 declare class ClientDownload {
@@ -21,6 +23,14 @@ declare class ClientDownload {
   private readonly localStorageIdsToRoot;
   /** Concurrency limiter for local file reads - limit to 32 parallel reads */
   private readonly localFileReadLimiter;
+  /** Caches presigned download URLs by resource id until they (almost) expire. */
+  private readonly urlCache;
+  /** Active remote downloads; in-flight gauges are summed over this set on read. */
+  private readonly inFlight;
+  private presignedUrlCacheHits;
+  private presignedUrlCacheMisses;
+  private presignedUrlStaleHits;
+  private presignedUrlRequestSumLatencyMs;
   constructor(wireClientProviderFactory: WireClientProviderFactory, httpClient: Dispatcher, logger: MiLogger, /** Pl storages available locally */
 
   localProjections: LocalStorageProjection[]);
@@ -32,6 +42,10 @@ declare class ClientDownload {
    * @param toBytes - to byte excluding this byte
    */
   withBlobContent<T>(info: ResourceInfo, options: RpcOptions | undefined, ops: GetContentOptions, handler: ContentHandler$1<T>): Promise<T>;
+  /** Download-client-owned slice of {@link BlobDriverMetrics}: presigned-URL cache + in-flight downloads. */
+  metrics(): ClientDownloadMetrics;
+  /** Wraps a remote download so it appears in the in-flight gauges and its received bytes are counted live. */
+  private withTrackedRemoteContent;
   withLocalFileContent<T>(url: string, ops: GetContentOptions, handler: ContentHandler$1<T>): Promise<T>;
   private grpcGetDownloadUrl;
 }

package/dist/clients/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","names":[],"sources":["../../src/clients/download.ts"],"mappings":";;;;;;;;;;;;;;cA8Ba,cAAA;EAAA,SAYO,UAAA,EAAY,UAAA;EAAA,SACZ,MAAA,EAAQ,QAAA;EAAA,SAZV,IAAA,EAAM,kBAAA,CAAmB,sBAAA,GAAyB,cAAA;EAAA,iBACjD,oBAAA;EADwB;EAAA,iBAIxB,qBAAA;EAJK;EAAA,iBAOL,oBAAA;cAGf,yBAAA,EAA2B,yBAAA,EACX,UAAA,EAAY,UAAA,EACZ,MAAA,EAAQ,QAAA,EAAA;;EAExB,gBAAA,EAAkB,sBAAA;EAkBpB,KAAA,CAAA;EAUW;;;;;;EAFL,eAAA,GAAA,CACJ,IAAA,EAAM,YAAA,EACN,OAAA,EAAS,UAAA,cACT,GAAA,EAAK,iBAAA,EACL,OAAA,EAAS,gBAAA,CAAe,CAAA,IACvB,OAAA,CAAQ,CAAA;EAqBL,oBAAA,GAAA,CACJ,GAAA,UACA,GAAA,EAAK,iBAAA,EACL,OAAA,EAAS,gBAAA,CAAe,CAAA,IACvB,OAAA,CAAQ,CAAA;EAAA,QA+BG,kBAAA;AAAA;AAAA,iBA8BA,aAAA,CAAc,GAAA;;;;iBAWd,WAAA,CACd,SAAA,UACA,qBAAA,EAAuB,GAAA,kBACvB,YAAA;;cAgBW,iBAAA,SAA0B,KAAA;EACrC,IAAA;AAAA;;cAIW,mBAAA,SAA4B,KAAA;EACvC,IAAA;AAAA;AAAA,iBAGc,wBAAA,CAAyB,WAAA,EAAa,sBAAA,KAAwB,GAAA"}
+{"version":3,"file":"download.d.ts","names":[],"sources":["../../src/clients/download.ts"],"mappings":";;;;;;;;;;;;;KA+BK,qBAAA,GAAwB,IAAA,CAAK,iBAAA;AAJ+D;;AAAA,cAQpF,cAAA;EAAA,SAsBO,UAAA,EAAY,UAAA;EAAA,SACZ,MAAA,EAAQ,QAAA;EAAA,SAtBV,IAAA,EAAM,kBAAA,CAAmB,sBAAA,GAAyB,cAAA;EAAA,iBACjD,oBAAA;EAFQ;EAAA,iBAKR,qBAAA;EAkBS;EAAA,iBAfT,oBAAA;EAPiD;EAAA,iBAUjD,QAAA;EAUY;EAAA,iBAPZ,QAAA;EAAA,QACT,qBAAA;EAAA,QACA,uBAAA;EAAA,QACA,qBAAA;EAAA,QACA,+BAAA;cAGN,yBAAA,EAA2B,yBAAA,EACX,UAAA,EAAY,UAAA,EACZ,MAAA,EAAQ,QAAA,EAiCA;;EA/BxB,gBAAA,EAAkB,sBAAA;EAmBpB,KAAA,CAAA;EAaG;;;;;;EALG,eAAA,GAAA,CACJ,IAAA,EAAM,YAAA,EACN,OAAA,EAAS,UAAA,cACT,GAAA,EAAK,iBAAA,EACL,OAAA,EAAS,gBAAA,CAAe,CAAA,IACvB,OAAA,CAAQ,CAAA;EAwGD;EAtDV,OAAA,CAAA,GAAW,qBAAA;EArFO;EAAA,QAwGJ,wBAAA;EA+BR,oBAAA,GAAA,CACJ,GAAA,UACA,GAAA,EAAK,iBAAA,EACL,OAAA,EAAS,gBAAA,CAAe,CAAA,IACvB,OAAA,CAAQ,CAAA;EAAA,QA+BG,kBAAA;AAAA;AAAA,iBA8BA,aAAA,CAAc,GAAA;;;;iBAWd,WAAA,CACd,SAAA,UACA,qBAAA,EAAuB,GAAA,kBACvB,YAAA;;cAgBW,iBAAA,SAA0B,KAAA;EACrC,IAAA;AAAA;;cAIW,mBAAA,SAA4B,KAAA;EACvC,IAAA;AAAA;AAAA,iBAGc,wBAAA,CAAyB,WAAA,EAAa,sBAAA,KAAwB,GAAA"}

package/dist/clients/download.js

@@ -1,6 +1,8 @@
+import { isDownloadNetworkError400 } from "../helpers/download_errors.js";
 import { RemoteFileDownloader } from "../helpers/download.js";
 import { validateAbsolute } from "../helpers/validate.js";
 import { DownloadClient } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js";
+import { DownloadUrlCache } from "./download_url_cache.js";
 import { RestAPI, addRTypeToMetadata, createRTypeRoutingHeader, parseSignedResourceId, signatureToBase64Url, stringifyWithResourceId } from "@milaboratories/pl-client";
 import * as fsp from "node:fs/promises";
 import { ConcurrencyLimitingExecutor } from "@milaboratories/ts-helpers";
@@ -18,6 +20,14 @@ var ClientDownload = class {
 	localStorageIdsToRoot;
 	/** Concurrency limiter for local file reads - limit to 32 parallel reads */
 	localFileReadLimiter = new ConcurrencyLimitingExecutor(32);
+	/** Caches presigned download URLs by resource id until they (almost) expire. */
+	urlCache;
+	/** Active remote downloads; in-flight gauges are summed over this set on read. */
+	inFlight = /* @__PURE__ */ new Set();
+	presignedUrlCacheHits = 0;
+	presignedUrlCacheMisses = 0;
+	presignedUrlStaleHits = 0;
+	presignedUrlRequestSumLatencyMs = 0;
 	constructor(wireClientProviderFactory, httpClient, logger, localProjections) {
 		this.httpClient = httpClient;
 		this.logger = logger;
@@ -32,6 +42,7 @@ var ClientDownload = class {
 		});
 		this.remoteFileDownloader = new RemoteFileDownloader(httpClient);
 		this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);
+		this.urlCache = new DownloadUrlCache(logger);
 	}
 	close() {}
 	/**
@@ -41,13 +52,68 @@ var ClientDownload = class {
 	* @param toBytes - to byte excluding this byte
 	*/
 	async withBlobContent(info, options, ops, handler) {
-		const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
-		const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
-		this.logger.info(`blob ${stringifyWithResourceId(info)} download started, url: ${downloadUrl}, range: ${JSON.stringify(ops.range ?? null)}`);
-		const timer = PerfTimer.start();
-		const result = isLocal(downloadUrl) ? await this.withLocalFileContent(downloadUrl, ops, handler) : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
-		this.logger.info(`blob ${stringifyWithResourceId(info)} download finished, took: ${timer.elapsed()}`);
-		return result;
+		const attempt = async ({ downloadUrl, headers }) => {
+			const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
+			this.logger.info(`blob ${stringifyWithResourceId(info)} download started, url: ${downloadUrl}, range: ${JSON.stringify(ops.range ?? null)}`);
+			const timer = PerfTimer.start();
+			const result = isLocal(downloadUrl) ? await this.withLocalFileContent(downloadUrl, ops, handler) : await this.withTrackedRemoteContent(downloadUrl, remoteHeaders, ops, handler);
+			this.logger.info(`blob ${stringifyWithResourceId(info)} download finished, took: ${timer.elapsed()}`);
+			return result;
+		};
+		const cached = this.urlCache.get(info.id);
+		if (cached !== void 0) try {
+			const result = await attempt(cached);
+			this.presignedUrlCacheHits++;
+			return result;
+		} catch (error) {
+			if (!isDownloadNetworkError400(error)) throw error;
+			this.urlCache.delete(info.id);
+			this.presignedUrlStaleHits++;
+			this.logger.info(`cached download URL for blob ${stringifyWithResourceId(info)} rejected (status ${error.statusCode}), re-fetching`);
+		}
+		else this.presignedUrlCacheMisses++;
+		const urlFetchStartMs = performance.now();
+		const fresh = await this.grpcGetDownloadUrl(info, options, ops.signal);
+		this.presignedUrlRequestSumLatencyMs += performance.now() - urlFetchStartMs;
+		this.urlCache.set(info.id, fresh);
+		return await attempt(fresh);
+	}
+	/** Download-client-owned slice of {@link BlobDriverMetrics}: presigned-URL cache + in-flight downloads. */
+	metrics() {
+		let inFlightBytesTotal = 0;
+		let inFlightBytesReceived = 0;
+		for (const rec of this.inFlight) {
+			inFlightBytesTotal += rec.size;
+			inFlightBytesReceived += rec.received;
+		}
+		return {
+			downloadsInFlight: this.inFlight.size,
+			inFlightBytesTotal,
+			inFlightBytesReceived,
+			presignedUrlCacheHits: this.presignedUrlCacheHits,
+			presignedUrlCacheMisses: this.presignedUrlCacheMisses,
+			presignedUrlStaleHits: this.presignedUrlStaleHits,
+			presignedUrlRequestSumLatencyMs: this.presignedUrlRequestSumLatencyMs
+		};
+	}
+	/** Wraps a remote download so it appears in the in-flight gauges and its received bytes are counted live. */
+	async withTrackedRemoteContent(url, headers, ops, handler) {
+		const rec = {
+			size: 0,
+			received: 0
+		};
+		this.inFlight.add(rec);
+		try {
+			return await this.remoteFileDownloader.withContent(url, headers, ops, async (content, size) => {
+				rec.size = size;
+				return await handler(content.pipeThrough(new TransformStream({ transform: (chunk, controller) => {
+					rec.received += chunk.byteLength;
+					controller.enqueue(chunk);
+				} })), size);
+			});
+		} finally {
+			this.inFlight.delete(rec);
+		}
 	}
 	async withLocalFileContent(url, ops, handler) {
 		const { storageId, relativePath } = parseLocalUrl(url);

package/dist/clients/download.js.map

@@ -1 +1 @@
-{"version":3,"file":"download.js","names":["fs","path"],"sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { WireClientProvider, WireClientProviderFactory } from \"@milaboratories/pl-client\";\nimport {\n  addRTypeToMetadata,\n  stringifyWithResourceId,\n  RestAPI,\n  createRTypeRoutingHeader,\n  parseSignedResourceId,\n  signatureToBase64Url,\n} from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { PerfTimer } from \"@milaboratories/helpers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport type { Dispatcher } from \"undici\";\nimport type { LocalStorageProjection } from \"../drivers/types\";\nimport { type ContentHandler, RemoteFileDownloader } from \"../helpers/download\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport type { DownloadAPI_GetDownloadURL_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol\";\nimport { DownloadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client\";\nimport type { DownloadApiPaths, DownloadRestClientType } from \"../proto-rest\";\nimport { type GetContentOptions } from \"@milaboratories/pl-model-common\";\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly wire: WireClientProvider<DownloadRestClientType | DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider((wireConn) => {\n      if (wireConn.type === \"grpc\") {\n        return new DownloadClient(wireConn.Transport);\n      } else {\n        return RestAPI.createClient<DownloadApiPaths>({\n          hostAndPort: wireConn.Config.hostAndPort,\n          ssl: wireConn.Config.ssl,\n          dispatcher: wireConn.Dispatcher,\n          middlewares: wireConn.Middlewares,\n        });\n      }\n    });\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download started, ` +\n        `url: ${downloadUrl}, ` +\n        `range: ${JSON.stringify(ops.range ?? null)}`,\n    );\n\n    const timer = PerfTimer.start();\n    const result = isLocal(downloadUrl)\n      ? await this.withLocalFileContent(downloadUrl, ops, handler)\n      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n\n    this.logger.info(\n      `blob ${stringifyWithResourceId(info)} download finished, ` + `took: ${timer.elapsed()}`,\n    );\n    return result;\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    const { globalId, signature } = parseSignedResourceId(id);\n    const client = this.wire.get();\n    if (client instanceof DownloadClient) {\n      return await client.getDownloadURL(\n        { resourceId: globalId, resourceSignature: signature, isInternalUse: false },\n        addRTypeToMetadata(type, withAbort),\n      ).response;\n    } else {\n      return (\n        await client.POST(\"/v1/get-download-url\", {\n          body: {\n            resourceId: globalId.toString(),\n            resourceSignature: signatureToBase64Url(signature),\n            isInternalUse: false,\n          },\n          headers: { ...createRTypeRoutingHeader(type) },\n        })\n      ).data!;\n    }\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == \"\")\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === \"\") return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = \"storage://\";\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = \"WrongLocalFileUrl\";\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = \"UnknownStorageError\";\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== \"\") {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"mappings":";;;;;;;;;;;;;AA8BA,IAAa,iBAAb,MAA4B;CAC1B;CACA;;CAGA;;CAGA,uBAAwC,IAAI,4BAA4B,GAAG;CAE3E,YACE,2BACA,YACA,QAEA,kBACA;AAJgB,OAAA,aAAA;AACA,OAAA,SAAA;AAIhB,OAAK,OAAO,0BAA0B,0BAA0B,aAAa;AAC3E,OAAI,SAAS,SAAS,OACpB,QAAO,IAAI,eAAe,SAAS,UAAU;OAE7C,QAAO,QAAQ,aAA+B;IAC5C,aAAa,SAAS,OAAO;IAC7B,KAAK,SAAS,OAAO;IACrB,YAAY,SAAS;IACrB,aAAa,SAAS;IACvB,CAAC;IAEJ;AACF,OAAK,uBAAuB,IAAI,qBAAqB,WAAW;AAChE,OAAK,wBAAwB,yBAAyB,iBAAiB;;CAGzE,QAAQ;;;;;;;CAQR,MAAM,gBACJ,MACA,SACA,KACA,SACY;EACZ,MAAM,EAAE,aAAa,YAAY,MAAM,KAAK,mBAAmB,MAAM,SAAS,IAAI,OAAO;EAEzF,MAAM,gBAAgB,OAAO,YAAY,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,MAAM,CAAC,CAAC;AACzF,OAAK,OAAO,KACV,QAAQ,wBAAwB,KAAK,CAAC,0BAC5B,YAAY,WACV,KAAK,UAAU,IAAI,SAAS,KAAK,GAC9C;EAED,MAAM,QAAQ,UAAU,OAAO;EAC/B,MAAM,SAAS,QAAQ,YAAY,GAC/B,MAAM,KAAK,qBAAqB,aAAa,KAAK,QAAQ,GAC1D,MAAM,KAAK,qBAAqB,YAAY,aAAa,eAAe,KAAK,QAAQ;AAEzF,OAAK,OAAO,KACV,QAAQ,wBAAwB,KAAK,CAAC,4BAAiC,MAAM,SAAS,GACvF;AACD,SAAO;;CAGT,MAAM,qBACJ,KACA,KACA,SACY;EACZ,MAAM,EAAE,WAAW,iBAAiB,cAAc,IAAI;EACtD,MAAM,WAAW,YAAY,WAAW,KAAK,uBAAuB,aAAa;AAEjF,SAAO,MAAM,KAAK,qBAAqB,IAAI,YAAY;GACrD,MAAM,UAAU;IACd,OAAO,IAAI,OAAO;IAClB,KAAK,IAAI,OAAO,OAAO,KAAA,IAAY,IAAI,MAAM,KAAK,IAAI,KAAA;IACtD,QAAQ,IAAI;IACb;GACD,IAAI;GACJ,IAAI,iBAAiB;AAErB,OAAI;IACF,MAAM,OAAO,MAAM,IAAI,KAAK,SAAS;AACrC,aAASA,KAAG,iBAAiB,UAAU,QAAQ;IAG/C,MAAM,SAAS,MAAM,QAFH,SAAS,MAAM,OAAO,EAEA,KAAK,KAAK;AAClD,qBAAiB;AACjB,WAAO;YACA,OAAO;AAEd,QAAI,CAAC,kBAAkB,UAAU,CAAC,OAAO,UACvC,QAAO,SAAS;AAElB,UAAM;;IAER;;CAGJ,MAAc,mBACZ,EAAE,IAAI,QACN,SACA,QAC8C;EAC9C,MAAM,YAAY,WAAW,EAAE;AAC/B,YAAU,QAAQ;EAElB,MAAM,EAAE,UAAU,cAAc,sBAAsB,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAC9B,MAAI,kBAAkB,eACpB,QAAO,MAAM,OAAO,eAClB;GAAE,YAAY;GAAU,mBAAmB;GAAW,eAAe;GAAO,EAC5E,mBAAmB,MAAM,UAAU,CACpC,CAAC;MAEF,SACE,MAAM,OAAO,KAAK,wBAAwB;GACxC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,mBAAmB,qBAAqB,UAAU;IAClD,eAAe;IAChB;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;GAC/C,CAAC,EACF;;;AAKR,SAAgB,cAAc,KAAa;CACzC,MAAM,SAAS,IAAI,IAAI,IAAI;AAC3B,KAAI,OAAO,YAAY,GACrB,OAAM,IAAI,kBAAkB,0BAA0B,IAAI,4BAA4B;AAExF,QAAO;EACL,WAAW,OAAO;EAClB,cAAc,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;EAC3D;;AAGH,SAAgB,YACd,WACA,uBACA,cACA;CACA,MAAM,OAAO,sBAAsB,IAAI,UAAU;AACjD,KAAI,SAAS,KAAA,EAAW,OAAM,IAAI,oBAAoB,6BAA6B,YAAY;AAE/F,KAAI,SAAS,GAAI,QAAO;AAExB,QAAOC,OAAK,KAAK,MAAM,aAAa;;AAGtC,MAAM,kBAAkB;AACxB,SAAS,QAAQ,KAAa;AAC5B,QAAO,IAAI,WAAW,gBAAgB;;;AAIxC,IAAa,oBAAb,cAAuC,MAAM;CAC3C,OAAO;;;AAIT,IAAa,sBAAb,cAAyC,MAAM;CAC7C,OAAO;;AAGT,SAAgB,yBAAyB,aAAuC;CAC9E,MAAM,2BAAgC,IAAI,KAAK;AAC/C,MAAK,MAAM,MAAM,aAAa;AAE5B,MAAI,GAAG,cAAc,GACnB,kBAAiB,GAAG,UAAU;AAEhC,WAAS,IAAI,GAAG,WAAW,GAAG,UAAU;;AAG1C,QAAO"}
+{"version":3,"file":"download.js","names":["fs","path"],"sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { WireClientProvider, WireClientProviderFactory } from \"@milaboratories/pl-client\";\nimport {\n  addRTypeToMetadata,\n  stringifyWithResourceId,\n  RestAPI,\n  createRTypeRoutingHeader,\n  parseSignedResourceId,\n  signatureToBase64Url,\n} from \"@milaboratories/pl-client\";\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { PerfTimer } from \"@milaboratories/helpers\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport { ConcurrencyLimitingExecutor } from \"@milaboratories/ts-helpers\";\nimport type { RpcOptions } from \"@protobuf-ts/runtime-rpc\";\nimport * as fs from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport type { Dispatcher } from \"undici\";\nimport type { LocalStorageProjection } from \"../drivers/types\";\nimport { type ContentHandler, RemoteFileDownloader } from \"../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../helpers/download_errors\";\nimport { validateAbsolute } from \"../helpers/validate\";\nimport type { DownloadAPI_GetDownloadURL_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol\";\nimport { DownloadClient } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client\";\nimport type { DownloadApiPaths, DownloadRestClientType } from \"../proto-rest\";\nimport { type GetContentOptions, type BlobDriverMetrics } from \"@milaboratories/pl-model-common\";\nimport { DownloadUrlCache } from \"./download_url_cache\";\n\n/** Subset of {@link BlobDriverMetrics} owned by the download client (presigned cache + in-flight downloads). */\ntype ClientDownloadMetrics = Omit<BlobDriverMetrics, \"uncachedRequests\" | \"uncachedRequestBytes\">;\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n  public readonly wire: WireClientProvider<DownloadRestClientType | DownloadClient>;\n  private readonly remoteFileDownloader: RemoteFileDownloader;\n\n  /** Helps to find a storage root directory by a storage id from URL scheme. */\n  private readonly localStorageIdsToRoot: Map<string, string>;\n\n  /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n  private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n  /** Caches presigned download URLs by resource id until they (almost) expire. */\n  private readonly urlCache: DownloadUrlCache;\n\n  /** Active remote downloads; in-flight gauges are summed over this set on read. */\n  private readonly inFlight = new Set<{ size: number; received: number }>();\n  private presignedUrlCacheHits = 0;\n  private presignedUrlCacheMisses = 0;\n  private presignedUrlStaleHits = 0;\n  private presignedUrlRequestSumLatencyMs = 0;\n\n  constructor(\n    wireClientProviderFactory: WireClientProviderFactory,\n    public readonly httpClient: Dispatcher,\n    public readonly logger: MiLogger,\n    /** Pl storages available locally */\n    localProjections: LocalStorageProjection[],\n  ) {\n    this.wire = wireClientProviderFactory.createWireClientProvider((wireConn) => {\n      if (wireConn.type === \"grpc\") {\n        return new DownloadClient(wireConn.Transport);\n      } else {\n        return RestAPI.createClient<DownloadApiPaths>({\n          hostAndPort: wireConn.Config.hostAndPort,\n          ssl: wireConn.Config.ssl,\n          dispatcher: wireConn.Dispatcher,\n          middlewares: wireConn.Middlewares,\n        });\n      }\n    });\n    this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n    this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n    this.urlCache = new DownloadUrlCache(logger);\n  }\n\n  close() {}\n\n  /**\n   * Gets a presign URL and downloads the file.\n   * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n   * @param fromBytes - from byte including this byte\n   * @param toBytes - to byte excluding this byte\n   */\n  async withBlobContent<T>(\n    info: ResourceInfo,\n    options: RpcOptions | undefined,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const attempt = async ({\n      downloadUrl,\n      headers,\n    }: DownloadAPI_GetDownloadURL_Response): Promise<T> => {\n      const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n      this.logger.info(\n        `blob ${stringifyWithResourceId(info)} download started, ` +\n          `url: ${downloadUrl}, ` +\n          `range: ${JSON.stringify(ops.range ?? null)}`,\n      );\n\n      const timer = PerfTimer.start();\n      const result = isLocal(downloadUrl)\n        ? await this.withLocalFileContent(downloadUrl, ops, handler)\n        : await this.withTrackedRemoteContent(downloadUrl, remoteHeaders, ops, handler);\n\n      this.logger.info(\n        `blob ${stringifyWithResourceId(info)} download finished, ` + `took: ${timer.elapsed()}`,\n      );\n      return result;\n    };\n\n    const cached = this.urlCache.get(info.id);\n    if (cached !== undefined) {\n      try {\n        const result = await attempt(cached);\n        this.presignedUrlCacheHits++;\n        return result;\n      } catch (error) {\n        if (!isDownloadNetworkError400(error)) throw error;\n        this.urlCache.delete(info.id);\n        this.presignedUrlStaleHits++;\n        this.logger.info(\n          `cached download URL for blob ${stringifyWithResourceId(info)} rejected ` +\n            `(status ${error.statusCode}), re-fetching`,\n        );\n      }\n    } else {\n      this.presignedUrlCacheMisses++;\n    }\n\n    const urlFetchStartMs = performance.now();\n    const fresh = await this.grpcGetDownloadUrl(info, options, ops.signal);\n    this.presignedUrlRequestSumLatencyMs += performance.now() - urlFetchStartMs;\n    this.urlCache.set(info.id, fresh);\n    return await attempt(fresh);\n  }\n\n  /** Download-client-owned slice of {@link BlobDriverMetrics}: presigned-URL cache + in-flight downloads. */\n  metrics(): ClientDownloadMetrics {\n    let inFlightBytesTotal = 0;\n    let inFlightBytesReceived = 0;\n    for (const rec of this.inFlight) {\n      inFlightBytesTotal += rec.size;\n      inFlightBytesReceived += rec.received;\n    }\n    return {\n      downloadsInFlight: this.inFlight.size,\n      inFlightBytesTotal,\n      inFlightBytesReceived,\n      presignedUrlCacheHits: this.presignedUrlCacheHits,\n      presignedUrlCacheMisses: this.presignedUrlCacheMisses,\n      presignedUrlStaleHits: this.presignedUrlStaleHits,\n      presignedUrlRequestSumLatencyMs: this.presignedUrlRequestSumLatencyMs,\n    };\n  }\n\n  /** Wraps a remote download so it appears in the in-flight gauges and its received bytes are counted live. */\n  private async withTrackedRemoteContent<T>(\n    url: string,\n    headers: Record<string, string>,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const rec = { size: 0, received: 0 };\n    this.inFlight.add(rec);\n    try {\n      return await this.remoteFileDownloader.withContent(\n        url,\n        headers,\n        ops,\n        async (content, size) => {\n          rec.size = size;\n          const counted = content.pipeThrough(\n            new TransformStream<Uint8Array, Uint8Array>({\n              transform: (chunk, controller) => {\n                rec.received += chunk.byteLength;\n                controller.enqueue(chunk);\n              },\n            }),\n          );\n          return await handler(counted, size);\n        },\n      );\n    } finally {\n      this.inFlight.delete(rec);\n    }\n  }\n\n  async withLocalFileContent<T>(\n    url: string,\n    ops: GetContentOptions,\n    handler: ContentHandler<T>,\n  ): Promise<T> {\n    const { storageId, relativePath } = parseLocalUrl(url);\n    const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n    return await this.localFileReadLimiter.run(async () => {\n      const readOps = {\n        start: ops.range?.from,\n        end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n        signal: ops.signal,\n      };\n      let stream: fs.ReadStream | undefined;\n      let handlerSuccess = false;\n\n      try {\n        const stat = await fsp.stat(fullPath);\n        stream = fs.createReadStream(fullPath, readOps);\n        const webStream = Readable.toWeb(stream);\n\n        const result = await handler(webStream, stat.size);\n        handlerSuccess = true;\n        return result;\n      } catch (error) {\n        // Cleanup on error (including handler errors)\n        if (!handlerSuccess && stream && !stream.destroyed) {\n          stream.destroy();\n        }\n        throw error;\n      }\n    });\n  }\n\n  private async grpcGetDownloadUrl(\n    { id, type }: ResourceInfo,\n    options?: RpcOptions,\n    signal?: AbortSignal,\n  ): Promise<DownloadAPI_GetDownloadURL_Response> {\n    const withAbort = options ?? {};\n    withAbort.abort = signal;\n\n    const { globalId, signature } = parseSignedResourceId(id);\n    const client = this.wire.get();\n    if (client instanceof DownloadClient) {\n      return await client.getDownloadURL(\n        { resourceId: globalId, resourceSignature: signature, isInternalUse: false },\n        addRTypeToMetadata(type, withAbort),\n      ).response;\n    } else {\n      return (\n        await client.POST(\"/v1/get-download-url\", {\n          body: {\n            resourceId: globalId.toString(),\n            resourceSignature: signatureToBase64Url(signature),\n            isInternalUse: false,\n          },\n          headers: { ...createRTypeRoutingHeader(type) },\n        })\n      ).data!;\n    }\n  }\n}\n\nexport function parseLocalUrl(url: string) {\n  const parsed = new URL(url);\n  if (parsed.pathname == \"\")\n    throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n  return {\n    storageId: parsed.host,\n    relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n  };\n}\n\nexport function getFullPath(\n  storageId: string,\n  localStorageIdsToRoot: Map<string, string>,\n  relativePath: string,\n) {\n  const root = localStorageIdsToRoot.get(storageId);\n  if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n  if (root === \"\") return relativePath;\n\n  return path.join(root, relativePath);\n}\n\nconst storageProtocol = \"storage://\";\nfunction isLocal(url: string) {\n  return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n  name = \"WrongLocalFileUrl\";\n}\n\n/** Happens when a storage for a local file can't be found.  */\nexport class UnknownStorageError extends Error {\n  name = \"UnknownStorageError\";\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n  const idToRoot: Map<string, string> = new Map();\n  for (const lp of projections) {\n    // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n    if (lp.localPath !== \"\") {\n      validateAbsolute(lp.localPath);\n    }\n    idToRoot.set(lp.storageId, lp.localPath);\n  }\n\n  return idToRoot;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAmCA,IAAa,iBAAb,MAA4B;CAC1B;CACA;;CAGA;;CAGA,uBAAwC,IAAI,4BAA4B,GAAG;;CAG3E;;CAGA,2BAA4B,IAAI,KAAyC;CACzE,wBAAgC;CAChC,0BAAkC;CAClC,wBAAgC;CAChC,kCAA0C;CAE1C,YACE,2BACA,YACA,QAEA,kBACA;AAJgB,OAAA,aAAA;AACA,OAAA,SAAA;AAIhB,OAAK,OAAO,0BAA0B,0BAA0B,aAAa;AAC3E,OAAI,SAAS,SAAS,OACpB,QAAO,IAAI,eAAe,SAAS,UAAU;OAE7C,QAAO,QAAQ,aAA+B;IAC5C,aAAa,SAAS,OAAO;IAC7B,KAAK,SAAS,OAAO;IACrB,YAAY,SAAS;IACrB,aAAa,SAAS;IACvB,CAAC;IAEJ;AACF,OAAK,uBAAuB,IAAI,qBAAqB,WAAW;AAChE,OAAK,wBAAwB,yBAAyB,iBAAiB;AACvE,OAAK,WAAW,IAAI,iBAAiB,OAAO;;CAG9C,QAAQ;;;;;;;CAQR,MAAM,gBACJ,MACA,SACA,KACA,SACY;EACZ,MAAM,UAAU,OAAO,EACrB,aACA,cACqD;GACrD,MAAM,gBAAgB,OAAO,YAAY,QAAQ,KAAK,EAAE,MAAM,YAAY,CAAC,MAAM,MAAM,CAAC,CAAC;AACzF,QAAK,OAAO,KACV,QAAQ,wBAAwB,KAAK,CAAC,0BAC5B,YAAY,WACV,KAAK,UAAU,IAAI,SAAS,KAAK,GAC9C;GAED,MAAM,QAAQ,UAAU,OAAO;GAC/B,MAAM,SAAS,QAAQ,YAAY,GAC/B,MAAM,KAAK,qBAAqB,aAAa,KAAK,QAAQ,GAC1D,MAAM,KAAK,yBAAyB,aAAa,eAAe,KAAK,QAAQ;AAEjF,QAAK,OAAO,KACV,QAAQ,wBAAwB,KAAK,CAAC,4BAAiC,MAAM,SAAS,GACvF;AACD,UAAO;;EAGT,MAAM,SAAS,KAAK,SAAS,IAAI,KAAK,GAAG;AACzC,MAAI,WAAW,KAAA,EACb,KAAI;GACF,MAAM,SAAS,MAAM,QAAQ,OAAO;AACpC,QAAK;AACL,UAAO;WACA,OAAO;AACd,OAAI,CAAC,0BAA0B,MAAM,CAAE,OAAM;AAC7C,QAAK,SAAS,OAAO,KAAK,GAAG;AAC7B,QAAK;AACL,QAAK,OAAO,KACV,gCAAgC,wBAAwB,KAAK,CAAC,oBACjD,MAAM,WAAW,gBAC/B;;MAGH,MAAK;EAGP,MAAM,kBAAkB,YAAY,KAAK;EACzC,MAAM,QAAQ,MAAM,KAAK,mBAAmB,MAAM,SAAS,IAAI,OAAO;AACtE,OAAK,mCAAmC,YAAY,KAAK,GAAG;AAC5D,OAAK,SAAS,IAAI,KAAK,IAAI,MAAM;AACjC,SAAO,MAAM,QAAQ,MAAM;;;CAI7B,UAAiC;EAC/B,IAAI,qBAAqB;EACzB,IAAI,wBAAwB;AAC5B,OAAK,MAAM,OAAO,KAAK,UAAU;AAC/B,yBAAsB,IAAI;AAC1B,4BAAyB,IAAI;;AAE/B,SAAO;GACL,mBAAmB,KAAK,SAAS;GACjC;GACA;GACA,uBAAuB,KAAK;GAC5B,yBAAyB,KAAK;GAC9B,uBAAuB,KAAK;GAC5B,iCAAiC,KAAK;GACvC;;;CAIH,MAAc,yBACZ,KACA,SACA,KACA,SACY;EACZ,MAAM,MAAM;GAAE,MAAM;GAAG,UAAU;GAAG;AACpC,OAAK,SAAS,IAAI,IAAI;AACtB,MAAI;AACF,UAAO,MAAM,KAAK,qBAAqB,YACrC,KACA,SACA,KACA,OAAO,SAAS,SAAS;AACvB,QAAI,OAAO;AASX,WAAO,MAAM,QARG,QAAQ,YACtB,IAAI,gBAAwC,EAC1C,YAAY,OAAO,eAAe;AAChC,SAAI,YAAY,MAAM;AACtB,gBAAW,QAAQ,MAAM;OAE5B,CAAC,CACH,EAC6B,KAAK;KAEtC;YACO;AACR,QAAK,SAAS,OAAO,IAAI;;;CAI7B,MAAM,qBACJ,KACA,KACA,SACY;EACZ,MAAM,EAAE,WAAW,iBAAiB,cAAc,IAAI;EACtD,MAAM,WAAW,YAAY,WAAW,KAAK,uBAAuB,aAAa;AAEjF,SAAO,MAAM,KAAK,qBAAqB,IAAI,YAAY;GACrD,MAAM,UAAU;IACd,OAAO,IAAI,OAAO;IAClB,KAAK,IAAI,OAAO,OAAO,KAAA,IAAY,IAAI,MAAM,KAAK,IAAI,KAAA;IACtD,QAAQ,IAAI;IACb;GACD,IAAI;GACJ,IAAI,iBAAiB;AAErB,OAAI;IACF,MAAM,OAAO,MAAM,IAAI,KAAK,SAAS;AACrC,aAASA,KAAG,iBAAiB,UAAU,QAAQ;IAG/C,MAAM,SAAS,MAAM,QAFH,SAAS,MAAM,OAAO,EAEA,KAAK,KAAK;AAClD,qBAAiB;AACjB,WAAO;YACA,OAAO;AAEd,QAAI,CAAC,kBAAkB,UAAU,CAAC,OAAO,UACvC,QAAO,SAAS;AAElB,UAAM;;IAER;;CAGJ,MAAc,mBACZ,EAAE,IAAI,QACN,SACA,QAC8C;EAC9C,MAAM,YAAY,WAAW,EAAE;AAC/B,YAAU,QAAQ;EAElB,MAAM,EAAE,UAAU,cAAc,sBAAsB,GAAG;EACzD,MAAM,SAAS,KAAK,KAAK,KAAK;AAC9B,MAAI,kBAAkB,eACpB,QAAO,MAAM,OAAO,eAClB;GAAE,YAAY;GAAU,mBAAmB;GAAW,eAAe;GAAO,EAC5E,mBAAmB,MAAM,UAAU,CACpC,CAAC;MAEF,SACE,MAAM,OAAO,KAAK,wBAAwB;GACxC,MAAM;IACJ,YAAY,SAAS,UAAU;IAC/B,mBAAmB,qBAAqB,UAAU;IAClD,eAAe;IAChB;GACD,SAAS,EAAE,GAAG,yBAAyB,KAAK,EAAE;GAC/C,CAAC,EACF;;;AAKR,SAAgB,cAAc,KAAa;CACzC,MAAM,SAAS,IAAI,IAAI,IAAI;AAC3B,KAAI,OAAO,YAAY,GACrB,OAAM,IAAI,kBAAkB,0BAA0B,IAAI,4BAA4B;AAExF,QAAO;EACL,WAAW,OAAO;EAClB,cAAc,mBAAmB,OAAO,SAAS,MAAM,EAAE,CAAC;EAC3D;;AAGH,SAAgB,YACd,WACA,uBACA,cACA;CACA,MAAM,OAAO,sBAAsB,IAAI,UAAU;AACjD,KAAI,SAAS,KAAA,EAAW,OAAM,IAAI,oBAAoB,6BAA6B,YAAY;AAE/F,KAAI,SAAS,GAAI,QAAO;AAExB,QAAOC,OAAK,KAAK,MAAM,aAAa;;AAGtC,MAAM,kBAAkB;AACxB,SAAS,QAAQ,KAAa;AAC5B,QAAO,IAAI,WAAW,gBAAgB;;;AAIxC,IAAa,oBAAb,cAAuC,MAAM;CAC3C,OAAO;;;AAIT,IAAa,sBAAb,cAAyC,MAAM;CAC7C,OAAO;;AAGT,SAAgB,yBAAyB,aAAuC;CAC9E,MAAM,2BAAgC,IAAI,KAAK;AAC/C,MAAK,MAAM,MAAM,aAAa;AAE5B,MAAI,GAAG,cAAc,GACnB,kBAAiB,GAAG,UAAU;AAEhC,WAAS,IAAI,GAAG,WAAW,GAAG,UAAU;;AAG1C,QAAO"}

package/dist/clients/download_url_cache.cjs

@@ -0,0 +1,102 @@
+require("../_virtual/_rolldown/runtime.cjs");
+let lru_cache = require("lru-cache");
+//#region src/clients/download_url_cache.ts
+/**
+* Safety margin subtracted from the encoded URL expiry. Covers clock skew
+* between this host and pl-core (the timestamp is the server's signing clock,
+* not ours) plus in-flight time between signing and use.
+*/
+const SAFETY_MARGIN_MS = 3e4;
+/**
+* TTL applied to download URLs that carry no expiry in their query string -
+* notably local `storage://` URLs, which are deterministic for a given resource
+* and effectively never expire. Bounded so a changed storage projection is
+* eventually re-resolved.
+*/
+const NO_EXPIRY_DEFAULT_TTL_MS = 3600 * 1e3;
+/** Max number of cached {url, headers} entries. Each entry is tiny. */
+const DEFAULT_MAX_ENTRIES = 4096;
+/**
+* Extracts the absolute expiry time (epoch ms) encoded in a presigned download
+* URL, or `undefined` if the URL carries no expiry.
+*
+* Supports AWS SigV4 (`X-Amz-Date` + `X-Amz-Expires`, used by pl's S3 and FS
+* remote drivers) and GCS V4 (`X-Goog-Date` + `X-Goog-Expires`). Both encode
+* the date as a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` (trailing `Z`
+* = Zulu/UTC; verified against pl `util/storage/v4sign/presigner.go`) and the
+* lifetime as integer seconds.
+*/
+function extractUrlExpiryMs(url) {
+	let query;
+	try {
+		query = new URL(url).searchParams;
+	} catch {
+		return;
+	}
+	for (const prefix of ["X-Amz", "X-Goog"]) {
+		const date = query.get(`${prefix}-Date`);
+		const expires = query.get(`${prefix}-Expires`);
+		if (date === null || expires === null) continue;
+		const signedAtMs = parseCompactIso8601Utc(date);
+		const expiresSec = Number(expires);
+		if (signedAtMs === void 0 || !Number.isFinite(expiresSec) || expiresSec <= 0) return null;
+		return signedAtMs + expiresSec * 1e3;
+	}
+}
+/**
+* Parses a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` into epoch ms.
+* `new Date()` cannot parse the compact form, so we expand it to the extended
+* form `YYYY-MM-DDTHH:MM:SSZ`; the trailing `Z` makes it UTC regardless of the
+* host's local timezone.
+*/
+function parseCompactIso8601Utc(value) {
+	const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(value);
+	if (m === null) return void 0;
+	const [, y, mo, d, h, mi, s] = m;
+	const ms = Date.parse(`${y}-${mo}-${d}T${h}:${mi}:${s}Z`);
+	return Number.isNaN(ms) ? void 0 : ms;
+}
+/**
+* Computes the cache TTL (ms from now) for a download URL, with the safety
+* margin already subtracted. Returns a non-positive number when the URL is
+* already within the margin of expiring - the caller should then skip caching.
+*/
+function downloadUrlCacheTtlMs(url) {
+	const expiry = extractUrlExpiryMs(url);
+	if (expiry === null) return 0;
+	if (expiry === void 0) return NO_EXPIRY_DEFAULT_TTL_MS;
+	return expiry - Date.now() - SAFETY_MARGIN_MS;
+}
+/**
+* LRU cache of `GetDownloadURL` responses keyed by `SignedResourceId`. Each
+* entry's TTL is derived from the expiry encoded in the presigned URL (minus a
+* safety margin), so an entry never outlives the URL it holds.
+*
+* Note: the key intentionally omits `isInternalUse` because the only caller
+* always requests `isInternalUse: false`. Revisit if that ever varies.
+*/
+var DownloadUrlCache = class {
+	cache;
+	constructor(logger, maxEntries = DEFAULT_MAX_ENTRIES) {
+		this.logger = logger;
+		this.cache = new lru_cache.LRUCache({
+			max: maxEntries,
+			updateAgeOnGet: false
+		});
+	}
+	get(key) {
+		return this.cache.get(key);
+	}
+	set(key, value) {
+		const ttl = downloadUrlCacheTtlMs(value.downloadUrl);
+		if (ttl <= 0) return;
+		this.cache.set(key, value, { ttl });
+	}
+	delete(key) {
+		this.cache.delete(key);
+	}
+};
+//#endregion
+exports.DownloadUrlCache = DownloadUrlCache;
+
+//# sourceMappingURL=download_url_cache.cjs.map

package/dist/clients/download_url_cache.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"download_url_cache.cjs","names":["LRUCache"],"sources":["../../src/clients/download_url_cache.ts"],"sourcesContent":["import { LRUCache } from \"lru-cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { DownloadAPI_GetDownloadURL_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol\";\n\n/**\n * Safety margin subtracted from the encoded URL expiry. Covers clock skew\n * between this host and pl-core (the timestamp is the server's signing clock,\n * not ours) plus in-flight time between signing and use.\n */\nconst SAFETY_MARGIN_MS = 30_000;\n\n/**\n * TTL applied to download URLs that carry no expiry in their query string -\n * notably local `storage://` URLs, which are deterministic for a given resource\n * and effectively never expire. Bounded so a changed storage projection is\n * eventually re-resolved.\n */\nconst NO_EXPIRY_DEFAULT_TTL_MS = 60 * 60 * 1000; // 1h\n\n/** Max number of cached {url, headers} entries. Each entry is tiny. */\nconst DEFAULT_MAX_ENTRIES = 4096;\n\n/**\n * Extracts the absolute expiry time (epoch ms) encoded in a presigned download\n * URL, or `undefined` if the URL carries no expiry.\n *\n * Supports AWS SigV4 (`X-Amz-Date` + `X-Amz-Expires`, used by pl's S3 and FS\n * remote drivers) and GCS V4 (`X-Goog-Date` + `X-Goog-Expires`). Both encode\n * the date as a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` (trailing `Z`\n * = Zulu/UTC; verified against pl `util/storage/v4sign/presigner.go`) and the\n * lifetime as integer seconds.\n */\nexport function extractUrlExpiryMs(url: string): number | null | undefined {\n  let query: URLSearchParams;\n  try {\n    query = new URL(url).searchParams;\n  } catch {\n    return undefined;\n  }\n\n  for (const prefix of [\"X-Amz\", \"X-Goog\"]) {\n    const date = query.get(`${prefix}-Date`);\n    const expires = query.get(`${prefix}-Expires`);\n    if (date === null || expires === null) continue;\n\n    const signedAtMs = parseCompactIso8601Utc(date);\n    const expiresSec = Number(expires);\n    if (signedAtMs === undefined || !Number.isFinite(expiresSec) || expiresSec <= 0) return null;\n\n    return signedAtMs + expiresSec * 1000;\n  }\n\n  return undefined;\n}\n\n/**\n * Parses a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` into epoch ms.\n * `new Date()` cannot parse the compact form, so we expand it to the extended\n * form `YYYY-MM-DDTHH:MM:SSZ`; the trailing `Z` makes it UTC regardless of the\n * host's local timezone.\n */\nfunction parseCompactIso8601Utc(value: string): number | undefined {\n  const m = /^(\\d{4})(\\d{2})(\\d{2})T(\\d{2})(\\d{2})(\\d{2})Z$/.exec(value);\n  if (m === null) return undefined;\n  const [, y, mo, d, h, mi, s] = m;\n  const ms = Date.parse(`${y}-${mo}-${d}T${h}:${mi}:${s}Z`);\n  return Number.isNaN(ms) ? undefined : ms;\n}\n\n/**\n * Computes the cache TTL (ms from now) for a download URL, with the safety\n * margin already subtracted. Returns a non-positive number when the URL is\n * already within the margin of expiring - the caller should then skip caching.\n */\nexport function downloadUrlCacheTtlMs(url: string): number {\n  const expiry = extractUrlExpiryMs(url);\n  if (expiry === null) return 0;\n  if (expiry === undefined) return NO_EXPIRY_DEFAULT_TTL_MS;\n  return expiry - Date.now() - SAFETY_MARGIN_MS;\n}\n\n/**\n * LRU cache of `GetDownloadURL` responses keyed by `SignedResourceId`. Each\n * entry's TTL is derived from the expiry encoded in the presigned URL (minus a\n * safety margin), so an entry never outlives the URL it holds.\n *\n * Note: the key intentionally omits `isInternalUse` because the only caller\n * always requests `isInternalUse: false`. Revisit if that ever varies.\n */\nexport class DownloadUrlCache {\n  private readonly cache: LRUCache<SignedResourceId, DownloadAPI_GetDownloadURL_Response>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    maxEntries: number = DEFAULT_MAX_ENTRIES,\n  ) {\n    this.cache = new LRUCache<SignedResourceId, DownloadAPI_GetDownloadURL_Response>({\n      max: maxEntries,\n      // URL expiry is absolute, not sliding - do not extend TTL on access.\n      updateAgeOnGet: false,\n    });\n  }\n\n  get(key: SignedResourceId): DownloadAPI_GetDownloadURL_Response | undefined {\n    return this.cache.get(key);\n  }\n\n  set(key: SignedResourceId, value: DownloadAPI_GetDownloadURL_Response): void {\n    const ttl = downloadUrlCacheTtlMs(value.downloadUrl);\n    if (ttl <= 0) return; // Cache miss.\n    this.cache.set(key, value, { ttl });\n  }\n\n  delete(key: SignedResourceId): void {\n    this.cache.delete(key);\n  }\n}\n"],"mappings":";;;;;;;;AAUA,MAAM,mBAAmB;;;;;;;AAQzB,MAAM,2BAA2B,OAAU;;AAG3C,MAAM,sBAAsB;;;;;;;;;;;AAY5B,SAAgB,mBAAmB,KAAwC;CACzE,IAAI;AACJ,KAAI;AACF,UAAQ,IAAI,IAAI,IAAI,CAAC;SACf;AACN;;AAGF,MAAK,MAAM,UAAU,CAAC,SAAS,SAAS,EAAE;EACxC,MAAM,OAAO,MAAM,IAAI,GAAG,OAAO,OAAO;EACxC,MAAM,UAAU,MAAM,IAAI,GAAG,OAAO,UAAU;AAC9C,MAAI,SAAS,QAAQ,YAAY,KAAM;EAEvC,MAAM,aAAa,uBAAuB,KAAK;EAC/C,MAAM,aAAa,OAAO,QAAQ;AAClC,MAAI,eAAe,KAAA,KAAa,CAAC,OAAO,SAAS,WAAW,IAAI,cAAc,EAAG,QAAO;AAExF,SAAO,aAAa,aAAa;;;;;;;;;AAYrC,SAAS,uBAAuB,OAAmC;CACjE,MAAM,IAAI,iDAAiD,KAAK,MAAM;AACtE,KAAI,MAAM,KAAM,QAAO,KAAA;CACvB,MAAM,GAAG,GAAG,IAAI,GAAG,GAAG,IAAI,KAAK;CAC/B,MAAM,KAAK,KAAK,MAAM,GAAG,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG;AACzD,QAAO,OAAO,MAAM,GAAG,GAAG,KAAA,IAAY;;;;;;;AAQxC,SAAgB,sBAAsB,KAAqB;CACzD,MAAM,SAAS,mBAAmB,IAAI;AACtC,KAAI,WAAW,KAAM,QAAO;AAC5B,KAAI,WAAW,KAAA,EAAW,QAAO;AACjC,QAAO,SAAS,KAAK,KAAK,GAAG;;;;;;;;;;AAW/B,IAAa,mBAAb,MAA8B;CAC5B;CAEA,YACE,QACA,aAAqB,qBACrB;AAFiB,OAAA,SAAA;AAGjB,OAAK,QAAQ,IAAIA,UAAAA,SAAgE;GAC/E,KAAK;GAEL,gBAAgB;GACjB,CAAC;;CAGJ,IAAI,KAAwE;AAC1E,SAAO,KAAK,MAAM,IAAI,IAAI;;CAG5B,IAAI,KAAuB,OAAkD;EAC3E,MAAM,MAAM,sBAAsB,MAAM,YAAY;AACpD,MAAI,OAAO,EAAG;AACd,OAAK,MAAM,IAAI,KAAK,OAAO,EAAE,KAAK,CAAC;;CAGrC,OAAO,KAA6B;AAClC,OAAK,MAAM,OAAO,IAAI"}

package/dist/clients/download_url_cache.js

@@ -0,0 +1,101 @@
+import { LRUCache } from "lru-cache";
+//#region src/clients/download_url_cache.ts
+/**
+* Safety margin subtracted from the encoded URL expiry. Covers clock skew
+* between this host and pl-core (the timestamp is the server's signing clock,
+* not ours) plus in-flight time between signing and use.
+*/
+const SAFETY_MARGIN_MS = 3e4;
+/**
+* TTL applied to download URLs that carry no expiry in their query string -
+* notably local `storage://` URLs, which are deterministic for a given resource
+* and effectively never expire. Bounded so a changed storage projection is
+* eventually re-resolved.
+*/
+const NO_EXPIRY_DEFAULT_TTL_MS = 3600 * 1e3;
+/** Max number of cached {url, headers} entries. Each entry is tiny. */
+const DEFAULT_MAX_ENTRIES = 4096;
+/**
+* Extracts the absolute expiry time (epoch ms) encoded in a presigned download
+* URL, or `undefined` if the URL carries no expiry.
+*
+* Supports AWS SigV4 (`X-Amz-Date` + `X-Amz-Expires`, used by pl's S3 and FS
+* remote drivers) and GCS V4 (`X-Goog-Date` + `X-Goog-Expires`). Both encode
+* the date as a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` (trailing `Z`
+* = Zulu/UTC; verified against pl `util/storage/v4sign/presigner.go`) and the
+* lifetime as integer seconds.
+*/
+function extractUrlExpiryMs(url) {
+	let query;
+	try {
+		query = new URL(url).searchParams;
+	} catch {
+		return;
+	}
+	for (const prefix of ["X-Amz", "X-Goog"]) {
+		const date = query.get(`${prefix}-Date`);
+		const expires = query.get(`${prefix}-Expires`);
+		if (date === null || expires === null) continue;
+		const signedAtMs = parseCompactIso8601Utc(date);
+		const expiresSec = Number(expires);
+		if (signedAtMs === void 0 || !Number.isFinite(expiresSec) || expiresSec <= 0) return null;
+		return signedAtMs + expiresSec * 1e3;
+	}
+}
+/**
+* Parses a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` into epoch ms.
+* `new Date()` cannot parse the compact form, so we expand it to the extended
+* form `YYYY-MM-DDTHH:MM:SSZ`; the trailing `Z` makes it UTC regardless of the
+* host's local timezone.
+*/
+function parseCompactIso8601Utc(value) {
+	const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(value);
+	if (m === null) return void 0;
+	const [, y, mo, d, h, mi, s] = m;
+	const ms = Date.parse(`${y}-${mo}-${d}T${h}:${mi}:${s}Z`);
+	return Number.isNaN(ms) ? void 0 : ms;
+}
+/**
+* Computes the cache TTL (ms from now) for a download URL, with the safety
+* margin already subtracted. Returns a non-positive number when the URL is
+* already within the margin of expiring - the caller should then skip caching.
+*/
+function downloadUrlCacheTtlMs(url) {
+	const expiry = extractUrlExpiryMs(url);
+	if (expiry === null) return 0;
+	if (expiry === void 0) return NO_EXPIRY_DEFAULT_TTL_MS;
+	return expiry - Date.now() - SAFETY_MARGIN_MS;
+}
+/**
+* LRU cache of `GetDownloadURL` responses keyed by `SignedResourceId`. Each
+* entry's TTL is derived from the expiry encoded in the presigned URL (minus a
+* safety margin), so an entry never outlives the URL it holds.
+*
+* Note: the key intentionally omits `isInternalUse` because the only caller
+* always requests `isInternalUse: false`. Revisit if that ever varies.
+*/
+var DownloadUrlCache = class {
+	cache;
+	constructor(logger, maxEntries = DEFAULT_MAX_ENTRIES) {
+		this.logger = logger;
+		this.cache = new LRUCache({
+			max: maxEntries,
+			updateAgeOnGet: false
+		});
+	}
+	get(key) {
+		return this.cache.get(key);
+	}
+	set(key, value) {
+		const ttl = downloadUrlCacheTtlMs(value.downloadUrl);
+		if (ttl <= 0) return;
+		this.cache.set(key, value, { ttl });
+	}
+	delete(key) {
+		this.cache.delete(key);
+	}
+};
+//#endregion
+export { DownloadUrlCache };
+
+//# sourceMappingURL=download_url_cache.js.map

package/dist/clients/download_url_cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"download_url_cache.js","names":[],"sources":["../../src/clients/download_url_cache.ts"],"sourcesContent":["import { LRUCache } from \"lru-cache\";\nimport type { SignedResourceId } from \"@milaboratories/pl-client\";\nimport type { MiLogger } from \"@milaboratories/ts-helpers\";\nimport type { DownloadAPI_GetDownloadURL_Response } from \"../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol\";\n\n/**\n * Safety margin subtracted from the encoded URL expiry. Covers clock skew\n * between this host and pl-core (the timestamp is the server's signing clock,\n * not ours) plus in-flight time between signing and use.\n */\nconst SAFETY_MARGIN_MS = 30_000;\n\n/**\n * TTL applied to download URLs that carry no expiry in their query string -\n * notably local `storage://` URLs, which are deterministic for a given resource\n * and effectively never expire. Bounded so a changed storage projection is\n * eventually re-resolved.\n */\nconst NO_EXPIRY_DEFAULT_TTL_MS = 60 * 60 * 1000; // 1h\n\n/** Max number of cached {url, headers} entries. Each entry is tiny. */\nconst DEFAULT_MAX_ENTRIES = 4096;\n\n/**\n * Extracts the absolute expiry time (epoch ms) encoded in a presigned download\n * URL, or `undefined` if the URL carries no expiry.\n *\n * Supports AWS SigV4 (`X-Amz-Date` + `X-Amz-Expires`, used by pl's S3 and FS\n * remote drivers) and GCS V4 (`X-Goog-Date` + `X-Goog-Expires`). Both encode\n * the date as a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` (trailing `Z`\n * = Zulu/UTC; verified against pl `util/storage/v4sign/presigner.go`) and the\n * lifetime as integer seconds.\n */\nexport function extractUrlExpiryMs(url: string): number | null | undefined {\n  let query: URLSearchParams;\n  try {\n    query = new URL(url).searchParams;\n  } catch {\n    return undefined;\n  }\n\n  for (const prefix of [\"X-Amz\", \"X-Goog\"]) {\n    const date = query.get(`${prefix}-Date`);\n    const expires = query.get(`${prefix}-Expires`);\n    if (date === null || expires === null) continue;\n\n    const signedAtMs = parseCompactIso8601Utc(date);\n    const expiresSec = Number(expires);\n    if (signedAtMs === undefined || !Number.isFinite(expiresSec) || expiresSec <= 0) return null;\n\n    return signedAtMs + expiresSec * 1000;\n  }\n\n  return undefined;\n}\n\n/**\n * Parses a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` into epoch ms.\n * `new Date()` cannot parse the compact form, so we expand it to the extended\n * form `YYYY-MM-DDTHH:MM:SSZ`; the trailing `Z` makes it UTC regardless of the\n * host's local timezone.\n */\nfunction parseCompactIso8601Utc(value: string): number | undefined {\n  const m = /^(\\d{4})(\\d{2})(\\d{2})T(\\d{2})(\\d{2})(\\d{2})Z$/.exec(value);\n  if (m === null) return undefined;\n  const [, y, mo, d, h, mi, s] = m;\n  const ms = Date.parse(`${y}-${mo}-${d}T${h}:${mi}:${s}Z`);\n  return Number.isNaN(ms) ? undefined : ms;\n}\n\n/**\n * Computes the cache TTL (ms from now) for a download URL, with the safety\n * margin already subtracted. Returns a non-positive number when the URL is\n * already within the margin of expiring - the caller should then skip caching.\n */\nexport function downloadUrlCacheTtlMs(url: string): number {\n  const expiry = extractUrlExpiryMs(url);\n  if (expiry === null) return 0;\n  if (expiry === undefined) return NO_EXPIRY_DEFAULT_TTL_MS;\n  return expiry - Date.now() - SAFETY_MARGIN_MS;\n}\n\n/**\n * LRU cache of `GetDownloadURL` responses keyed by `SignedResourceId`. Each\n * entry's TTL is derived from the expiry encoded in the presigned URL (minus a\n * safety margin), so an entry never outlives the URL it holds.\n *\n * Note: the key intentionally omits `isInternalUse` because the only caller\n * always requests `isInternalUse: false`. Revisit if that ever varies.\n */\nexport class DownloadUrlCache {\n  private readonly cache: LRUCache<SignedResourceId, DownloadAPI_GetDownloadURL_Response>;\n\n  constructor(\n    private readonly logger: MiLogger,\n    maxEntries: number = DEFAULT_MAX_ENTRIES,\n  ) {\n    this.cache = new LRUCache<SignedResourceId, DownloadAPI_GetDownloadURL_Response>({\n      max: maxEntries,\n      // URL expiry is absolute, not sliding - do not extend TTL on access.\n      updateAgeOnGet: false,\n    });\n  }\n\n  get(key: SignedResourceId): DownloadAPI_GetDownloadURL_Response | undefined {\n    return this.cache.get(key);\n  }\n\n  set(key: SignedResourceId, value: DownloadAPI_GetDownloadURL_Response): void {\n    const ttl = downloadUrlCacheTtlMs(value.downloadUrl);\n    if (ttl <= 0) return; // Cache miss.\n    this.cache.set(key, value, { ttl });\n  }\n\n  delete(key: SignedResourceId): void {\n    this.cache.delete(key);\n  }\n}\n"],"mappings":";;;;;;;AAUA,MAAM,mBAAmB;;;;;;;AAQzB,MAAM,2BAA2B,OAAU;;AAG3C,MAAM,sBAAsB;;;;;;;;;;;AAY5B,SAAgB,mBAAmB,KAAwC;CACzE,IAAI;AACJ,KAAI;AACF,UAAQ,IAAI,IAAI,IAAI,CAAC;SACf;AACN;;AAGF,MAAK,MAAM,UAAU,CAAC,SAAS,SAAS,EAAE;EACxC,MAAM,OAAO,MAAM,IAAI,GAAG,OAAO,OAAO;EACxC,MAAM,UAAU,MAAM,IAAI,GAAG,OAAO,UAAU;AAC9C,MAAI,SAAS,QAAQ,YAAY,KAAM;EAEvC,MAAM,aAAa,uBAAuB,KAAK;EAC/C,MAAM,aAAa,OAAO,QAAQ;AAClC,MAAI,eAAe,KAAA,KAAa,CAAC,OAAO,SAAS,WAAW,IAAI,cAAc,EAAG,QAAO;AAExF,SAAO,aAAa,aAAa;;;;;;;;;AAYrC,SAAS,uBAAuB,OAAmC;CACjE,MAAM,IAAI,iDAAiD,KAAK,MAAM;AACtE,KAAI,MAAM,KAAM,QAAO,KAAA;CACvB,MAAM,GAAG,GAAG,IAAI,GAAG,GAAG,IAAI,KAAK;CAC/B,MAAM,KAAK,KAAK,MAAM,GAAG,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG;AACzD,QAAO,OAAO,MAAM,GAAG,GAAG,KAAA,IAAY;;;;;;;AAQxC,SAAgB,sBAAsB,KAAqB;CACzD,MAAM,SAAS,mBAAmB,IAAI;AACtC,KAAI,WAAW,KAAM,QAAO;AAC5B,KAAI,WAAW,KAAA,EAAW,QAAO;AACjC,QAAO,SAAS,KAAK,KAAK,GAAG;;;;;;;;;;AAW/B,IAAa,mBAAb,MAA8B;CAC5B;CAEA,YACE,QACA,aAAqB,qBACrB;AAFiB,OAAA,SAAA;AAGjB,OAAK,QAAQ,IAAI,SAAgE;GAC/E,KAAK;GAEL,gBAAgB;GACjB,CAAC;;CAGJ,IAAI,KAAwE;AAC1E,SAAO,KAAK,MAAM,IAAI,IAAI;;CAG5B,IAAI,KAAuB,OAAkD;EAC3E,MAAM,MAAM,sBAAsB,MAAM,YAAY;AACpD,MAAI,OAAO,EAAG;AACd,OAAK,MAAM,IAAI,KAAK,OAAO,EAAE,KAAK,CAAC;;CAGrC,OAAO,KAA6B;AAClC,OAAK,MAAM,OAAO,IAAI"}