@milaboratories/pl-drivers 1.15.6 → 1.16.1

package/src/clients/download.ts

@@ -20,11 +20,16 @@ import { Readable } from "node:stream";
 import type { Dispatcher } from "undici";
 import type { LocalStorageProjection } from "../drivers/types";
 import { type ContentHandler, RemoteFileDownloader } from "../helpers/download";
+import { isDownloadNetworkError400 } from "../helpers/download_errors";
 import { validateAbsolute } from "../helpers/validate";
 import type { DownloadAPI_GetDownloadURL_Response } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol";
 import { DownloadClient } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client";
 import type { DownloadApiPaths, DownloadRestClientType } from "../proto-rest";
-import { type GetContentOptions } from "@milaboratories/pl-model-common";
+import { type GetContentOptions, type BlobDriverMetrics } from "@milaboratories/pl-model-common";
+import { DownloadUrlCache } from "./download_url_cache";
+
+/** Subset of {@link BlobDriverMetrics} owned by the download client (presigned cache + in-flight downloads). */
+type ClientDownloadMetrics = Omit<BlobDriverMetrics, "uncachedRequests" | "uncachedRequestBytes">;
 
 /** Gets URLs for downloading from pl-core, parses them and reads or downloads
  * files locally and from the web. */
@@ -38,6 +43,16 @@ export class ClientDownload {
   /** Concurrency limiter for local file reads - limit to 32 parallel reads */
   private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);
 
+  /** Caches presigned download URLs by resource id until they (almost) expire. */
+  private readonly urlCache: DownloadUrlCache;
+
+  /** Active remote downloads; in-flight gauges are summed over this set on read. */
+  private readonly inFlight = new Set<{ size: number; received: number }>();
+  private presignedUrlCacheHits = 0;
+  private presignedUrlCacheMisses = 0;
+  private presignedUrlStaleHits = 0;
+  private presignedUrlRequestSumLatencyMs = 0;
+
   constructor(
     wireClientProviderFactory: WireClientProviderFactory,
     public readonly httpClient: Dispatcher,
@@ -59,6 +74,7 @@ export class ClientDownload {
     });
     this.remoteFileDownloader = new RemoteFileDownloader(httpClient);
     this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);
+    this.urlCache = new DownloadUrlCache(logger);
   }
 
   close() {}
@@ -75,24 +91,103 @@ export class ClientDownload {
     ops: GetContentOptions,
     handler: ContentHandler<T>,
   ): Promise<T> {
-    const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
-
-    const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
-    this.logger.info(
-      `blob ${stringifyWithResourceId(info)} download started, ` +
-        `url: ${downloadUrl}, ` +
-        `range: ${JSON.stringify(ops.range ?? null)}`,
-    );
-
-    const timer = PerfTimer.start();
-    const result = isLocal(downloadUrl)
-      ? await this.withLocalFileContent(downloadUrl, ops, handler)
-      : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
-
-    this.logger.info(
-      `blob ${stringifyWithResourceId(info)} download finished, ` + `took: ${timer.elapsed()}`,
-    );
-    return result;
+    const attempt = async ({
+      downloadUrl,
+      headers,
+    }: DownloadAPI_GetDownloadURL_Response): Promise<T> => {
+      const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
+      this.logger.info(
+        `blob ${stringifyWithResourceId(info)} download started, ` +
+          `url: ${downloadUrl}, ` +
+          `range: ${JSON.stringify(ops.range ?? null)}`,
+      );
+
+      const timer = PerfTimer.start();
+      const result = isLocal(downloadUrl)
+        ? await this.withLocalFileContent(downloadUrl, ops, handler)
+        : await this.withTrackedRemoteContent(downloadUrl, remoteHeaders, ops, handler);
+
+      this.logger.info(
+        `blob ${stringifyWithResourceId(info)} download finished, ` + `took: ${timer.elapsed()}`,
+      );
+      return result;
+    };
+
+    const cached = this.urlCache.get(info.id);
+    if (cached !== undefined) {
+      try {
+        const result = await attempt(cached);
+        this.presignedUrlCacheHits++;
+        return result;
+      } catch (error) {
+        if (!isDownloadNetworkError400(error)) throw error;
+        this.urlCache.delete(info.id);
+        this.presignedUrlStaleHits++;
+        this.logger.info(
+          `cached download URL for blob ${stringifyWithResourceId(info)} rejected ` +
+            `(status ${error.statusCode}), re-fetching`,
+        );
+      }
+    } else {
+      this.presignedUrlCacheMisses++;
+    }
+
+    const urlFetchStartMs = performance.now();
+    const fresh = await this.grpcGetDownloadUrl(info, options, ops.signal);
+    this.presignedUrlRequestSumLatencyMs += performance.now() - urlFetchStartMs;
+    this.urlCache.set(info.id, fresh);
+    return await attempt(fresh);
+  }
+
+  /** Download-client-owned slice of {@link BlobDriverMetrics}: presigned-URL cache + in-flight downloads. */
+  metrics(): ClientDownloadMetrics {
+    let inFlightBytesTotal = 0;
+    let inFlightBytesReceived = 0;
+    for (const rec of this.inFlight) {
+      inFlightBytesTotal += rec.size;
+      inFlightBytesReceived += rec.received;
+    }
+    return {
+      downloadsInFlight: this.inFlight.size,
+      inFlightBytesTotal,
+      inFlightBytesReceived,
+      presignedUrlCacheHits: this.presignedUrlCacheHits,
+      presignedUrlCacheMisses: this.presignedUrlCacheMisses,
+      presignedUrlStaleHits: this.presignedUrlStaleHits,
+      presignedUrlRequestSumLatencyMs: this.presignedUrlRequestSumLatencyMs,
+    };
+  }
+
+  /** Wraps a remote download so it appears in the in-flight gauges and its received bytes are counted live. */
+  private async withTrackedRemoteContent<T>(
+    url: string,
+    headers: Record<string, string>,
+    ops: GetContentOptions,
+    handler: ContentHandler<T>,
+  ): Promise<T> {
+    const rec = { size: 0, received: 0 };
+    this.inFlight.add(rec);
+    try {
+      return await this.remoteFileDownloader.withContent(
+        url,
+        headers,
+        ops,
+        async (content, size) => {
+          rec.size = size;
+          const counted = content.pipeThrough(
+            new TransformStream<Uint8Array, Uint8Array>({
+              transform: (chunk, controller) => {
+                rec.received += chunk.byteLength;
+                controller.enqueue(chunk);
+              },
+            }),
+          );
+          return await handler(counted, size);
+        },
+      );
+    } finally {
+      this.inFlight.delete(rec);
+    }
   }
 
   async withLocalFileContent<T>(

package/src/clients/download_url_cache.test.ts

@@ -0,0 +1,68 @@
+import { test, expect, describe, vi, afterEach } from "vitest";
+import { downloadUrlCacheTtlMs, extractUrlExpiryMs } from "./download_url_cache";
+
+// 2026-06-16T12:00:00Z, expressed timezone-independently.
+const SIGNED_AT = Date.UTC(2026, 5, 16, 12, 0, 0);
+
+describe("extractUrlExpiryMs", () => {
+  test("AWS SigV4 URL (S3 / FS remote driver): X-Amz-Date + X-Amz-Expires", () => {
+    const url =
+      "https://bucket.s3.amazonaws.com/key?X-Amz-Algorithm=AWS4-HMAC-SHA256" +
+      "&X-Amz-Date=20260616T120000Z&X-Amz-Expires=3600&X-Amz-Signature=deadbeef";
+    expect(extractUrlExpiryMs(url)).toBe(SIGNED_AT + 3600_000);
+  });
+
+  test("GCS V4 signed URL: X-Goog-Date + X-Goog-Expires", () => {
+    const url =
+      "https://storage.googleapis.com/bucket/key?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
+      "&X-Goog-Date=20260616T120000Z&X-Goog-Expires=600&X-Goog-Signature=deadbeef";
+    expect(extractUrlExpiryMs(url)).toBe(SIGNED_AT + 600_000);
+  });
+
+  test("the encoded timestamp is UTC, independent of host timezone", () => {
+    // The compact form ends with 'Z' (Zulu/UTC); the result must equal the
+    // UTC epoch and never shift by the runner's local offset.
+    const url = "https://x/key?X-Amz-Date=20260101T000000Z&X-Amz-Expires=1&X-Amz-Signature=x";
+    expect(extractUrlExpiryMs(url)).toBe(Date.UTC(2026, 0, 1, 0, 0, 0) + 1000);
+  });
+
+  test("local storage:// URL has no encoded expiry", () => {
+    expect(extractUrlExpiryMs("storage://main/some/relative/path.json")).toBeUndefined();
+  });
+
+  test("plain URL without presign params has no expiry", () => {
+    expect(extractUrlExpiryMs("https://example.com/file?foo=bar")).toBeUndefined();
+  });
+
+  test("malformed presign params yield null (present but invalid - do not cache)", () => {
+    const url = "https://x/key?X-Amz-Date=not-a-date&X-Amz-Expires=3600&X-Amz-Signature=x";
+    expect(extractUrlExpiryMs(url)).toBeNull();
+  });
+
+  test("non-URL input yields no expiry", () => {
+    expect(extractUrlExpiryMs("not a url")).toBeUndefined();
+  });
+});
+
+describe("downloadUrlCacheTtlMs", () => {
+  // downloadUrlCacheTtlMs reads Date.now(); pin it so the TTL math is deterministic.
+  afterEach(() => vi.useRealTimers());
+
+  test("subtracts the 30s safety margin from the encoded expiry", () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(SIGNED_AT); // now = signing time -> raw remaining 3600s, minus 30s margin.
+    const url = "https://x/key?X-Amz-Date=20260616T120000Z&X-Amz-Expires=3600&X-Amz-Signature=x";
+    expect(downloadUrlCacheTtlMs(url)).toBe(3600_000 - 30_000);
+  });
+
+  test("returns non-positive when within the safety margin of expiry", () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(SIGNED_AT + 50_000); // 10s of life left -> after a 30s margin, not worth caching.
+    const url = "https://x/key?X-Amz-Date=20260616T120000Z&X-Amz-Expires=60&X-Amz-Signature=x";
+    expect(downloadUrlCacheTtlMs(url)).toBeLessThanOrEqual(0);
+  });
+
+  test("falls back to a positive default TTL for URLs without encoded expiry", () => {
+    expect(downloadUrlCacheTtlMs("storage://main/path")).toBeGreaterThan(0);
+  });
+});

package/src/clients/download_url_cache.ts

@@ -0,0 +1,118 @@
+import { LRUCache } from "lru-cache";
+import type { SignedResourceId } from "@milaboratories/pl-client";
+import type { MiLogger } from "@milaboratories/ts-helpers";
+import type { DownloadAPI_GetDownloadURL_Response } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol";
+
+/**
+ * Safety margin subtracted from the encoded URL expiry. Covers clock skew
+ * between this host and pl-core (the timestamp is the server's signing clock,
+ * not ours) plus in-flight time between signing and use.
+ */
+const SAFETY_MARGIN_MS = 30_000;
+
+/**
+ * TTL applied to download URLs that carry no expiry in their query string -
+ * notably local `storage://` URLs, which are deterministic for a given resource
+ * and effectively never expire. Bounded so a changed storage projection is
+ * eventually re-resolved.
+ */
+const NO_EXPIRY_DEFAULT_TTL_MS = 60 * 60 * 1000; // 1h
+
+/** Max number of cached {url, headers} entries. Each entry is tiny. */
+const DEFAULT_MAX_ENTRIES = 4096;
+
+/**
+ * Extracts the absolute expiry time (epoch ms) encoded in a presigned download
+ * URL, or `undefined` if the URL carries no expiry.
+ *
+ * Supports AWS SigV4 (`X-Amz-Date` + `X-Amz-Expires`, used by pl's S3 and FS
+ * remote drivers) and GCS V4 (`X-Goog-Date` + `X-Goog-Expires`). Both encode
+ * the date as a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` (trailing `Z`
+ * = Zulu/UTC; verified against pl `util/storage/v4sign/presigner.go`) and the
+ * lifetime as integer seconds.
+ */
+export function extractUrlExpiryMs(url: string): number | null | undefined {
+  let query: URLSearchParams;
+  try {
+    query = new URL(url).searchParams;
+  } catch {
+    return undefined;
+  }
+
+  for (const prefix of ["X-Amz", "X-Goog"]) {
+    const date = query.get(`${prefix}-Date`);
+    const expires = query.get(`${prefix}-Expires`);
+    if (date === null || expires === null) continue;
+
+    const signedAtMs = parseCompactIso8601Utc(date);
+    const expiresSec = Number(expires);
+    if (signedAtMs === undefined || !Number.isFinite(expiresSec) || expiresSec <= 0) return null;
+
+    return signedAtMs + expiresSec * 1000;
+  }
+
+  return undefined;
+}
+
+/**
+ * Parses a compact ISO-8601 UTC timestamp `YYYYMMDDTHHMMSSZ` into epoch ms.
+ * `new Date()` cannot parse the compact form, so we expand it to the extended
+ * form `YYYY-MM-DDTHH:MM:SSZ`; the trailing `Z` makes it UTC regardless of the
+ * host's local timezone.
+ */
+function parseCompactIso8601Utc(value: string): number | undefined {
+  const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(value);
+  if (m === null) return undefined;
+  const [, y, mo, d, h, mi, s] = m;
+  const ms = Date.parse(`${y}-${mo}-${d}T${h}:${mi}:${s}Z`);
+  return Number.isNaN(ms) ? undefined : ms;
+}
+
+/**
+ * Computes the cache TTL (ms from now) for a download URL, with the safety
+ * margin already subtracted. Returns a non-positive number when the URL is
+ * already within the margin of expiring - the caller should then skip caching.
+ */
+export function downloadUrlCacheTtlMs(url: string): number {
+  const expiry = extractUrlExpiryMs(url);
+  if (expiry === null) return 0;
+  if (expiry === undefined) return NO_EXPIRY_DEFAULT_TTL_MS;
+  return expiry - Date.now() - SAFETY_MARGIN_MS;
+}
+
+/**
+ * LRU cache of `GetDownloadURL` responses keyed by `SignedResourceId`. Each
+ * entry's TTL is derived from the expiry encoded in the presigned URL (minus a
+ * safety margin), so an entry never outlives the URL it holds.
+ *
+ * Note: the key intentionally omits `isInternalUse` because the only caller
+ * always requests `isInternalUse: false`. Revisit if that ever varies.
+ */
+export class DownloadUrlCache {
+  private readonly cache: LRUCache<SignedResourceId, DownloadAPI_GetDownloadURL_Response>;
+
+  constructor(
+    private readonly logger: MiLogger,
+    maxEntries: number = DEFAULT_MAX_ENTRIES,
+  ) {
+    this.cache = new LRUCache<SignedResourceId, DownloadAPI_GetDownloadURL_Response>({
+      max: maxEntries,
+      // URL expiry is absolute, not sliding - do not extend TTL on access.
+      updateAgeOnGet: false,
+    });
+  }
+
+  get(key: SignedResourceId): DownloadAPI_GetDownloadURL_Response | undefined {
+    return this.cache.get(key);
+  }
+
+  set(key: SignedResourceId, value: DownloadAPI_GetDownloadURL_Response): void {
+    const ttl = downloadUrlCacheTtlMs(value.downloadUrl);
+    if (ttl <= 0) return; // Cache miss.
+    this.cache.set(key, value, { ttl });
+  }
+
+  delete(key: SignedResourceId): void {
+    this.cache.delete(key);
+  }
+}

package/src/drivers/download_blob/download_blob.test.ts

@@ -227,10 +227,54 @@ test("should get undefined when releasing a blob from a small cache and the blob
   });
 });
 
+test("getContentDirect returns the same content but bypasses the ranges cache", async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), "test-download-direct-"));
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), "test-download-ranges-"));
+
+    const driver = await genDriver(client, dir, rangesCacheDir, genSigner());
+    const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
+
+    // On-demand (remote) handle - this is the path that uses the ranges cache.
+    const c = driver.getOnDemandBlob(downloadable);
+    const blob = await c.getValue();
+    expect(blob).toBeDefined();
+    expect(blob.size).toEqual(3);
+
+    const baseline = await dirSizeBytes(rangesCacheDir);
+
+    // Direct reads return the correct bytes (full + range) ...
+    expect((await driver.getContentDirect(blob.handle))?.toString()).toBe("42\n");
+    expect(
+      (await driver.getContentDirect(blob.handle, { range: { from: 0, to: 2 } }))?.toString(),
+    ).toBe("42");
+
+    // ... and leave the ranges cache untouched (cache writes are fire-and-forget, so settle first).
+    await scheduler.wait(100);
+    expect(await dirSizeBytes(rangesCacheDir)).toBe(baseline);
+
+    // A normal cached read of the same handle DOES populate the ranges cache.
+    expect((await driver.getContent(blob.handle))?.toString()).toBe("42\n");
+    await scheduler.wait(100);
+    expect(await dirSizeBytes(rangesCacheDir)).toBeGreaterThan(baseline);
+  });
+});
+
 function genSigner() {
   return new HmacSha256Signer(HmacSha256Signer.generateSecret());
 }
 
+/** Total bytes of all files under `dir` (recursive), for asserting cache population. */
+async function dirSizeBytes(dir: string): Promise<number> {
+  let total = 0;
+  for (const entry of await fsp.readdir(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) total += await dirSizeBytes(full);
+    else if (entry.isFile()) total += (await fsp.stat(full)).size;
+  }
+  return total;
+}
+
 async function genDriver(
   client: PlClient,
   dir: string,

package/src/drivers/download_blob/download_blob.ts

@@ -9,6 +9,7 @@ import {
 import type {
   AnyLogHandle,
   BlobDriver,
+  BlobDriverMetrics,
   ContentHandler,
   GetContentOptions,
   LocalBlobHandle,
@@ -101,6 +102,10 @@ export class DownloadDriver implements BlobDriver, AsyncDisposable {
 
   private readonly saveDir: string;
 
+  /** Downloads that bypassed the ranges cache; counted when issued. */
+  private uncachedRequests = 0;
+  private uncachedRequestBytes = 0;
+
   constructor(
     private readonly logger: MiLogger,
     private readonly clientDownload: ClientDownload,
@@ -322,9 +327,50 @@ export class DownloadDriver implements BlobDriver, AsyncDisposable {
       }
     }
 
+    return await this.getContentImpl({ handle, options });
+  }
+
+  /**
+   * Same as {@link getContent}, but bypasses the ranges cache entirely (no read, no write).
+   * For local handles this is identical to {@link getContent}, since local content never
+   * uses the ranges cache.
+   */
+  public async getContentDirect(
+    handle: LocalBlobHandle | RemoteBlobHandle,
+    options?: GetContentOptions,
+  ): Promise<Uint8Array> {
+    return await this.getContentImpl({
+      handle,
+      options: options ?? {},
+      bypassRangesCache: true,
+    });
+  }
+
+  /**
+   * Operational metrics for a monitoring panel. Serv cache metrics are reported separately
+   * (different owner) — the panel composes both.
+   */
+  public getMetrics(): BlobDriverMetrics {
+    return {
+      uncachedRequests: this.uncachedRequests,
+      uncachedRequestBytes: this.uncachedRequestBytes,
+      ...this.clientDownload.metrics(),
+    };
+  }
+
+  private async getContentImpl({
+    handle,
+    options,
+    bypassRangesCache = false,
+  }: {
+    handle: LocalBlobHandle | RemoteBlobHandle;
+    options: GetContentOptions;
+    bypassRangesCache?: boolean;
+  }): Promise<Uint8Array> {
     const request = () =>
       this.withContent(handle, {
         ...options,
+        bypassRangesCache,
         handler: async (content) => {
           const chunks: Uint8Array[] = [];
           for await (const chunk of content) {
@@ -350,9 +396,10 @@ export class DownloadDriver implements BlobDriver, AsyncDisposable {
     handle: LocalBlobHandle | RemoteBlobHandle,
     options: GetContentOptions & {
       handler: ContentHandler<T>;
+      bypassRangesCache?: boolean;
     },
   ): Promise<T> {
-    const { range, signal, handler } = options;
+    const { range, signal, handler, bypassRangesCache } = options;
 
     if (isLocalBlobHandle(handle)) {
       return await withFileContent({ path: this.getLocalPath(handle), range, signal, handler });
@@ -362,16 +409,32 @@ export class DownloadDriver implements BlobDriver, AsyncDisposable {
       const result = parseRemoteHandle(handle, this.signer);
 
       const key = blobKey(result.info.id);
-      const filePath = await this.rangesCache.get(key, range ?? { from: 0, to: result.size });
+      const filePath = bypassRangesCache
+        ? undefined
+        : await this.rangesCache.get(key, range ?? { from: 0, to: result.size });
       signal?.throwIfAborted();
 
       if (filePath) return await withFileContent({ path: filePath, range, signal, handler });
 
+      if (bypassRangesCache) this.uncachedRequests++;
+
       return await this.clientDownload.withBlobContent(
         result.info,
         { signal },
         options,
         async (content, size) => {
+          if (bypassRangesCache) {
+            const counted = content.pipeThrough(
+              new TransformStream<Uint8Array, Uint8Array>({
+                transform: (chunk, controller) => {
+                  this.uncachedRequestBytes += chunk.byteLength;
+                  controller.enqueue(chunk);
+                },
+              }),
+            );
+            return await handler(counted, size);
+          }
+
           const [handlerStream, cacheStream] = content.tee();
 
           const handlerPromise = handler(handlerStream, size);