npm - @milaboratories/pl-drivers - Versions diffs - 1.16.0 → 1.16.2 - Mend

@milaboratories/pl-drivers 1.16.0 → 1.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (243) hide show

package/src/drivers/download_url/get_local_tgz.test.ts ADDED Viewed

@@ -0,0 +1,106 @@
+import { ConsoleLoggerAdapter, HmacSha256Signer } from "@milaboratories/ts-helpers";
+import * as os from "node:os";
+import { createHash } from "node:crypto";
+import { text } from "node:stream/consumers";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import * as fs from "node:fs";
+import * as fsp from "node:fs/promises";
+import * as path from "node:path";
+import * as zlib from "node:zlib";
+import * as tar from "tar-fs";
+import { Agent } from "undici";
+import { DownloadUrlDriver } from "./driver";
+import { test, expect } from "vitest";
+function genSigner() {
+  return new HmacSha256Signer(HmacSha256Signer.generateSecret());
+}
+/** Packs a tiny UI folder into a real gzipped `ui.tgz` at `tgzPath`. */
+async function makeUiTgz(tgzPath: string): Promise<void> {
+  const srcDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ui-src-"));
+  await fsp.writeFile(path.join(srcDir, "index.html"), "<html>hello local tgz</html>");
+  await fsp.writeFile(path.join(srcDir, "index.js"), "'use strict'; console.log('ui');");
+  await pipeline(tar.pack(srcDir), zlib.createGzip(), fs.createWriteStream(tgzPath));
+}
+test("getLocalTgz unpacks a local ui.tgz, serves a signed path, and caches by path:mtime", async () => {
+  const logger = new ConsoleLoggerAdapter();
+  const saveDir = await fsp.mkdtemp(path.join(os.tmpdir(), "local-tgz-save-"));
+  const tgzPath = path.join(await fsp.mkdtemp(path.join(os.tmpdir(), "local-tgz-src-")), "ui.tgz");
+  await makeUiTgz(tgzPath);
+  // local-tgz never touches the network; a bare dispatcher is enough.
+  const driver = new DownloadUrlDriver(logger, new Agent(), saveDir, genSigner());
+  const mtime = "1782150933623";
+  const c = driver.getLocalTgz(tgzPath, mtime);
+  const first = await c.getValue();
+  expect(first).toBeUndefined();
+  await c.awaitChange();
+  const result = await c.getValue();
+  expect(result?.error).toBeUndefined();
+  expect(result?.url).not.toBeUndefined();
+  // The served URL resolves to the unpacked index.html.
+  const u = new URL(result!.url!);
+  u.pathname = "index.html";
+  const indexPath = driver.getPathForBlockUI(u.toString());
+  const html = await text(Readable.toWeb(fs.createReadStream(indexPath)));
+  expect(html).toContain("hello local tgz");
+  // Cache key is sha256(path + ":" + mtime) — the extraction dir lives there.
+  const expectedDir = path.join(
+    saveDir,
+    createHash("sha256").update(`${tgzPath}:${mtime}`).digest("hex"),
+  );
+  expect(fs.existsSync(expectedDir)).toBe(true);
+  // A second call (same path:mtime) is a cache hit — same served folder, no
+  // re-unpack, refcount still held by the first computable.
+  const c2 = driver.getLocalTgz(tgzPath, mtime);
+  let hit = await c2.getValue();
+  if (hit === undefined) {
+    await c2.awaitChange();
+    hit = await c2.getValue();
+  }
+  expect(hit?.url).toBe(result!.url);
+  c.resetState();
+  c2.resetState();
+  await driver.releaseAll();
+});
+test("getLocalTgz re-derives the extraction dir when mtime changes", async () => {
+  const logger = new ConsoleLoggerAdapter();
+  const saveDir = await fsp.mkdtemp(path.join(os.tmpdir(), "local-tgz-save2-"));
+  const tgzPath = path.join(await fsp.mkdtemp(path.join(os.tmpdir(), "local-tgz-src2-")), "ui.tgz");
+  await makeUiTgz(tgzPath);
+  const driver = new DownloadUrlDriver(logger, new Agent(), saveDir, genSigner());
+  const dirFor = (mtime: string) =>
+    path.join(saveDir, createHash("sha256").update(`${tgzPath}:${mtime}`).digest("hex"));
+  const cA = driver.getLocalTgz(tgzPath, "1");
+  expect(await cA.getValue()).toBeUndefined();
+  await cA.awaitChange();
+  expect((await cA.getValue())?.url).not.toBeUndefined();
+  const cB = driver.getLocalTgz(tgzPath, "2");
+  expect(await cB.getValue()).toBeUndefined();
+  await cB.awaitChange();
+  expect((await cB.getValue())?.url).not.toBeUndefined();
+  expect(dirFor("1")).not.toBe(dirFor("2"));
+  expect(fs.existsSync(dirFor("1"))).toBe(true);
+  expect(fs.existsSync(dirFor("2"))).toBe(true);
+  cA.resetState();
+  cB.resetState();
+  await driver.releaseAll();
+});

package/src/drivers/download_url/task.ts CHANGED Viewed

@@ -8,9 +8,11 @@ import {
   fileExists,
   notEmpty,
 } from "@milaboratories/ts-helpers";
+import { createReadStream } from "node:fs";
 import * as fsp from "node:fs/promises";
 import * as path from "node:path";
-import { Transform, Writable } from "node:stream";
+import { Readable, Transform, Writable } from "node:stream";
+import type { ReadableStream } from "node:stream/web";
 import * as zlib from "node:zlib";
 import * as tar from "tar-fs";
 import type { RemoteFileDownloader } from "../../helpers/download";
@@ -18,7 +20,14 @@ import { isDownloadNetworkError400 } from "../../helpers/download_errors";
 import type { UrlResult } from "./driver";
 import { newBlockUIURL } from "../urls/url";
-/** Downloads and extracts an archive to a directory. */
+/** Source an archive is fetched from before extraction: a remote URL or a
+ * local `.tgz` file on disk. Both flow through the same untar machinery. */
+export type DownloadSource =
+  | { type: "url"; url: URL }
+  | { type: "local-tgz"; path: string; mtime: string };
+/** Fetches (over HTTP or from a local file) and extracts an archive to a
+ * directory. `key` is the task's identity in the driver's task map and cache. */
 export class DownloadByUrlTask {
   readonly counter = new CallersCounter();
   readonly change = new ChangeSource();
@@ -30,14 +39,15 @@ export class DownloadByUrlTask {
   constructor(
     private readonly logger: MiLogger,
     readonly path: string,
-    readonly url: URL,
+    readonly source: DownloadSource,
+    readonly key: string,
     readonly signer: Signer,
     readonly saveDir: string,
   ) {}
   public info() {
     return {
-      url: this.url.toString(),
+      key: this.key,
       path: this.path,
       done: this.done,
       size: this.size,
@@ -52,9 +62,9 @@ export class DownloadByUrlTask {
   async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {
     try {
-      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
+      const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
       this.setDone(size);
-      this.change.markChanged(`download of ${this.url} finished`);
+      this.change.markChanged(`download of ${this.key} finished`);
     } catch (e: unknown) {
       if (
         e instanceof URLAborted ||
@@ -62,7 +72,7 @@ export class DownloadByUrlTask {
         this.signalCtl.signal.aborted
       ) {
         this.setError(e);
-        this.change.markChanged(`download of ${this.url} failed`);
+        this.change.markChanged(`download of ${this.key} failed`);
         // Just in case we were half-way extracting an archive.
         await rmRFDir(this.path);
         return;
@@ -72,7 +82,7 @@ export class DownloadByUrlTask {
     }
   }
-  private async downloadAndUntar(
+  private async fetchAndUntar(
     clientDownload: RemoteFileDownloader,
     withGunzip: boolean,
     signal: AbortSignal,
@@ -83,28 +93,41 @@ export class DownloadByUrlTask {
       return await dirSize(this.path);
     }
-    const size = await clientDownload.withContent(
-      this.url.toString(),
-      {},
-      { signal },
-      async (content, size) => {
-        let processedContent = content;
-        if (withGunzip) {
-          const gunzip = Transform.toWeb(zlib.createGunzip());
-          processedContent = content.pipeThrough(gunzip, { signal });
-        }
-        await createPathAtomically(this.logger, this.path, async (fPath: string) => {
-          await fsp.mkdir(fPath); // throws if a directory already exists.
-          const untar = Writable.toWeb(tar.extract(fPath));
-          await processedContent.pipeTo(untar, { signal });
-        });
-        return size;
-      },
-    );
-    return size;
+    if (this.source.type === "url") {
+      return await clientDownload.withContent(
+        this.source.url.toString(),
+        {},
+        { signal },
+        async (content, size) => {
+          await this.untarStream(content, withGunzip, signal);
+          return size;
+        },
+      );
+    }
+    // local-tgz: read the archive from disk and untar through the same path.
+    const content = Readable.toWeb(createReadStream(this.source.path)) as ReadableStream;
+    await this.untarStream(content, withGunzip, signal);
+    return await dirSize(this.path);
+  }
+  /** Optionally gunzips, then untars the content stream into `this.path`. */
+  private async untarStream(
+    content: ReadableStream,
+    withGunzip: boolean,
+    signal: AbortSignal,
+  ): Promise<void> {
+    let processedContent = content;
+    if (withGunzip) {
+      const gunzip = Transform.toWeb(zlib.createGunzip());
+      processedContent = content.pipeThrough(gunzip, { signal });
+    }
+    await createPathAtomically(this.logger, this.path, async (fPath: string) => {
+      await fsp.mkdir(fPath); // throws if a directory already exists.
+      const untar = Writable.toWeb(tar.extract(fPath));
+      await processedContent.pipeTo(untar, { signal });
+    });
   }
   getUrl(): UrlResult | undefined {