@milaboratories/pl-drivers 1.16.1 → 1.16.3

package/src/drivers/download_url/task.ts

@@ -8,9 +8,11 @@ import {
   fileExists,
   notEmpty,
 } from "@milaboratories/ts-helpers";
+import { createReadStream } from "node:fs";
 import * as fsp from "node:fs/promises";
 import * as path from "node:path";
-import { Transform, Writable } from "node:stream";
+import { Readable, Transform, Writable } from "node:stream";
+import type { ReadableStream } from "node:stream/web";
 import * as zlib from "node:zlib";
 import * as tar from "tar-fs";
 import type { RemoteFileDownloader } from "../../helpers/download";
@@ -18,7 +20,14 @@ import { isDownloadNetworkError400 } from "../../helpers/download_errors";
 import type { UrlResult } from "./driver";
 import { newBlockUIURL } from "../urls/url";
 
-/** Downloads and extracts an archive to a directory. */
+/** Source an archive is fetched from before extraction: a remote URL or a
+ * local `.tgz` file on disk. Both flow through the same untar machinery. */
+export type DownloadSource =
+  | { type: "url"; url: URL }
+  | { type: "local-tgz"; path: string; mtime: string };
+
+/** Fetches (over HTTP or from a local file) and extracts an archive to a
+ * directory. `key` is the task's identity in the driver's task map and cache. */
 export class DownloadByUrlTask {
   readonly counter = new CallersCounter();
   readonly change = new ChangeSource();
@@ -30,14 +39,15 @@ export class DownloadByUrlTask {
   constructor(
     private readonly logger: MiLogger,
     readonly path: string,
-    readonly url: URL,
+    readonly source: DownloadSource,
+    readonly key: string,
     readonly signer: Signer,
     readonly saveDir: string,
   ) {}
 
   public info() {
     return {
-      url: this.url.toString(),
+      key: this.key,
       path: this.path,
       done: this.done,
       size: this.size,
@@ -52,9 +62,9 @@ export class DownloadByUrlTask {
 
   async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {
     try {
-      const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
+      const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
       this.setDone(size);
-      this.change.markChanged(`download of ${this.url} finished`);
+      this.change.markChanged(`download of ${this.key} finished`);
     } catch (e: unknown) {
       if (
         e instanceof URLAborted ||
@@ -62,7 +72,7 @@ export class DownloadByUrlTask {
         this.signalCtl.signal.aborted
       ) {
         this.setError(e);
-        this.change.markChanged(`download of ${this.url} failed`);
+        this.change.markChanged(`download of ${this.key} failed`);
         // Just in case we were half-way extracting an archive.
         await rmRFDir(this.path);
         return;
@@ -72,7 +82,7 @@ export class DownloadByUrlTask {
     }
   }
 
-  private async downloadAndUntar(
+  private async fetchAndUntar(
     clientDownload: RemoteFileDownloader,
     withGunzip: boolean,
     signal: AbortSignal,
@@ -83,28 +93,41 @@ export class DownloadByUrlTask {
       return await dirSize(this.path);
     }
 
-    const size = await clientDownload.withContent(
-      this.url.toString(),
-      {},
-      { signal },
-      async (content, size) => {
-        let processedContent = content;
-        if (withGunzip) {
-          const gunzip = Transform.toWeb(zlib.createGunzip());
-          processedContent = content.pipeThrough(gunzip, { signal });
-        }
-
-        await createPathAtomically(this.logger, this.path, async (fPath: string) => {
-          await fsp.mkdir(fPath); // throws if a directory already exists.
-          const untar = Writable.toWeb(tar.extract(fPath));
-          await processedContent.pipeTo(untar, { signal });
-        });
-
-        return size;
-      },
-    );
-
-    return size;
+    if (this.source.type === "url") {
+      return await clientDownload.withContent(
+        this.source.url.toString(),
+        {},
+        { signal },
+        async (content, size) => {
+          await this.untarStream(content, withGunzip, signal);
+          return size;
+        },
+      );
+    }
+
+    // local-tgz: read the archive from disk and untar through the same path.
+    const content = Readable.toWeb(createReadStream(this.source.path)) as ReadableStream;
+    await this.untarStream(content, withGunzip, signal);
+    return await dirSize(this.path);
+  }
+
+  /** Optionally gunzips, then untars the content stream into `this.path`. */
+  private async untarStream(
+    content: ReadableStream,
+    withGunzip: boolean,
+    signal: AbortSignal,
+  ): Promise<void> {
+    let processedContent = content;
+    if (withGunzip) {
+      const gunzip = Transform.toWeb(zlib.createGunzip());
+      processedContent = content.pipeThrough(gunzip, { signal });
+    }
+
+    await createPathAtomically(this.logger, this.path, async (fPath: string) => {
+      await fsp.mkdir(fPath); // throws if a directory already exists.
+      const untar = Writable.toWeb(tar.extract(fPath));
+      await processedContent.pipeTo(untar, { signal });
+    });
   }
 
   getUrl(): UrlResult | undefined {