@milaboratories/pl-deployments 1.1.5 → 1.1.7

package/src/common/pl_binary.ts

@@ -1,8 +1,10 @@
 import type { MiLogger } from '@milaboratories/ts-helpers';
 import { assertNever } from '@milaboratories/ts-helpers';
-import { downloadPlBinary, DownloadBinaryResult } from './pl_binary_download';
+import { downloadBinary } from './pl_binary_download';
 import { getDefaultPlVersion } from './pl_version';
 import os from 'os';
+import upath from 'upath';
+import { newOs, OSType } from './os_and_arch';
 
 /** Shows how the binary should be got. */
 export type PlBinarySource = PlBinarySourceDownload | PlBinarySourceLocal;
@@ -28,7 +30,8 @@ export async function resolveLocalPlBinaryPath(
 ): Promise<string> {
   switch (src.type) {
     case 'Download':
-      return (await downloadPlBinary(logger, downloadDir, src.version, os.arch(), os.platform())).binaryPath!;
+      const ops = await downloadBinary(logger, downloadDir, 'pl', `pl-${src.version}`, os.arch(), os.platform());
+      return upath.join(ops.baseName, 'binaries', osToBinaryName[newOs(os.platform())]);
 
     case 'Local':
       return src.path;
@@ -37,3 +40,9 @@ export async function resolveLocalPlBinaryPath(
       assertNever(src);
   }
 }
+
+export const osToBinaryName: Record<OSType, string> = {
+  linux: 'platforma',
+  macos: 'platforma',
+  windows: 'platforma.exe',
+};

package/src/common/pl_binary_download.ts

@@ -11,13 +11,19 @@ import decompress from 'decompress';
 import type { ArchType, OSType } from './os_and_arch';
 import { newOs, newArch } from './os_and_arch';
 
+const cdn = 'https://cdn.platforma.bio/software';
+// We'll download things from Global Access if downloading from CDN has failed
+// (it might be that it's blocked from the company's network.)
+const gaCdn = 'https://cdn-ga.pl-open.science/software';
+
 export type DownloadBinaryResult = {
   archiveUrl: string;
+  alternativeArchiveGAUrl: string;
+  wasDownloadedFrom?: string;
   archivePath: string;
   archiveType: ArchiveType;
   targetFolder: string;
   baseName: string;
-  binaryPath?: string;
 };
 
 export async function downloadBinaryNoExtract(
@@ -29,9 +35,15 @@ export async function downloadBinaryNoExtract(
   platform: string,
 ): Promise<DownloadBinaryResult> {
   const opts = getPathsForDownload(softwareName, tgzName, baseDir, newArch(arch), newOs(platform));
-  const { archiveUrl, archivePath } = opts;
+  const { archiveUrl, alternativeArchiveGAUrl, archivePath } = opts;
 
-  await downloadArchive(logger, archiveUrl, archivePath);
+  try {
+    await downloadArchive(logger, archiveUrl, archivePath);
+    opts.wasDownloadedFrom = archiveUrl;
+  } catch (e: unknown) {
+    await downloadArchive(logger, alternativeArchiveGAUrl, archivePath);
+    opts.wasDownloadedFrom = alternativeArchiveGAUrl;
+  }
 
   return opts;
 }
@@ -45,25 +57,16 @@ export async function downloadBinary(
   platform: string,
 ): Promise<DownloadBinaryResult> {
   const opts = getPathsForDownload(softwareName, archiveName, baseDir, newArch(arch), newOs(platform));
-  const { archiveUrl, archivePath, archiveType, targetFolder, baseName } = opts;
-
-  await downloadArchive(logger, archiveUrl, archivePath);
-  await extractArchive(logger, archivePath, archiveType, targetFolder);
+  const { archiveUrl, alternativeArchiveGAUrl, archivePath, archiveType, targetFolder, baseName } = opts;
 
-  return opts;
-}
-
-export async function downloadPlBinary(
-  logger: MiLogger,
-  baseDir: string,
-  plVersion: string,
-  arch: string,
-  platform: string,
-): Promise<DownloadBinaryResult> {
-  const opts = localDownloadPlOptions(plVersion, baseDir, newArch(arch), newOs(platform));
-  const { archiveUrl, archivePath, archiveType, targetFolder, binaryPath } = opts;
+  try {
+    await downloadArchive(logger, archiveUrl, archivePath);
+    opts.wasDownloadedFrom = archiveUrl;
+  } catch (e: unknown) {
+    await downloadArchive(logger, alternativeArchiveGAUrl, archivePath);
+    opts.wasDownloadedFrom = alternativeArchiveGAUrl;
+  }
 
-  await downloadArchive(logger, archiveUrl, archivePath);
   await extractArchive(logger, archivePath, archiveType, targetFolder);
 
   return opts;
@@ -80,13 +83,15 @@ function getPathsForDownload(
   const archiveType = osToArchiveType[os];
 
   const archiveFileName = `${baseName}.${archiveType}`;
-  const archiveUrl = `https://cdn.platforma.bio/software/${softwareName}/${os}/${archiveFileName}`;
+  const archiveUrl = `${cdn}/${softwareName}/${os}/${archiveFileName}`;
+  const alternativeArchiveGAUrl = `${gaCdn}/${softwareName}/${os}/${archiveFileName}`;
   const archivePath = upath.join(baseDir, archiveFileName);
   // folder where binary distribution of pl will be unpacked
   const targetFolder = upath.join(baseDir, baseName);
 
   return {
     archiveUrl,
+    alternativeArchiveGAUrl,
     archivePath,
     archiveType,
     targetFolder,
@@ -94,34 +99,6 @@ function getPathsForDownload(
   };
 }
 
-export function localDownloadPlOptions(
-  plVersion: string,
-  baseDir: string,
-  arch: ArchType,
-  os: OSType,
-): DownloadBinaryResult {
-  const baseName = `pl-${plVersion}-${arch}`;
-  const archiveType = osToArchiveType[os];
-
-  const archiveFileName = `${baseName}.${archiveType}`;
-  const archiveUrl = `https://cdn.platforma.bio/software/pl/${os}/${archiveFileName}`;
-  const archivePath = upath.join(baseDir, archiveFileName);
-
-  // folder where binary distribution of pl will be unpacked
-  const targetFolder = upath.join(baseDir, baseName);
-
-  const binaryPath = upath.join(baseName, 'binaries', osToBinaryName[os]);
-
-  return {
-    archiveUrl,
-    archivePath,
-    archiveType,
-    targetFolder,
-    binaryPath,
-    baseName,
-  };
-}
-
 export type DownloadInfo = {
   dstArchive?: string;
   fileExisted?: boolean;
@@ -250,9 +227,3 @@ const osToArchiveType: Record<OSType, ArchiveType> = {
   macos: 'tgz',
   windows: 'zip',
 };
-
-const osToBinaryName: Record<OSType, string> = {
-  linux: 'platforma',
-  macos: 'platforma',
-  windows: 'platforma.exe',
-};

package/src/local/pl.ts

@@ -162,7 +162,9 @@ export async function localPlatformaInit(logger: MiLogger, _ops: LocalPlOptions)
     logger.info(`writing configuration '${configPath}'...`);
     await fsp.writeFile(configPath, ops.config);
 
-    const baseBinaryPath = await resolveLocalPlBinaryPath(logger, upath.join(workDir, 'binaries'), ops.plBinary);
+    const plBinPath = upath.join(workDir, 'binaries');
+    const baseBinaryPath = await resolveLocalPlBinaryPath(logger, plBinPath, ops.plBinary);
+
     const binaryPath = trace('binaryPath', upath.join('binaries', baseBinaryPath));
 
     const processOpts: ProcessOptions = {

package/src/ssh/__tests__/pl-docker.test.ts

@@ -5,7 +5,7 @@ import upath from 'upath';
 import { getDefaultPlVersion } from '../../common/pl_version';
 import { existsSync, unlinkSync, rmSync } from 'fs';
 import { newArch } from '../../common/os_and_arch';
-import { downloadBinary, downloadPlBinary } from '../../common/pl_binary_download';
+import { downloadBinary } from '../../common/pl_binary_download';
 import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
 import * as plpath from '../pl_paths';
 
@@ -84,10 +84,10 @@ describe('SshPl', async () => {
   it('Transfer Platforma to server', async () => {
     const arch = await sshPl.getArch();
 
-    const plPath = await downloadPlBinary(
+    const plPath = await downloadBinary(
       new ConsoleLoggerAdapter(),
       downloadDestination,
-      getDefaultPlVersion(),
+      'pl', `pl-${getDefaultPlVersion()}`,
       arch.arch,
       arch.platform,
     );
@@ -154,10 +154,10 @@ describe('SshPl download binaries', async () => {
   it('Download pl. We have archive and extracted data', async () => {
     const arch = await sshPl.getArch();
 
-    const result = await downloadPlBinary(
+    const result = await downloadBinary(
       new ConsoleLoggerAdapter(),
       downloadDestination,
-      getDefaultPlVersion(),
+      'pl', `pl-${getDefaultPlVersion()}`,
       arch.arch,
       arch.platform,
     );

package/src/ssh/pl.ts

@@ -1,7 +1,7 @@
 import type * as ssh from 'ssh2';
 import { SshClient } from './ssh';
 import type { MiLogger } from '@milaboratories/ts-helpers';
-import { sleep, notEmpty, fileExists } from '@milaboratories/ts-helpers';
+import { sleep, notEmpty } from '@milaboratories/ts-helpers';
 import type { DownloadBinaryResult } from '../common/pl_binary_download';
 import { downloadBinaryNoExtract } from '../common/pl_binary_download';
 import upath from 'upath';
@@ -19,7 +19,7 @@ export class SshPl {
     public readonly logger: MiLogger,
     public readonly sshClient: SshClient,
     private readonly username: string,
-  ) {}
+  ) { }
 
   public info() {
     return {
@@ -38,6 +38,10 @@ export class SshPl {
     }
   }
 
+  public cleanUp() {
+    this.sshClient.close();
+  }
+
   public async isAlive(): Promise<boolean> {
     const arch = await this.getArch();
     const remoteHome = await this.getUserHomeDirectory();
@@ -89,6 +93,8 @@ export class SshPl {
     this.logger.info(`pl.reset: Deleting Platforma workDir ${workDir} on the server`);
     await this.sshClient.deleteFolder(plpath.workDir(workDir));
 
+    this.cleanUp();
+
     return true;
   }
 

package/src/ssh/ssh.ts

@@ -23,6 +23,7 @@ export type SshDirContent = {
 export class SshClient {
   private config?: ConnectConfig;
   public homeDir?: string;
+  private forwardedServers: net.Server[] = [];
 
   constructor(
     private readonly logger: MiLogger,
@@ -46,6 +47,10 @@ export class SshClient {
     return client;
   }
 
+  public getForwardedServers() {
+    return this.forwardedServers;
+  }
+
   public getFullHostName() {
     return `${this.config?.host}:${this.config?.port}`;
   }
@@ -224,6 +229,7 @@ export class SshClient {
 
       server.listen(ports.localPort, '127.0.0.1', () => {
         this.logger.info(`${log}.server: started listening`);
+        this.forwardedServers.push(server);
         resolve({ server });
       });
 
@@ -234,10 +240,25 @@ export class SshClient {
 
       server.on('close', () => {
         this.logger.info(`${log}.server: closed ${JSON.stringify(ports)}`);
+        this.forwardedServers = this.forwardedServers.filter((s) => s !== server);
       });
     });
   }
 
+  public closeForwardedPorts(): void {
+    this.logger.info('[SSH] Closing all forwarded ports...');
+    this.forwardedServers.forEach((server) => {
+      const rawAddress = server.address();
+      if (rawAddress && typeof rawAddress !== 'string') {
+        const address: net.AddressInfo = rawAddress;
+        this.logger.info(`[SSH] Closing port forward for server ${address.address}:${address.port}`);
+      }
+
+      server.close();
+    });
+    this.forwardedServers = [];
+  }
+
   /**
   * Checks if a specified host is available by performing a DNS lookup.
   * @param hostname - The hostname or IP address to check.
@@ -616,9 +637,10 @@ export class SshClient {
   }
 
   /**
-   * Closes the SSH client connection.
+   * Closes the SSH client connection and forwarded ports.
    */
   public close(): void {
+    this.closeForwardedPorts();
     this.client.end();
   }
 }