@milaboratories/pl-deployments 2.12.5 → 2.12.7

package/src/ssh/pl.ts

@@ -7,6 +7,9 @@ import { downloadBinaryNoExtract } from '../common/pl_binary_download';
 import upath from 'upath';
 import * as plpath from './pl_paths';
 import { getDefaultPlVersion } from '../common/pl_version';
+import type { ProxySettings } from '@milaboratories/pl-http';
+import { defaultHttpDispatcher } from '@milaboratories/pl-http';
+import type { Dispatcher } from 'undici';
 
 import net from 'node:net';
 import type { PlConfig, PlLicenseMode, SshPlConfigGenerationResult } from '@milaboratories/pl-config';
@@ -148,6 +151,7 @@ export class SshPl {
         return state.existedSettings!;
       }
       await this.doStepStopExistedPlatforma(state, onProgress);
+      await this.doStepCheckDbLock(state, onProgress);
 
       await onProgress?.('Installation platforma...');
 
@@ -213,6 +217,65 @@ export class SshPl {
     await onProgress?.('Connection information saved.');
   }
 
+  private async doStepCheckDbLock(
+    state: PlatformaInitState,
+    onProgress?: (...args: any[]) => Promise<any>,
+  ) {
+    const removeLockFile = async (lockFilePath: string) => {
+      try {
+        await this.sshClient.exec(`rm -f ${lockFilePath}`);
+        this.logger.info(`Removed stale lock file ${lockFilePath}`);
+      } catch (e: unknown) {
+        const msg = `Failed to remove stale lock file ${lockFilePath}: ${e}`;
+        this.logger.error(msg);
+        throw new Error(msg);
+      }
+    };
+
+    state.step = 'checkDbLock';
+    await onProgress?.('Checking for DB lock...');
+
+    const lockFilePath = plpath.platformaDbLock(state.remoteHome!);
+    const lockFileExists = await this.sshClient.checkFileExists(lockFilePath);
+
+    if (!lockFileExists) {
+      await onProgress?.('No DB lock found. Proceeding...');
+      return;
+    }
+
+    this.logger.info(`DB lock file found at ${lockFilePath}. Checking which process holds it...`);
+    const lockProcessInfo = await this.findLockHolder(lockFilePath);
+
+    if (!lockProcessInfo) {
+      this.logger.warn('Lock file exists but no process is holding it. Removing stale lock file...');
+      await removeLockFile(lockFilePath);
+      return;
+    }
+
+    this.logger.info(
+      `Found process ${lockProcessInfo.pid} (user: ${lockProcessInfo.user}) holding DB lock`,
+    );
+
+    if (lockProcessInfo.user !== this.username) {
+      const msg
+        = `DB lock is held by process ${lockProcessInfo.pid} `
+        + `owned by user '${lockProcessInfo.user}', but current user is '${this.username}'. `
+        + 'Cannot kill process owned by different user.';
+      this.logger.error(msg);
+      throw new Error(msg);
+    }
+
+    this.logger.info(`Process ${lockProcessInfo.pid} belongs to current user ${this.username}. Killing it...`);
+    await this.killRemoteProcess(lockProcessInfo.pid);
+    this.logger.info('Process holding DB lock has been terminated.');
+
+    // Verify lock file is gone or can be removed
+    const lockStillExists = await this.sshClient.checkFileExists(lockFilePath);
+    if (lockStillExists) {
+      await removeLockFile(lockFilePath);
+    }
+  }
+
   private async doStepConfigureSupervisord(state: PlatformaInitState, onProgress: ((...args: any) => Promise<any>) | undefined) {
     await onProgress?.('Writing supervisord configuration...');
     state.step = 'configureSupervisord';
@@ -317,6 +380,7 @@ export class SshPl {
 
     const downloadRes = await this.downloadBinariesAndUploadToTheServer(
       ops.localWorkdir, ops.plBinary!, state.remoteHome!, state.arch!, state.shouldUseMinio ?? false,
+      ops.proxy,
     );
     await onProgress?.('All required binaries have been downloaded and uploaded.');
 
@@ -387,18 +451,22 @@ export class SshPl {
     remoteHome: string,
     arch: Arch,
     shouldUseMinio: boolean,
+    proxy?: ProxySettings,
   ) {
     const state: DownloadAndUntarState[] = [];
+    const dispatcher = defaultHttpDispatcher(proxy);
     try {
       const pl = await this.downloadAndUntar(
         localWorkdir, remoteHome, arch,
         'pl', `pl-${plBinary.version}`,
+        dispatcher,
       );
       state.push(pl);
 
       const supervisor = await this.downloadAndUntar(
         localWorkdir, remoteHome, arch,
         'supervisord', plpath.supervisordDirName,
+        dispatcher,
       );
       state.push(supervisor);
 
@@ -407,6 +475,7 @@ export class SshPl {
         const minio = await this.downloadAndUntar(
           localWorkdir, remoteHome, arch,
           'minio', plpath.minioDirName,
+          dispatcher,
         );
         state.push(minio);
         await this.sshClient.chmod(minioPath, 0o750);
@@ -421,6 +490,105 @@ export class SshPl {
       const msg = `SshPl.downloadBinariesAndUploadToServer: ${e}, state: ${JSON.stringify(state)}`;
       this.logger.error(msg);
       throw e;
+    } finally {
+      await dispatcher.close();
+    }
+  }
+
+  private async findLockHolderWithLsof(lockFilePath: string): Promise<LockProcessInfo | null> {
+    try {
+      const { stdout } = await this.sshClient.exec(`lsof ${lockFilePath} 2>/dev/null || true`);
+      const output = stdout.trim();
+      if (!output) {
+        return null;
+      }
+
+      // Example:
+      // COMMAND     PID    USER   FD   TYPE DEVICE SIZE/OFF     NODE NAME
+      // platforma 11628 rfiskov   10u   REG   1,16        0 66670038 ./LOCK
+      const lines = output.split('\n');
+      if (lines.length <= 1) {
+        return null;
+      }
+
+      const parts = lines[1].trim().split(/\s+/);
+      if (parts.length < 3) {
+        return null;
+      }
+
+      const pid = Number.parseInt(parts[1], 10);
+      const user = parts[2];
+
+      return Number.isNaN(pid) || !user ? null : { pid, user };
+    } catch (e: unknown) {
+      this.logger.warn(`Failed to use lsof to check lock: ${e}`);
+      return null;
+    }
+  }
+
+  private async findLockHolderWithFuser(lockFilePath: string): Promise<LockProcessInfo | null> {
+    try {
+      const { stdout } = await this.sshClient.exec(`fuser ${lockFilePath} 2>/dev/null || true`);
+      const output = stdout.trim();
+      if (!output) {
+        return null;
+      }
+
+      // Example: ./LOCK: 11628
+      const match = output.match(/: (\d+)/);
+      if (!match) {
+        return null;
+      }
+
+      const pid = Number.parseInt(match[1], 10);
+      if (Number.isNaN(pid)) {
+        return null;
+      }
+
+      try {
+        const psResult = await this.sshClient.exec(`ps -o user= -p ${pid} 2>/dev/null || true`);
+        const user = psResult.stdout.trim();
+        return user ? { pid, user } : null;
+      } catch (e: unknown) {
+        this.logger.warn(`Failed to get user for PID ${pid}: ${e}`);
+        return null;
+      }
+    } catch (e: unknown) {
+      this.logger.warn(`Failed to use fuser to check lock: ${e}`);
+      return null;
+    }
+  }
+
+  private async findLockHolder(lockFilePath: string): Promise<LockProcessInfo | null> {
+    const viaLsof = await this.findLockHolderWithLsof(lockFilePath);
+    if (viaLsof) {
+      return viaLsof;
+    }
+    return this.findLockHolderWithFuser(lockFilePath);
+  }
+
+  private async killRemoteProcess(pid: number): Promise<void> {
+    this.logger.info(`Killing process ${pid}...`);
+
+    try {
+      // Try graceful termination first
+      await this.sshClient.exec(`kill ${pid} 2>/dev/null || true`);
+      await sleep(1000);
+
+      // Check if process still exists
+      try {
+        await this.sshClient.exec(`kill -0 ${pid} 2>/dev/null`);
+        // Process still exists, force kill
+        this.logger.warn(`Process ${pid} still alive after SIGTERM, forcing kill...`);
+        await this.sshClient.exec(`kill -9 ${pid} 2>/dev/null || true`);
+        await sleep(500);
+      } catch {
+        // Process is dead, nothing to do
+      }
+    } catch (e: unknown) {
+      const msg = `Failed to kill process ${pid}: ${e}`;
+      this.logger.error(msg);
+      throw new Error(msg);
     }
   }
 
@@ -436,6 +604,7 @@ export class SshPl {
     arch: Arch,
     softwareName: string,
     tgzName: string,
+    dispatcher?: Dispatcher,
   ): Promise<DownloadAndUntarState> {
     const state: DownloadAndUntarState = {};
     state.binBasePath = plpath.binariesDir(remoteHome);
@@ -453,6 +622,7 @@ export class SshPl {
           tgzName,
           arch: arch.arch,
           platform: arch.platform,
+          dispatcher,
         });
         break;
       } catch (e: unknown) {
@@ -611,11 +781,14 @@ export type SshPlConfig = {
   license: PlLicenseMode;
   useGlobalAccess?: boolean;
   plBinary?: PlBinarySourceDownload;
+  proxy?: ProxySettings;
 
   onProgress?: (...args: any) => Promise<any>;
   plConfigPostprocessing?: (config: PlConfig) => PlConfig;
 };
 
+export type LockProcessInfo = { pid: number; user: string };
+
 const defaultSshPlConfig: Pick<
   SshPlConfig,
   | 'useGlobalAccess'
@@ -653,6 +826,7 @@ type PlatformaInitStep =
   | 'detectHome'
   | 'checkAlive'
   | 'stopExistedPlatforma'
+  | 'checkDbLock'
   | 'downloadBinaries'
   | 'fetchPorts'
   | 'generateNewConfig'

package/src/ssh/pl_paths.ts

@@ -63,3 +63,7 @@ export function connectionInfo(remoteHome: string) {
 export function platformaCliLogs(remoteHome: string) {
   return upath.join(workDir(remoteHome), 'platforma_cli_logs.log');
 }
+
+export function platformaDbLock(remoteHome: string) {
+  return upath.join(workDir(remoteHome), 'db', 'LOCK');
+}