@milaboratories/pl-deployments 1.1.0

package/src/local/pl.test.ts

@@ -0,0 +1,129 @@
+import { test } from 'vitest';
+import type { LocalPlOptions } from './pl';
+import { localPlatformaInit } from './pl';
+import { ConsoleLoggerAdapter, sleep } from '@milaboratories/ts-helpers';
+import * as fs from 'fs/promises';
+import upath from 'upath';
+import { processStop } from './process';
+import * as yaml from 'yaml';
+import * as os from 'os';
+
+test(
+  'should start and stop platforma of the current version with hardcoded config',
+  { timeout: 25000 },
+  async ({ expect }) => {
+    const logger = new ConsoleLoggerAdapter();
+    const config = await readTestConfig();
+
+    const dir = await prepareDirForTestConfig();
+
+    const pl = await localPlatformaInit(logger, {
+      workingDir: dir,
+      config,
+      closeOld: false,
+    });
+
+    await sleep(5000);
+
+    console.log(`Platforma: %o`, pl.debugInfo());
+
+    expect(await pl.isAlive()).toBeTruthy();
+    expect(pl.pid).not.toBeUndefined();
+
+    pl.stop();
+    await pl.waitStopped();
+  },
+);
+
+test(
+  'should close old platforma when starting a new one if the option is set',
+  { timeout: 25000 },
+  async ({ expect }) => {
+    const logger = new ConsoleLoggerAdapter();
+
+    const config = await readTestConfig();
+    const dir = await prepareDirForTestConfig();
+    const options: LocalPlOptions = {
+      workingDir: dir,
+      config,
+    };
+
+    const oldPl = await localPlatformaInit(logger, options);
+    await sleep(5000);
+    console.log(`OldPlatforma: %o`, oldPl.debugInfo());
+
+    expect(await oldPl.isAlive()).toBeTruthy();
+    const newPl = await localPlatformaInit(logger, options);
+    expect(await oldPl.isAlive()).toBeFalsy();
+    await sleep(5000);
+
+    console.log(`NewPlatforma: %o`, newPl.debugInfo());
+
+    expect(await newPl.isAlive()).toBeTruthy();
+    expect(newPl.pid).not.toBeUndefined();
+    newPl.stop();
+    await newPl.waitStopped();
+  },
+);
+
+test(
+  'should restart platforma if restart option was provided',
+  { timeout: 25000 },
+  async ({ expect }) => {
+    const logger = new ConsoleLoggerAdapter();
+    const config = await readTestConfig();
+    const dir = await prepareDirForTestConfig();
+
+    const pl = await localPlatformaInit(logger, {
+      workingDir: dir,
+      config,
+      closeOld: false,
+      onCloseAndErrorNoStop: async (pl) => await pl.start(),
+    });
+    await sleep(1000);
+
+    expect(await pl.isAlive()).toBeTruthy();
+    processStop(pl.pid!);
+    await sleep(3000);
+    console.log(`Platforma after first stop: %o`, pl.debugInfo());
+
+    expect(await pl.isAlive()).toBeTruthy();
+    processStop(pl.pid!);
+    await sleep(3000);
+    console.log(`Platforma after second stop: %o`, pl.debugInfo());
+
+    expect(await pl.isAlive()).toBeTruthy();
+    expect(pl.debugInfo().nRuns).toEqual(3);
+
+    pl.stop();
+    await pl.waitStopped();
+  },
+);
+
+async function readTestConfig() {
+  const testConfig = upath.join(__dirname, 'config.test.yaml');
+  const config = (await fs.readFile(testConfig)).toString();
+
+  const parsed = yaml.parse(config);
+  parsed.license.value = process.env.MI_LICENSE;
+  if ((parsed.license.value ?? '') == '') {
+    parsed.license.file = process.env.MI_LICENSE_FILE;
+    if ((parsed.license.file ?? '') == '') {
+      parsed.license.file = upath.join(os.homedir(), '.pl.license');
+    }
+  }
+
+  return yaml.stringify(parsed);
+}
+
+async function prepareDirForTestConfig() {
+  const dir = upath.join(__dirname, '..', '.test');
+  await fs.rm(dir, { recursive: true, force: true });
+  await fs.mkdir(dir);
+
+  await fs.mkdir(upath.join(dir, 'storages', 'work'), { recursive: true });
+  await fs.mkdir(upath.join(dir, 'storages', 'main'), { recursive: true });
+  await fs.mkdir(upath.join(dir, 'packages'), { recursive: true });
+
+  return dir;
+}

package/src/local/pl.ts

@@ -0,0 +1,220 @@
+import type {
+  ProcessOptions } from './process';
+import {
+  isProcessAlive,
+  processStop,
+  processWaitStopped,
+  processRun,
+} from './process';
+import type { PlBinarySource } from '../common/pl_binary';
+import { newDefaultPlBinarySource, resolveLocalPlBinaryPath } from '../common/pl_binary';
+import type { MiLogger } from '@milaboratories/ts-helpers';
+import { notEmpty } from '@milaboratories/ts-helpers';
+import type { ChildProcess, SpawnOptions } from 'child_process';
+import { filePid, readPid, writePid } from './pid';
+import type { Trace } from './trace';
+import { withTrace } from './trace';
+import upath from 'upath';
+import fsp from 'fs/promises';
+import type { Required } from 'utility-types';
+
+export const LocalConfigYaml = 'config-local.yaml';
+
+/**
+ * Represents a local running pl-core,
+ * and has methods to start, check if it's running, stop and wait for stopping it.
+ * Also, a hook on pl-core closed can be provided.
+ */
+export class LocalPl {
+  private instance?: ChildProcess;
+  public pid?: number;
+  private nRuns: number = 0;
+  private lastRunHistory: Trace = {};
+  private wasStopped = false;
+
+  constructor(
+    private readonly logger: MiLogger,
+    private readonly workingDir: string,
+    private readonly startOptions: ProcessOptions,
+    private readonly initialStartHistory: Trace,
+    private readonly onClose?: (pl: LocalPl) => Promise<void>,
+    private readonly onError?: (pl: LocalPl) => Promise<void>,
+    private readonly onCloseAndError?: (pl: LocalPl) => Promise<void>,
+    private readonly onCloseAndErrorNoStop?: (pl: LocalPl) => Promise<void>,
+  ) {}
+
+  async start() {
+    await withTrace(this.logger, async (trace, t) => {
+      this.wasStopped = false;
+      const instance = processRun(this.logger, this.startOptions);
+      instance.on('error', (e: any) => {
+        this.logger.error(
+          `error '${e}', while running platforma, started opts: ${JSON.stringify(this.debugInfo())}`,
+        );
+
+        // keep in mind there are no awaits here, it will be asynchronous
+        if (this.onError !== undefined) this.onError(this);
+        if (this.onCloseAndError !== undefined) this.onCloseAndError(this);
+        if (this.onCloseAndErrorNoStop !== undefined && !this.wasStopped)
+          this.onCloseAndErrorNoStop(this);
+      });
+      instance.on('close', () => {
+        this.logger.warn(`platforma was closed, started opts: ${JSON.stringify(this.debugInfo())}`);
+
+        // keep in mind there are no awaits here, it will be asynchronous
+        if (this.onClose !== undefined) this.onClose(this);
+        if (this.onCloseAndError !== undefined) this.onCloseAndError(this);
+        if (this.onCloseAndErrorNoStop !== undefined && !this.wasStopped)
+          this.onCloseAndErrorNoStop(this);
+      });
+
+      trace('started', true);
+
+      const pidFile = trace('pidFile', filePid(this.workingDir));
+      trace('pid', notEmpty(instance.pid));
+      trace('pidWritten', await writePid(pidFile, notEmpty(instance.pid)));
+
+      this.nRuns++;
+      this.instance = instance;
+      this.pid = instance.pid;
+      this.lastRunHistory = t;
+    });
+  }
+
+  stop() {
+    // TODO use this.instance to stop the process
+    this.wasStopped = true;
+    processStop(notEmpty(this.pid));
+  }
+
+  async waitStopped() {
+    await processWaitStopped(notEmpty(this.pid), 15000);
+  }
+
+  stopped() {
+    return this.wasStopped;
+  }
+
+  async isAlive(): Promise<boolean> {
+    return await isProcessAlive(notEmpty(this.pid));
+  }
+
+  debugInfo() {
+    return {
+      lastRunHistory: this.lastRunHistory,
+      nRuns: this.nRuns,
+      pid: this.pid,
+      workingDir: this.workingDir,
+      initialStartHistory: this.initialStartHistory,
+      wasStopped: this.wasStopped,
+    };
+  }
+}
+
+/** Options to start a local pl-core. */
+export type LocalPlOptions = {
+  /** From what directory start a process. */
+  readonly workingDir: string;
+  /** A string representation of yaml config. */
+  readonly config: string;
+  /** How to get a binary, download it or get an existing one (default: download latest version) */
+  readonly plBinary?: PlBinarySource;
+  /** Additional options for a process, environments, stdout, stderr etc. */
+  readonly spawnOptions?: SpawnOptions;
+  /**
+   * If the previous pl-core was started from the same directory,
+   * we can check if it's still running and then stop it before starting a new one.
+   * (default: true)
+   */
+  readonly closeOld?: boolean;
+
+  readonly onClose?: (pl: LocalPl) => Promise<void>;
+  readonly onError?: (pl: LocalPl) => Promise<void>;
+  readonly onCloseAndError?: (pl: LocalPl) => Promise<void>;
+  readonly onCloseAndErrorNoStop?: (pl: LocalPl) => Promise<void>;
+};
+
+type LocalPlOptionsFull = Required<LocalPlOptions, 'plBinary' | 'spawnOptions' | 'closeOld'>;
+
+/**
+ * Starts pl-core, if the option was provided downloads a binary, reads license environments etc.
+ */
+export async function localPlatformaInit(logger: MiLogger, _ops: LocalPlOptions): Promise<LocalPl> {
+  // filling-in default values
+  const ops = {
+    plBinary: newDefaultPlBinarySource(),
+    spawnOptions: {},
+    closeOld: true,
+    ..._ops,
+  } satisfies LocalPlOptionsFull;
+
+  return await withTrace(logger, async (trace, t) => {
+    trace('startOptions', { ...ops, config: 'too wordy' });
+
+    const workDir = upath.resolve(ops.workingDir);
+
+    if (ops.closeOld) {
+      trace('closeOld', await localPlatformaReadPidAndStop(logger, workDir));
+    }
+
+    const configPath = upath.join(workDir, LocalConfigYaml);
+
+    logger.info(`writing configuration '${configPath}'...`);
+    await fsp.writeFile(configPath, ops.config);
+
+    const baseBinaryPath = await resolveLocalPlBinaryPath(logger, upath.join(workDir, 'binaries'), ops.plBinary);
+    const binaryPath = trace('binaryPath', upath.join('binaries', baseBinaryPath));
+
+    const processOpts: ProcessOptions = {
+      cmd: binaryPath,
+      args: ['-config', configPath],
+      opts: {
+        env: { ...process.env },
+        cwd: workDir,
+        stdio: ['pipe', 'ignore', 'inherit'],
+        windowsHide: true, // hide a terminal on Windows
+        ...ops.spawnOptions,
+      },
+    };
+    trace('processOpts', {
+      cmd: processOpts.cmd,
+      args: processOpts.args,
+      cwd: processOpts.opts.cwd,
+    });
+
+    const pl = new LocalPl(
+      logger,
+      ops.workingDir,
+      processOpts,
+      t,
+      ops.onClose,
+      ops.onError,
+      ops.onCloseAndError,
+      ops.onCloseAndErrorNoStop,
+    );
+    await pl.start();
+
+    return pl;
+  });
+}
+
+/** Reads a pid of the old pl-core if it was started in the same working directory,
+ * and closes it. */
+async function localPlatformaReadPidAndStop(
+  logger: MiLogger,
+  workingDir: string,
+): Promise<Record<string, any>> {
+  return await withTrace(logger, async (trace, t) => {
+    const file = trace('pidFilePath', filePid(workingDir));
+
+    const oldPid = trace('pid', await readPid(file));
+    const alive = trace('wasAlive', await isProcessAlive(oldPid));
+
+    if (oldPid !== undefined && alive) {
+      trace('stopped', processStop(oldPid));
+      trace('waitStopped', await processWaitStopped(oldPid, 10_000));
+    }
+
+    return t;
+  });
+}

package/src/local/process.ts

@@ -0,0 +1,44 @@
+import type { SpawnOptions, ChildProcess } from 'child_process';
+import { spawn } from 'child_process';
+import type { MiLogger } from '@milaboratories/ts-helpers';
+import { sleep } from '@milaboratories/ts-helpers';
+
+export type ProcessOptions = {
+  cmd: string;
+  args: string[];
+  opts: SpawnOptions;
+};
+
+export function processRun(logger: MiLogger, opts: ProcessOptions): ChildProcess {
+  logger.info(`Running:
+cmd: ${JSON.stringify([opts.cmd, ...opts.args])}
+wd: ${opts.opts.cwd}`);
+
+  logger.info('  spawning child process');
+  return spawn(opts.cmd, opts.args, opts.opts);
+}
+
+export async function isProcessAlive(pid: number) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+export function processStop(pid: number) {
+  return process.kill(pid, 'SIGINT');
+}
+
+export async function processWaitStopped(pid: number, maxMs: number) {
+  const sleepMs = 100;
+  let total = 0;
+  while (await isProcessAlive(pid)) {
+    await sleep(sleepMs);
+    total += sleepMs;
+    if (total > maxMs) {
+      throw new Error(`The process did not stopped after ${maxMs} ms.`);
+    }
+  }
+}

package/src/local/trace.ts

@@ -0,0 +1,30 @@
+import type { MiLogger } from '@milaboratories/ts-helpers';
+
+/** Records all inputs and outputs of one's choice, so if the error happened
+ * one can check how it was by just printing this structure. */
+export type Trace = Record<string, any>;
+
+export function newTrace(): Trace {
+  return {};
+}
+
+export function trace(t: Trace, k: string, v: any) {
+  t[k] = v;
+  return v;
+}
+
+/** Creates a trace and runs a function with it. The function can record all its
+ * logs or traces using `trace` fn. */
+export async function withTrace<T>(
+  logger: MiLogger,
+  fn: (trace: (k: string, v: any) => typeof v, t: Trace) => Promise<T>,
+): Promise<T> {
+  const t = newTrace();
+  try {
+    const result = await fn((k: string, v: any) => trace(t, k, v), t);
+    return result;
+  } catch (e: any) {
+    logger.error(`error ${e} while doing traced operation, state: ${JSON.stringify(t)}`);
+    throw e;
+  }
+}

package/src/ssh/Dockerfile

@@ -0,0 +1,29 @@
+FROM node:20
+
+RUN apt-get update && \
+    apt-get install -y openssh-server && \
+    mkdir /var/run/sshd
+
+# RUN echo 'root:rootpassword' | chpasswd
+
+RUN useradd 'pl-doctor'
+RUN echo 'pl-doctor:pl-doctor-password' | chpasswd
+
+RUN sed -i 's/^#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config && \
+    sed -i 's/^#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config
+
+RUN apt-get install -y locales && \
+    locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+EXPOSE 22 3001
+
+COPY ./test-assets/pub-key.pub /home/pl-doctor/.ssh/authorized_keys
+COPY ./test-assets/simple-server.js /home/pl-doctor/simple-server.js
+
+RUN chmod 755 /home/pl-doctor
+RUN chown pl-doctor:pl-doctor /home/pl-doctor
+
+RUN chmod +x /home/pl-doctor/simple-server.js
+
+CMD ["/bin/bash", "-c", "/usr/sbin/sshd -D & node /home/pl-doctor/simple-server.js"]

package/src/ssh/__tests__/common-utils.ts

@@ -0,0 +1,131 @@
+import type { StartedTestContainer } from 'testcontainers';
+import { GenericContainer } from 'testcontainers';
+import { writeFileSync, readFileSync, unlinkSync, existsSync } from 'fs';
+import { mkdir, rm } from 'fs/promises';
+import path from 'path';
+import type { ConnectConfig } from 'ssh2';
+import ssh from 'ssh2';
+import fs from 'fs';
+
+const SSH_PORT = [22, 3001];
+
+const publicKeyPath = getPathForFile('pub-key.pub');
+const privateKeyPath = getPathForFile('private-key.private');
+
+export const localFileUpload = getPathForFile('test-file.txt');
+export const localFileDownload = getPathForFile('test-file-download.txt');
+export const downloadsFolder = path.resolve(__dirname, '..', 'test-assets', 'downloads');
+export const recUpload = path.resolve(__dirname, '..', 'test-assets', 'downloads', 'rec-upload');
+
+export async function createTestDirForRecursiveUpload() {
+  const pathBase = path.resolve(__dirname, '..', 'test-assets', 'downloads', 'rec-upload', 'sub-1');
+  const path2 = path.resolve(__dirname, '..', 'test-assets', 'downloads', 'rec-upload', 'sub-1', 'sub-1-1');
+
+  await mkdir(pathBase, { recursive: true });
+  await mkdir(path2, { recursive: true });
+
+  for (let i = 0; i < 19; i++) {
+    const path2 = path.resolve(__dirname, '..', 'test-assets', 'downloads', 'rec-upload', 'sub-1', `sub-1-${i}`);
+    await mkdir(path2, { recursive: true });
+
+    for (let i = 0; i < 3; i++) {
+      writeFileSync(path.resolve(path2, `test-${i}.txt`), `test-${i}`);
+    }
+  }
+
+  for (let i = 1; i < 100; i++) {
+    writeFileSync(path.resolve(pathBase, `test-${i}.txt`), `test-${i}`);
+  }
+  writeFileSync(path.resolve(pathBase, `test.txt`), `test-1`);
+  writeFileSync(path.resolve(path2, 'test-5.txt'), 'test-5');
+}
+
+export function getPathForFile(fileName: string) {
+  return path.resolve(__dirname, '..', 'test-assets', fileName);
+}
+
+export function generateKeys() {
+  const keys = ssh.utils.generateKeyPairSync('ecdsa', { bits: 256, comment: 'node.js rules!' });
+  if (!existsSync(publicKeyPath) || !existsSync(privateKeyPath)) {
+    writeFileSync(publicKeyPath, keys.public);
+    writeFileSync(privateKeyPath, keys.private);
+  }
+}
+
+export function initPrivateKey(): string {
+  generateKeys();
+  return readFileSync(privateKeyPath, { encoding: 'utf-8' });
+}
+
+export async function initContainer(name: string): Promise<StartedTestContainer> {
+  await createTestDirForRecursiveUpload();
+
+  const fromCacheContainer = await new GenericContainer(`pl-ssh-test-container-${name}:1.0.0`)
+    .withExposedPorts(...SSH_PORT)
+    .withReuse()
+    .withName(`pl-ssh-test-${name}`)
+    .start()
+    .catch((err) => console.log('No worries, creating a new container'));
+
+  if (!fromCacheContainer) {
+    generateKeys();
+    const container1 = await GenericContainer.fromDockerfile(path.resolve(__dirname, '..'))
+      .withCache(true)
+      .build(`pl-ssh-test-container-${name}:1.0.0`, { deleteOnExit: false });
+
+    return container1.withExposedPorts(...SSH_PORT).withReuse().start();
+  }
+
+  return fromCacheContainer;
+}
+
+export function getContainerHostAndPort(container: StartedTestContainer) {
+  return {
+    port: container.getMappedPort(22),
+    host: container.getHost(),
+  };
+}
+
+function logToFile(message: string) {
+  const logFileName = 'log.txt';
+  const logFilePath = path.join(__dirname, logFileName);
+
+  const timestamp = new Date().toISOString();
+  const logMessage = `[${timestamp}] ${message}\n`;
+
+  fs.appendFile(logFilePath, logMessage, (err) => {
+    if (err) {
+      console.error('Error writing to log file:', err);
+    }
+  });
+}
+
+export function getConnectionForSsh(container: StartedTestContainer, debug: boolean = false): ConnectConfig {
+  const hostData = getContainerHostAndPort(container);
+  const privateKey = initPrivateKey();
+  const config = {
+    host: hostData.host,
+    port: hostData.port,
+    username: 'pl-doctor',
+    privateKey: privateKey,
+    debug: debug ? logToFile : undefined,
+  };
+  logToFile(JSON.stringify(config, null, 4));
+  return config;
+}
+
+export async function cleanUp(container: StartedTestContainer) {
+  await container.stop();
+
+  if (existsSync(localFileUpload)) {
+    unlinkSync(localFileUpload);
+  }
+
+  if (existsSync(localFileDownload)) {
+    unlinkSync(localFileDownload);
+  }
+
+  if (existsSync(recUpload)) {
+    await rm(recUpload, { recursive: true });
+  }
+}