@milaboratories/pl-middle-layer 1.37.3 → 1.37.5

package/src/network_check/network_check.ts

@@ -0,0 +1,369 @@
+/** A utility to check network problems and gather statistics.
+ * It's useful when we cannot connect to the server of a company
+ * because of security reasons,
+ * but they can send us and their DevOps team this report.
+ *
+ * What we check:
+ * - pings to backend
+ * - block registry for block overview and ui.
+ * - autoupdate CDN.
+ * - upload workflow to backend (workflow part via our API).
+ * - the desktop could do multipart upload.
+ * - the desktop could download files from S3.
+ * - backend could download software and run it.
+ * - backend could run python software.
+ * TODO:
+ * - try to get something from backend's library storage.
+ *
+ * We don't check backend access to S3 storage, it is checked on the start of backend.
+ */
+
+import type { AuthInformation, PlClientConfig } from '@milaboratories/pl-client';
+import { PlClient, UnauthenticatedPlClient, plAddressToConfig } from '@milaboratories/pl-client';
+import type { MiLogger } from '@milaboratories/ts-helpers';
+import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
+import { channel } from 'node:diagnostics_channel';
+import type { ClientDownload, ClientUpload } from '@milaboratories/pl-drivers';
+import { LsDriver, createDownloadClient, createUploadBlobClient } from '@milaboratories/pl-drivers';
+import type { HttpNetworkReport, NetworkReport } from './pings';
+import { autoUpdateCdnPings, backendPings, blockGARegistryOverviewPings, blockGARegistryUiPings, blockRegistryOverviewPings, blockRegistryUiPings, reportToString } from './pings';
+import type { Dispatcher } from 'undici';
+import type { TemplateReport } from './template';
+import { uploadTemplate, uploadFile, downloadFile, createTempFile, pythonSoftware, softwareCheck, createBigTempFile } from './template';
+
+/** All reports we need to collect. */
+interface NetworkReports {
+  plPings: NetworkReport<string>[];
+
+  blockRegistryOverviewChecks: HttpNetworkReport[];
+  blockGARegistryOverviewChecks: HttpNetworkReport[];
+  blockRegistryUiChecks: HttpNetworkReport[];
+  blockGARegistryUiChecks: HttpNetworkReport[];
+
+  autoUpdateCdnChecks: HttpNetworkReport[];
+
+  uploadTemplateCheck: TemplateReport;
+  uploadFileCheck: TemplateReport;
+  downloadFileCheck: TemplateReport;
+  softwareCheck: TemplateReport;
+  pythonSoftwareCheck: TemplateReport;
+}
+
+export interface CheckNetworkOpts {
+  /** Platforma Backend pings options. */
+  pingCheckDurationMs: number;
+  pingTimeoutMs: number;
+  maxPingsPerSecond: number;
+
+  /** An options for CDN and block registry. */
+  httpTimeoutMs: number;
+
+  /** Block registry pings options. */
+  blockRegistryDurationMs: number;
+  maxRegistryChecksPerSecond: number;
+  blockRegistryUrl: string;
+  blockGARegistryUrl: string;
+  blockOverviewPath: string;
+  blockUiPath: string;
+
+  /** CDN for auto-update pings options. */
+  autoUpdateCdnDurationMs: number;
+  maxAutoUpdateCdnChecksPerSecond: number;
+  autoUpdateCdnUrl: string;
+
+  bodyLimit: number;
+}
+
+/** Checks connectivity to Platforma Backend, to block registry
+ * and to auto-update CDN,
+ * and generates a string report. */
+export async function checkNetwork(
+  plCredentials: string,
+  plUser: string | undefined,
+  plPassword: string | undefined,
+  optsOverrides: Partial<CheckNetworkOpts> = {},
+): Promise<string> {
+  const undiciLogs: any[] = [];
+  // Subscribe to all Undici diagnostic events
+  undiciEvents.forEach((event) => {
+    const diagnosticChannel = channel(event);
+    diagnosticChannel.subscribe((message: any) => {
+      const timestamp = new Date().toISOString();
+      const data = { ...message };
+      if (data?.response?.headers) {
+        data.response = { ...data.response };
+        data.response.headers = data.response.headers.slice();
+        data.response.headers = data.response.headers.map((h: any) => h.toString());
+      }
+
+      // we try to upload big files, don't include the buffer in the report.
+      if (data?.request?.body) {
+        data.request = { ...data.request };
+        data.request.body = `too big`;
+      }
+
+      undiciLogs.push(
+        JSON.stringify({
+          timestamp,
+          event,
+          data,
+        }),
+      );
+    });
+  });
+
+  const {
+    logger,
+    plConfig,
+    client,
+    downloadClient,
+    uploadBlobClient,
+    lsDriver,
+    httpClient,
+    ops,
+  } = await initNetworkCheck(plCredentials, plUser, plPassword, optsOverrides);
+
+  const { filePath: filePathToDownload, fileContent: fileContentToDownload } = await createTempFile();
+  const { filePath: filePathToUpload } = await createBigTempFile();
+
+  const report: NetworkReports = {
+    plPings: await backendPings(ops, plConfig),
+    blockRegistryOverviewChecks: await blockRegistryOverviewPings(ops, httpClient),
+    blockGARegistryOverviewChecks: await blockGARegistryOverviewPings(ops, httpClient),
+    blockRegistryUiChecks: await blockRegistryUiPings(ops, httpClient),
+    blockGARegistryUiChecks: await blockGARegistryUiPings(ops, httpClient),
+
+    autoUpdateCdnChecks: await autoUpdateCdnPings(ops, httpClient),
+
+    uploadTemplateCheck: await uploadTemplate(logger, client, 'Jack'),
+    uploadFileCheck: await uploadFile(logger, lsDriver, uploadBlobClient, client, filePathToUpload),
+    downloadFileCheck: await downloadFile(logger, client, lsDriver, uploadBlobClient, downloadClient, filePathToDownload, fileContentToDownload),
+    softwareCheck: await softwareCheck(client),
+    pythonSoftwareCheck: await pythonSoftware(client, 'Jack'),
+  };
+
+  return reportsToString(report, plCredentials, ops, undiciLogs);
+}
+
+export async function initNetworkCheck(
+  plCredentials: string,
+  plUser: string | undefined,
+  plPassword: string | undefined,
+  optsOverrides: Partial<CheckNetworkOpts> = {},
+): Promise<{
+    logger: MiLogger;
+    plConfig: PlClientConfig;
+    client: PlClient;
+    downloadClient: ClientDownload;
+    uploadBlobClient: ClientUpload;
+    lsDriver: LsDriver;
+    httpClient: Dispatcher;
+    ops: CheckNetworkOpts;
+    terminate: () => Promise<void>;
+  }> {
+  const ops: CheckNetworkOpts = {
+    pingCheckDurationMs: 10000,
+    pingTimeoutMs: 3000,
+    maxPingsPerSecond: 50,
+
+    httpTimeoutMs: 3000,
+
+    blockRegistryDurationMs: 3000,
+    maxRegistryChecksPerSecond: 1,
+
+    blockRegistryUrl: 'https://blocks.pl-open.science',
+    blockGARegistryUrl: 'https://blocks-ga.pl-open.science',
+    blockOverviewPath: 'v2/overview.json',
+    blockUiPath: 'v2/milaboratories/samples-and-data/1.7.0/ui.tgz',
+
+    autoUpdateCdnDurationMs: 5000,
+    maxAutoUpdateCdnChecksPerSecond: 1,
+    autoUpdateCdnUrl:
+      'https://cdn.platforma.bio/software/platforma-desktop-v2/windows/amd64/latest.yml',
+
+    bodyLimit: 300,
+
+    ...optsOverrides,
+  };
+
+  const plConfig = plAddressToConfig(plCredentials, {
+    defaultRequestTimeout: ops.pingTimeoutMs,
+  });
+
+  // exposing alternative root for fields not to interfere with
+  // projects of the user.
+  plConfig.alternativeRoot = `check_network_${Date.now()}`;
+
+  const uaClient = new UnauthenticatedPlClient(plConfig);
+
+  let auth: AuthInformation = {};
+  if (plUser && plPassword) {
+    auth = await uaClient.login(plUser, plPassword);
+  }
+
+  const client = await PlClient.init(plCredentials, { authInformation: auth });
+
+  const httpClient = uaClient.ll.httpDispatcher;
+  const logger = new ConsoleLoggerAdapter();
+
+  // FIXME: do we need to get an actual secret?
+  const signer = new HmacSha256Signer('localSecret');
+
+  // We could initialize middle-layer here, but for now it seems like an overkill.
+  // Here's the code to do it:
+  //
+  // const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'platforma-network-check-'));
+  // const ml = await MiddleLayer.init(client, tmpDir, {
+  //   logger,
+  //   localSecret: '',
+  //   localProjections: [],
+  //   openFileDialogCallback: () => Promise.resolve([]),
+  //   preferredUpdateChannel: 'stable',
+  // });
+
+  const downloadClient = createDownloadClient(logger, client, []);
+  const uploadBlobClient = createUploadBlobClient(client, logger);
+
+  const lsDriver = await LsDriver.init(
+    logger,
+    client,
+    signer,
+    [],
+    () => Promise.resolve([]),
+    [],
+  );
+
+  const terminate = async () => {
+    downloadClient.close();
+    uploadBlobClient.close();
+    await httpClient.close();
+    await client.close();
+  };
+
+  return {
+    logger,
+    plConfig,
+    client,
+    downloadClient,
+    uploadBlobClient,
+    lsDriver,
+    httpClient,
+    ops,
+    terminate,
+  };
+}
+
+function reportsToString(
+  report: NetworkReports,
+  plEndpoint: string,
+  opts: CheckNetworkOpts,
+  undiciLogs: any[],
+): string {
+  const successPings = report.plPings.filter((p) => p.response.ok);
+  const failedPings = report.plPings.filter((p) => !p.response.ok);
+  const successPingsBodies = [
+    ...new Set(successPings.map((p) => JSON.stringify((p.response as any).value))),
+  ];
+
+  const pings = reportToString(report.plPings);
+  const blockRegistryOverview = reportToString(report.blockRegistryOverviewChecks);
+  const blockGARegistryOverview = reportToString(report.blockGARegistryOverviewChecks);
+  const blockRegistryUi = reportToString(report.blockRegistryUiChecks);
+  const blockGARegistryUi = reportToString(report.blockGARegistryUiChecks);
+  const autoUpdateCdn = reportToString(report.autoUpdateCdnChecks);
+
+  const summary = (ok: boolean) => ok ? 'OK' : 'FAILED';
+
+  return `
+Network report:
+pl endpoint: ${plEndpoint};
+
+summary:
+${summary(pings.ok)} pings to Platforma Backend
+${summary(blockRegistryOverview.ok)} block registry overview
+${summary(blockGARegistryOverview.ok)} block ga registry overview
+${summary(blockRegistryUi.ok)} block registry ui
+${summary(blockGARegistryUi.ok)} block ga registry ui
+${summary(autoUpdateCdn.ok)} auto-update CDN
+${summary(report.uploadTemplateCheck.ok)} upload template
+${summary(report.uploadFileCheck.ok)} upload file
+${summary(report.downloadFileCheck.ok)} download file
+${summary(report.softwareCheck.ok)} software check
+${summary(report.pythonSoftwareCheck.ok)} python software check
+
+details:
+options: ${JSON.stringify(opts, null, 2)}.
+
+Upload template response: ${report.uploadTemplateCheck.message}
+
+Upload file response: ${report.uploadFileCheck.message}
+
+Download file response: ${report.downloadFileCheck.message}
+
+Software check response: ${report.softwareCheck.message}
+Python software check response: ${report.pythonSoftwareCheck.message}
+Platforma pings: ${pings.details}
+
+Block registry overview responses: ${blockRegistryOverview.details}
+
+Block ga registry overview responses: ${blockGARegistryOverview.details}
+
+Block registry ui responses: ${blockRegistryUi.details}
+
+Block ga registry ui responses: ${blockGARegistryUi.details}
+
+Auto-update CDN responses: ${autoUpdateCdn.details}
+
+dumps:
+Block registry overview dumps:
+${JSON.stringify(report.blockRegistryOverviewChecks, null, 2)}
+
+Block ga registry overview dumps:
+${JSON.stringify(report.blockGARegistryOverviewChecks, null, 2)}
+
+Block registry ui dumps:
+${JSON.stringify(report.blockRegistryUiChecks, null, 2)}
+
+Block ga registry ui dumps:
+${JSON.stringify(report.blockGARegistryUiChecks, null, 2)}
+
+Auto-update CDN dumps:
+${JSON.stringify(report.autoUpdateCdnChecks, null, 2)}
+
+Platforma pings error dumps:
+${JSON.stringify(failedPings, null, 2)}
+
+Platforma pings success dump examples:
+${JSON.stringify(successPingsBodies, null, 2)}
+
+Undici logs:
+${undiciLogs.join('\n')}
+`;
+}
+
+// List of Undici diagnostic channels
+const undiciEvents: string[] = [
+  'undici:request:create', // When a new request is created
+  'undici:request:bodySent', // When the request body is sent
+  'undici:request:headers', // When request headers are sent
+  'undici:request:error', // When a request encounters an error
+  'undici:request:trailers', // When a response completes.
+
+  'undici:client:sendHeaders',
+  'undici:client:beforeConnect',
+  'undici:client:connected',
+  'undici:client:connectError',
+
+  'undici:socket:close', // When a socket is closed
+  'undici:socket:connect', // When a socket connects
+  'undici:socket:error', // When a socket encounters an error
+
+  'undici:pool:request', // When a request is added to the pool
+  'undici:pool:connect', // When a pool creates a new connection
+  'undici:pool:disconnect', // When a pool connection is closed
+  'undici:pool:destroy', // When a pool is destroyed
+  'undici:dispatcher:request', // When a dispatcher processes a request
+  'undici:dispatcher:connect', // When a dispatcher connects
+  'undici:dispatcher:disconnect', // When a dispatcher disconnects
+  'undici:dispatcher:retry', // When a dispatcher retries a request
+];

package/src/network_check/pings.ts

@@ -0,0 +1,154 @@
+import type { ValueOrError } from '@milaboratories/ts-helpers';
+import { setTimeout } from 'node:timers/promises';
+import { request } from 'undici';
+import type { Dispatcher } from 'undici';
+import type { CheckNetworkOpts } from './network_check';
+import { UnauthenticatedPlClient, type PlClientConfig } from '@milaboratories/pl-client';
+
+/** A report about one concrete ping to the service. */
+export interface NetworkReport<T> {
+  elapsedMs: number;
+  response: ValueOrError<T>;
+}
+
+export type HttpNetworkReport = NetworkReport<{
+  statusCode: number;
+  beginningOfBody: string;
+}>;
+
+export async function backendPings(ops: CheckNetworkOpts, plConfig: PlClientConfig): Promise<NetworkReport<string>[]> {
+  return await recordPings(ops.pingCheckDurationMs, ops.maxPingsPerSecond, async () => {
+    const uaClient = new UnauthenticatedPlClient(plConfig);
+    const response = await uaClient.ping();
+    return JSON.stringify(response).slice(0, ops.bodyLimit) + '...';
+  });
+}
+
+export async function blockRegistryOverviewPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () =>
+      await requestUrl(new URL(ops.blockOverviewPath, ops.blockRegistryUrl), ops, httpClient),
+  );
+}
+
+export async function blockGARegistryOverviewPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () => await requestUrl(new URL(ops.blockOverviewPath, ops.blockGARegistryUrl), ops, httpClient),
+  );
+}
+
+export async function blockRegistryUiPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () => await requestUrl(new URL(ops.blockUiPath, ops.blockRegistryUrl), ops, httpClient),
+  );
+}
+
+export async function blockGARegistryUiPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () => await requestUrl(new URL(ops.blockUiPath, ops.blockGARegistryUrl), ops, httpClient),
+  );
+}
+
+export async function autoUpdateCdnPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.autoUpdateCdnDurationMs,
+    ops.maxAutoUpdateCdnChecksPerSecond,
+    async () => await requestUrl(ops.autoUpdateCdnUrl, ops, httpClient),
+  );
+}
+
+/** Executes a body several times per second up to the given duration,
+ * and returns results and elapsed time for every result. */
+export async function recordPings<T>(
+  pingCheckDurationMs: number,
+  maxPingsPerSecond: number,
+  body: () => Promise<T>,
+): Promise<NetworkReport<T>[]> {
+  const startPings = Date.now();
+  const reports: NetworkReport<T>[] = [];
+
+  while (elapsed(startPings) < pingCheckDurationMs) {
+    const startPing = Date.now();
+    let response: ValueOrError<T>;
+    try {
+      response = { ok: true, value: await body() };
+    } catch (e) {
+      response = { ok: false, error: e };
+    }
+    const elapsedPing = elapsed(startPing);
+
+    reports.push({
+      elapsedMs: elapsedPing,
+      response,
+    });
+
+    const sleepBetweenPings = 1000 / maxPingsPerSecond - elapsedPing;
+
+    if (sleepBetweenPings > 0) {
+      await setTimeout(sleepBetweenPings);
+    }
+  }
+
+  return reports;
+}
+
+export async function requestUrl(url: string | URL, ops: CheckNetworkOpts, httpClient: Dispatcher) {
+  const { body: rawBody, statusCode } = await request(url, {
+    dispatcher: httpClient,
+    headersTimeout: ops.httpTimeoutMs,
+    bodyTimeout: ops.httpTimeoutMs,
+  });
+  const body = await rawBody.text();
+
+  return {
+    statusCode: statusCode,
+    beginningOfBody: body.slice(0, ops.bodyLimit) + '...',
+  };
+}
+
+export function elapsed(startMs: number): number {
+  return Date.now() - startMs;
+}
+
+export function reportToString<T>(report: NetworkReport<T>[]): {
+  ok: boolean;
+  details: string;
+} {
+  const successes = report.filter((r) => r.response.ok);
+  const errorsLen = report.length - successes.length;
+  const { mean: mean, median: median } = elapsedStat(report);
+
+  const details = `
+  total: ${report.length};
+  successes: ${successes.length};
+  errors: ${errorsLen};
+  mean in ms: ${mean};
+  median in ms: ${median};
+  `;
+
+  return {
+    ok: errorsLen === 0,
+    details,
+  };
+}
+
+function elapsedStat(reports: { elapsedMs: number }[]) {
+  const checks = reports.map((p) => p.elapsedMs).sort();
+  const mean = checks.reduce((sum, p) => sum + p) / checks.length;
+
+  let median = undefined;
+  if (checks.length > 0) {
+    const mid = Math.floor(checks.length / 2);
+    median = checks.length % 2 ? checks[mid] : (checks[mid - 1] + checks[mid]) / 2;
+  }
+
+  return { mean, median };
+}

package/src/network_check/template.test.ts

@@ -0,0 +1,83 @@
+import { createBigTempFile, runDownloadFile, runPythonSoftware, runSoftware, runUploadFile, runUploadTemplate } from './template';
+import { initNetworkCheck } from './network_check';
+import { testCredentials } from './test_utils';
+import { test, expect } from 'vitest';
+import path from 'path';
+
+test('check runUploadTemplate', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const { logger, client, terminate } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const greeting = await runUploadTemplate(logger, client, 'Jason');
+
+  expect(greeting).toBe('Hello, Jason');
+
+  await terminate();
+});
+
+test('check runUploadFile', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const {
+    logger,
+    lsDriver,
+    uploadBlobClient,
+    client,
+    terminate,
+  } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const { filePath } = await createBigTempFile();
+
+  const blob = await runUploadFile(
+    logger,
+    lsDriver,
+    uploadBlobClient,
+    client,
+    filePath,
+  );
+
+  expect(blob.type.name).toBe('Blob');
+
+  await terminate();
+});
+
+test('check runDownloadFile', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const {
+    logger,
+    lsDriver,
+    uploadBlobClient,
+    downloadClient,
+    client,
+    terminate,
+  } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const filePath = path.join(__dirname, '..', '..', 'test_assets', 'answer.txt');
+
+  const content = await runDownloadFile(logger, client, lsDriver, uploadBlobClient, downloadClient, filePath);
+
+  expect(content).toBe('42');
+
+  await terminate();
+});
+
+test('check runSoftware', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const { client, terminate } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const greeting = await runSoftware(client);
+
+  expect(greeting).toBe('Hello from go binary\n');
+
+  await terminate();
+});
+
+test('check runPythonSoftware', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const { client, terminate } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const greeting = await runPythonSoftware(client, 'John');
+
+  expect(greeting).toBe('Hello, John!\n');
+
+  await terminate();
+});