@milaboratories/pl-middle-layer 1.37.4 → 1.37.6

package/src/network_check/pings.ts

@@ -0,0 +1,154 @@
+import type { ValueOrError } from '@milaboratories/ts-helpers';
+import { setTimeout } from 'node:timers/promises';
+import { request } from 'undici';
+import type { Dispatcher } from 'undici';
+import type { CheckNetworkOpts } from './network_check';
+import { UnauthenticatedPlClient, type PlClientConfig } from '@milaboratories/pl-client';
+
+/** A report about one concrete ping to the service. */
+export interface NetworkReport<T> {
+  elapsedMs: number;
+  response: ValueOrError<T>;
+}
+
+export type HttpNetworkReport = NetworkReport<{
+  statusCode: number;
+  beginningOfBody: string;
+}>;
+
+export async function backendPings(ops: CheckNetworkOpts, plConfig: PlClientConfig): Promise<NetworkReport<string>[]> {
+  return await recordPings(ops.pingCheckDurationMs, ops.maxPingsPerSecond, async () => {
+    const uaClient = new UnauthenticatedPlClient(plConfig);
+    const response = await uaClient.ping();
+    return JSON.stringify(response).slice(0, ops.bodyLimit) + '...';
+  });
+}
+
+export async function blockRegistryOverviewPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () =>
+      await requestUrl(new URL(ops.blockOverviewPath, ops.blockRegistryUrl), ops, httpClient),
+  );
+}
+
+export async function blockGARegistryOverviewPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () => await requestUrl(new URL(ops.blockOverviewPath, ops.blockGARegistryUrl), ops, httpClient),
+  );
+}
+
+export async function blockRegistryUiPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () => await requestUrl(new URL(ops.blockUiPath, ops.blockRegistryUrl), ops, httpClient),
+  );
+}
+
+export async function blockGARegistryUiPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.blockRegistryDurationMs,
+    ops.maxRegistryChecksPerSecond,
+    async () => await requestUrl(new URL(ops.blockUiPath, ops.blockGARegistryUrl), ops, httpClient),
+  );
+}
+
+export async function autoUpdateCdnPings(ops: CheckNetworkOpts, httpClient: Dispatcher): Promise<HttpNetworkReport[]> {
+  return await recordPings(
+    ops.autoUpdateCdnDurationMs,
+    ops.maxAutoUpdateCdnChecksPerSecond,
+    async () => await requestUrl(ops.autoUpdateCdnUrl, ops, httpClient),
+  );
+}
+
+/** Executes a body several times per second up to the given duration,
+ * and returns results and elapsed time for every result. */
+export async function recordPings<T>(
+  pingCheckDurationMs: number,
+  maxPingsPerSecond: number,
+  body: () => Promise<T>,
+): Promise<NetworkReport<T>[]> {
+  const startPings = Date.now();
+  const reports: NetworkReport<T>[] = [];
+
+  while (elapsed(startPings) < pingCheckDurationMs) {
+    const startPing = Date.now();
+    let response: ValueOrError<T>;
+    try {
+      response = { ok: true, value: await body() };
+    } catch (e) {
+      response = { ok: false, error: e };
+    }
+    const elapsedPing = elapsed(startPing);
+
+    reports.push({
+      elapsedMs: elapsedPing,
+      response,
+    });
+
+    const sleepBetweenPings = 1000 / maxPingsPerSecond - elapsedPing;
+
+    if (sleepBetweenPings > 0) {
+      await setTimeout(sleepBetweenPings);
+    }
+  }
+
+  return reports;
+}
+
+export async function requestUrl(url: string | URL, ops: CheckNetworkOpts, httpClient: Dispatcher) {
+  const { body: rawBody, statusCode } = await request(url, {
+    dispatcher: httpClient,
+    headersTimeout: ops.httpTimeoutMs,
+    bodyTimeout: ops.httpTimeoutMs,
+  });
+  const body = await rawBody.text();
+
+  return {
+    statusCode: statusCode,
+    beginningOfBody: body.slice(0, ops.bodyLimit) + '...',
+  };
+}
+
+export function elapsed(startMs: number): number {
+  return Date.now() - startMs;
+}
+
+export function reportToString<T>(report: NetworkReport<T>[]): {
+  ok: boolean;
+  details: string;
+} {
+  const successes = report.filter((r) => r.response.ok);
+  const errorsLen = report.length - successes.length;
+  const { mean: mean, median: median } = elapsedStat(report);
+
+  const details = `
+  total: ${report.length};
+  successes: ${successes.length};
+  errors: ${errorsLen};
+  mean in ms: ${mean};
+  median in ms: ${median};
+  `;
+
+  return {
+    ok: errorsLen === 0,
+    details,
+  };
+}
+
+function elapsedStat(reports: { elapsedMs: number }[]) {
+  const checks = reports.map((p) => p.elapsedMs).sort();
+  const mean = checks.reduce((sum, p) => sum + p) / checks.length;
+
+  let median = undefined;
+  if (checks.length > 0) {
+    const mid = Math.floor(checks.length / 2);
+    median = checks.length % 2 ? checks[mid] : (checks[mid - 1] + checks[mid]) / 2;
+  }
+
+  return { mean, median };
+}

package/src/network_check/template.test.ts

@@ -0,0 +1,83 @@
+import { createBigTempFile, runDownloadFile, runPythonSoftware, runSoftware, runUploadFile, runUploadTemplate } from './template';
+import { initNetworkCheck } from './network_check';
+import { testCredentials } from './test_utils';
+import { test, expect } from 'vitest';
+import path from 'path';
+
+test('check runUploadTemplate', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const { logger, client, terminate } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const greeting = await runUploadTemplate(logger, client, 'Jason');
+
+  expect(greeting).toBe('Hello, Jason');
+
+  await terminate();
+});
+
+test('check runUploadFile', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const {
+    logger,
+    lsDriver,
+    uploadBlobClient,
+    client,
+    terminate,
+  } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const { filePath } = await createBigTempFile();
+
+  const blob = await runUploadFile(
+    logger,
+    lsDriver,
+    uploadBlobClient,
+    client,
+    filePath,
+  );
+
+  expect(blob.type.name).toBe('Blob');
+
+  await terminate();
+});
+
+test('check runDownloadFile', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const {
+    logger,
+    lsDriver,
+    uploadBlobClient,
+    downloadClient,
+    client,
+    terminate,
+  } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const filePath = path.join(__dirname, '..', '..', 'test_assets', 'answer.txt');
+
+  const content = await runDownloadFile(logger, client, lsDriver, uploadBlobClient, downloadClient, filePath);
+
+  expect(content).toBe('42');
+
+  await terminate();
+});
+
+test('check runSoftware', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const { client, terminate } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const greeting = await runSoftware(client);
+
+  expect(greeting).toBe('Hello from go binary\n');
+
+  await terminate();
+});
+
+test('check runPythonSoftware', async () => {
+  const { plEndpoint, plUser, plPassword } = testCredentials();
+  const { client, terminate } = await initNetworkCheck(plEndpoint, plUser, plPassword);
+
+  const greeting = await runPythonSoftware(client, 'John');
+
+  expect(greeting).toBe('Hello, John!\n');
+
+  await terminate();
+});

package/src/network_check/template.ts

@@ -0,0 +1,330 @@
+import type { FieldId, FieldRef, PlClient, ResourceData } from '@milaboratories/pl-client';
+import { type PlTransaction, ContinuePolling, field, isNotNullResourceId, isNullResourceId, Pl, poll, toGlobalFieldId } from '@milaboratories/pl-client';
+import { createRenderTemplate } from '../mutator/template/render_template';
+import { Templates as SdkTemplates } from '@platforma-sdk/workflow-tengo';
+import type { TemplateSpecAny } from '../model/template_spec';
+import { loadTemplate, prepareTemplateSpec } from '../mutator/template/template_loading';
+import type { ClientDownload, LsDriver } from '@milaboratories/pl-drivers';
+import { ImportFileHandleUploadData, uploadBlob, type ClientUpload } from '@milaboratories/pl-drivers';
+import { notEmpty, type MiLogger } from '@milaboratories/ts-helpers';
+import type { ResourceInfo } from '@milaboratories/pl-tree';
+import { text } from 'node:stream/consumers';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import { randomBytes } from 'node:crypto';
+
+export interface TemplateReport {
+  ok: boolean;
+  message: string;
+}
+
+/** Uploads `hello-world` template and checks the output is correct. */
+export async function uploadTemplate(logger: MiLogger, pl: PlClient, name: string): Promise<TemplateReport> {
+  try {
+    const gotGreeting = await runUploadTemplate(logger, pl, name);
+    if (gotGreeting !== `Hello, ${name}`) {
+      return { ok: false, message: `Template uploading failed: expected: ${name}, got: ${gotGreeting}` };
+    }
+
+    return { ok: true, message: `Template uploading succeeded: ${gotGreeting}` };
+  } catch (e: unknown) {
+    return { ok: false, message: `Template uploading failed: error occurred: ${e}` };
+  }
+}
+
+export async function runUploadTemplate(
+  logger: MiLogger,
+  pl: PlClient,
+  name: string,
+): Promise<string> {
+  const greeting = await runTemplate(
+    pl,
+    SdkTemplates['check_network.upload_template'],
+    true,
+    (tx) => ({
+      name: tx.createValue(Pl.JsonObject, JSON.stringify(name)),
+    }),
+    ['greeting'],
+  );
+
+  try {
+    return JSON.parse(notEmpty((await getFieldValue(pl, greeting.greeting)).data?.toString()));
+  } finally {
+    await deleteFields(pl, Object.values(greeting));
+  }
+}
+
+/** Uploads a file to the backend and checks the output is a Blob resource. */
+export async function uploadFile(
+  logger: MiLogger,
+  lsDriver: LsDriver,
+  uploadClient: ClientUpload,
+  pl: PlClient,
+  filePath: string,
+): Promise<TemplateReport> {
+  try {
+    const gotBlob = await runUploadFile(logger, lsDriver, uploadClient, pl, filePath);
+
+    if (gotBlob.type.name !== 'Blob') {
+      return { ok: false, message: `File uploading failed: ${gotBlob.type.name}` };
+    }
+
+    return { ok: true, message: `File uploading succeeded: ${gotBlob.type.name}` };
+  } catch (e: unknown) {
+    return { ok: false, message: `File uploading failed: error occurred: ${e}` };
+  }
+}
+
+export async function runUploadFile(
+  logger: MiLogger,
+  lsDriver: LsDriver,
+  uploadClient: ClientUpload,
+  pl: PlClient,
+  filePath: string,
+): Promise<ResourceInfo> {
+  const handle = await lsDriver.getLocalFileHandle(filePath);
+  const result = await runTemplate(
+    pl,
+    SdkTemplates['check_network.upload_blob'],
+    true,
+    (tx) => ({
+      file: tx.createValue(Pl.JsonObject, JSON.stringify(handle)),
+    }),
+    ['progress', 'file'],
+  );
+
+  try {
+    const progress = await getFieldValue(pl, result.progress);
+
+    await uploadBlob(
+      logger,
+      uploadClient,
+      progress,
+      ImportFileHandleUploadData.parse(JSON.parse(notEmpty(progress.data?.toString()))),
+      () => false,
+      {
+        nPartsWithThisUploadSpeed: 10,
+        nPartsToIncreaseUpload: 10,
+        currentSpeed: 10,
+        maxSpeed: 10,
+      },
+    );
+
+    return await getFieldValue(pl, result.file);
+  } finally {
+    await deleteFields(pl, Object.values(result));
+  }
+}
+
+/** Uploads a file to the backend and then tries to download it back. */
+export async function downloadFile(
+  logger: MiLogger,
+  pl: PlClient,
+  lsDriver: LsDriver,
+  uploadClient: ClientUpload,
+  downloadClient: ClientDownload,
+  filePath: string,
+  fileContent: string,
+): Promise<TemplateReport> {
+  try {
+    const gotFileContent = await runDownloadFile(logger, pl, lsDriver, uploadClient, downloadClient, filePath);
+
+    if (gotFileContent !== fileContent) {
+      return { ok: false, message: `File downloading failed: expected: ${fileContent}, got: ${gotFileContent}` };
+    }
+    return { ok: true, message: `File downloading succeeded: ${gotFileContent}` };
+  } catch (e: unknown) {
+    return { ok: false, message: `File downloading failed: error occurred: ${e}` };
+  }
+}
+
+export async function runDownloadFile(
+  logger: MiLogger,
+  pl: PlClient,
+  lsDriver: LsDriver,
+  uploadClient: ClientUpload,
+  downloadClient: ClientDownload,
+  filePath: string,
+) {
+  const handle = await lsDriver.getLocalFileHandle(filePath);
+
+  const outputs = await runTemplate(
+    pl,
+    SdkTemplates['check_network.download_blob'],
+    true,
+    (tx) => ({ file: tx.createValue(Pl.JsonObject, JSON.stringify(handle)) }),
+    ['progress', 'file'],
+  );
+
+  try {
+    const progress = await getFieldValue(pl, outputs.progress);
+
+    await uploadBlob(
+      logger,
+      uploadClient,
+      progress,
+      ImportFileHandleUploadData.parse(JSON.parse(notEmpty(progress.data?.toString()))),
+      () => false,
+      {
+        nPartsWithThisUploadSpeed: 1,
+        nPartsToIncreaseUpload: 1,
+        currentSpeed: 1,
+        maxSpeed: 1,
+      },
+    );
+
+    const fileInfo = await getFieldValue(pl, outputs.file);
+    const { content } = await downloadClient.downloadBlob(fileInfo);
+
+    return await text(content);
+  } finally {
+    await deleteFields(pl, Object.values(outputs));
+  }
+}
+
+/** Runs Go's hello-world binary. */
+export async function softwareCheck(pl: PlClient): Promise<TemplateReport> {
+  try {
+    const gotGreeting = await runSoftware(pl);
+
+    if (gotGreeting !== 'Hello from go binary\n') {
+      return { ok: false, message: `Software check failed: got: ${gotGreeting}` };
+    }
+    return { ok: true, message: `Software check succeeded: ${gotGreeting}` };
+  } catch (e: unknown) {
+    return { ok: false, message: `Software check failed: error occurred: ${e}` };
+  }
+}
+
+export async function runSoftware(pl: PlClient): Promise<string> {
+  const result = await runTemplate(
+    pl,
+    SdkTemplates['check_network.run_hello_world'],
+    true,
+    (_: PlTransaction) => ({}),
+    ['greeting'],
+  );
+
+  try {
+    return notEmpty((await getFieldValue(pl, result.greeting)).data?.toString());
+  } finally {
+    await deleteFields(pl, Object.values(result));
+  }
+}
+
+/** Runs Python hello-world. */
+export async function pythonSoftware(pl: PlClient, name: string): Promise<TemplateReport> {
+  try {
+    const gotGreeting = await runPythonSoftware(pl, name);
+
+    if (gotGreeting !== `Hello, ${name}!\n`) {
+      return { ok: false, message: `Python software check failed: got: ${gotGreeting}` };
+    }
+    return { ok: true, message: `Python software check succeeded: ${gotGreeting}` };
+  } catch (e: unknown) {
+    return { ok: false, message: `Python software check failed: error occurred: ${e}` };
+  }
+}
+
+export async function runPythonSoftware(pl: PlClient, name: string): Promise<string> {
+  const result = await runTemplate(
+    pl,
+    SdkTemplates['check_network.run_hello_world_py'],
+    true,
+    (tx) => ({ name: tx.createValue(Pl.JsonObject, JSON.stringify(name)) }),
+    ['greeting'],
+  );
+
+  try {
+    return notEmpty((await getFieldValue(pl, result.greeting)).data?.toString());
+  } finally {
+    await deleteFields(pl, Object.values(result));
+  }
+}
+
+/** Creates a big temporary file with random content. */
+export async function createBigTempFile(): Promise<{ filePath: string }> {
+  const filePath = path.join(os.tmpdir(), `check-network-big-temp-${Date.now()}.bin`);
+  const fileSize = 20 * 1024 * 1024; // 20 MiB
+
+  const fileContent = randomBytes(fileSize);
+
+  await fs.appendFile(filePath, fileContent);
+
+  return { filePath };
+}
+
+/** Creates a temporarly file we could use for uploading and downloading. */
+export async function createTempFile(): Promise<{ filePath: string; fileContent: string }> {
+  const filePath = path.join(os.tmpdir(), `check-network-temp-${Date.now()}.txt`);
+
+  const fileContent = 'Hello, world! ' + new Date().toISOString();
+  await fs.writeFile(filePath, fileContent);
+
+  return { filePath, fileContent };
+}
+
+/** Creates a template and RenderTemplate resources, gets all resources from outputs.
+ * Throws a error if any of the outputs failed.
+ */
+async function runTemplate(
+  client: PlClient,
+  tpl: TemplateSpecAny,
+  ephemeral: boolean,
+  inputs: (tx: PlTransaction) => Pl.PlRecord,
+  outputs: string[],
+): Promise<Record<string, FieldId>> {
+  return await client.withWriteTx('TemplateRender', async (tx) => {
+    const preparedTemplate = await prepareTemplateSpec(tpl);
+    const tplResource = loadTemplate(tx, preparedTemplate);
+
+    const outputFields: Record<string, FieldRef> = createRenderTemplate(
+      tx, tplResource, ephemeral, inputs(tx), outputs,
+    );
+
+    const outputsIds: Record<string, FieldId> = {};
+
+    for (const output of outputs) {
+      const fieldRef = field(client.clientRoot, output);
+      tx.createField(fieldRef, 'Dynamic', outputFields[output]);
+      outputsIds[output] = await toGlobalFieldId(fieldRef);
+    }
+
+    await tx.commit();
+
+    return outputsIds;
+  });
+}
+
+/** Gets a resource from field's value or throws a error. */
+async function getFieldValue(
+  client: PlClient,
+  fieldId: FieldId,
+): Promise<ResourceData> {
+  // We could also do polling with pl-tree, but it seemed like an overkill,
+  // that's why we have a simple polling here.
+
+  return await poll(client, async (tx) => {
+    const field = await tx.tx.getField(fieldId);
+    if (isNotNullResourceId(field.error)) {
+      const err = await tx.tx.getResourceData(field.error, true);
+      throw new Error(`getFieldValue of "${fieldId.fieldName}" field failed: ${err.data}`);
+    }
+
+    if (isNullResourceId(field.value)) {
+      throw new ContinuePolling();
+    }
+
+    return await tx.tx.getResourceData(field.value, true);
+  });
+}
+
+async function deleteFields(client: PlClient, fieldIds: FieldId[]) {
+  await client.withWriteTx('DeleteFields', async (tx) => {
+    for (const fieldId of fieldIds) {
+      tx.resetField(fieldId);
+    }
+    await tx.commit();
+  });
+}

package/src/network_check/test_utils.ts

@@ -0,0 +1,9 @@
+export function testCredentials() {
+  expect(process.env.PL_ADDRESS).not.toBeUndefined();
+
+  return {
+    plEndpoint: process.env.PL_ADDRESS!,
+    plUser: process.env.PL_TEST_USER, // maybe undefined
+    plPassword: process.env.PL_TEST_PASSWORD, // maybe undefined
+  };
+}

package/dist/network_check.d.ts

@@ -1,29 +0,0 @@
-/** A utility to check network problems and gather statistics.
- * It's useful when we cannot connect to the server of a company
- * because of security reasons,
- * but they can send us and their DevOps team this report. */
-export interface CheckNetworkOpts {
-    /** Platforma Backend pings options. */
-    pingCheckDurationMs: number;
-    pingTimeoutMs: number;
-    maxPingsPerSecond: number;
-    /** An options for CDN and block registry. */
-    httpTimeoutMs: number;
-    /** Block registry pings options. */
-    blockRegistryDurationMs: number;
-    maxRegistryChecksPerSecond: number;
-    blockRegistryUrl: string;
-    blockGARegistryUrl: string;
-    blockOverviewPath: string;
-    blockUiPath: string;
-    /** CDN for auto-update pings options. */
-    autoUpdateCdnDurationMs: number;
-    maxAutoUpdateCdnChecksPerSecond: number;
-    autoUpdateCdnUrl: string;
-    bodyLimit: number;
-}
-/** Checks connectivity to Platforma Backend, to block registry
- * and to auto-update CDN,
- * and generates a string report. */
-export declare function checkNetwork(plCredentials: string, optsOverrides?: Partial<CheckNetworkOpts>): Promise<string>;
-//# sourceMappingURL=network_check.d.ts.map

package/dist/network_check.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"network_check.d.ts","sourceRoot":"","sources":["../src/network_check.ts"],"names":[],"mappings":"AAAA;;;6DAG6D;AAqB7D,MAAM,WAAW,gBAAgB;IAC/B,uCAAuC;IACvC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAE1B,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAC;IAEtB,oCAAoC;IACpC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IAEpB,yCAAyC;IACzC,uBAAuB,EAAE,MAAM,CAAC;IAChC,+BAA+B,EAAE,MAAM,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IAEzB,SAAS,EAAE,MAAM,CAAC;CACnB;AAwCD;;oCAEoC;AACpC,wBAAsB,YAAY,CAChC,aAAa,EAAE,MAAM,EACrB,aAAa,GAAE,OAAO,CAAC,gBAAgB,CAAM,GAC5C,OAAO,CAAC,MAAM,CAAC,CAmGjB"}

package/dist/network_check.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"network_check.test.d.ts","sourceRoot":"","sources":["../src/network_check.test.ts"],"names":[],"mappings":""}