@testomatio/reporter 2.0.1-beta.5-timestamp → 2.0.1-beta.7

package/src/uploader.js

@@ -0,0 +1,376 @@
+import createDebugMessages from 'debug';
+import { S3 } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import promiseRetry from 'promise-retry';
+import pc from 'picocolors';
+import { APP_PREFIX } from './constants.js';
+import { filesize as prettyBytes } from 'filesize';
+
+const debug = createDebugMessages('@testomatio/reporter:file-uploader');
+
+export class S3Uploader {
+  constructor() {
+    this.isEnabled = undefined;
+    this.storeEnabled = true;
+    this.config = undefined;
+
+    /**
+     * @type {{path: string, size: number}[]}
+     */
+    this.skippedUploads = [];
+    this.failedUploads = [];
+    /**
+     * @type {{path: string, size: number, link: string}[]}
+     */
+    this.successfulUploads = [];
+
+    this.configKeys = [
+      'S3_ENDPOINT',
+      'S3_REGION',
+      'S3_BUCKET',
+      'S3_ACCESS_KEY_ID',
+      'S3_SECRET_ACCESS_KEY',
+      'S3_SESSION_TOKEN',
+      'S3_FORCE_PATH_STYLE',
+      'TESTOMATIO_DISABLE_ARTIFACTS',
+      'TESTOMATIO_PRIVATE_ARTIFACTS',
+      'TESTOMATIO_ARTIFACT_MAX_SIZE_MB',
+    ];
+  }
+
+  resetConfig() {
+    this.config = undefined;
+    this.isEnabled = undefined;
+  }
+
+  /**
+   *
+   * @returns {Record<string, string>}
+   */
+  getConfig() {
+    if (this.config) return this.config;
+    this.config = this.configKeys.reduce((acc, key) => {
+      acc[key] = process.env[key];
+      return acc;
+    }, {});
+    return this.config;
+  }
+
+  getMaskedConfig() {
+    return Object.fromEntries(
+      Object.entries(this.getConfig()).map(([key, value]) => [
+        key,
+        key === 'S3_SECRET_ACCESS_KEY' || key === 'S3_ACCESS_KEY_ID' ? '***' : value,
+      ]),
+    );
+  }
+
+  checkEnabled() {
+    if (this.isEnabled !== undefined) return this.isEnabled;
+
+    const { S3_BUCKET, TESTOMATIO_DISABLE_ARTIFACTS } = this.getConfig();
+    if (!S3_BUCKET) debug(`Artifacts uploading is disabled because S3_BUCKET is not set`);
+    this.isEnabled = !!(S3_BUCKET && !TESTOMATIO_DISABLE_ARTIFACTS);
+
+    if (this.isEnabled) debug('S3 uploader is enabled');
+    debug(this.getMaskedConfig());
+
+    return this.isEnabled;
+  }
+
+  enableLogStorage() {
+    this.storeEnabled = true;
+  }
+
+  disableLogStorage() {
+    this.storeEnabled = false;
+  }
+
+  /**
+   *
+   * @param {*} Body
+   * @param {*} Key
+   * @param {{path: string, size?: number}} file
+   * @returns
+   */
+  async #uploadToS3(Body, Key, file) {
+    const { S3_BUCKET, TESTOMATIO_PRIVATE_ARTIFACTS } = this.getConfig();
+    const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+
+    if (!S3_BUCKET || !Body) {
+      console.log(
+        APP_PREFIX,
+        pc.bold(pc.red(`Failed uploading '${Key}'. Please check S3 credentials`)),
+        this.getMaskedConfig(),
+      );
+      return;
+    }
+
+    debug('Uploading to S3:', Key);
+
+    const s3Config = this.#getS3Config();
+    const s3 = new S3(s3Config);
+    const params = {
+      Bucket: S3_BUCKET,
+      Key,
+      Body,
+    };
+    // disable ACL for I AM roles
+    if (!s3Config.credentials.sessionToken) {
+      params.ACL = ACL;
+    }
+
+    try {
+      const upload = new Upload({ client: s3, params });
+
+      const link = await this.getS3LocationLink(upload);
+      this.successfulUploads.push({ path: file.path, size: file.size, link });
+      debug(`📤 Uploaded artifact. File: ${file.path}, size: ${prettyBytes(file.size || 0)}, link: ${link}`);
+      return link;
+    } catch (e) {
+      this.failedUploads.push({ path: file.path, size: file.size });
+      debug('S3 uploading error:', e);
+      console.log(APP_PREFIX, 'Upload failed:', e.message, '\nConfig:\n', this.getMaskedConfig());
+    }
+  }
+
+  /**
+   * Returns an array of uploaded files
+   *
+   * @returns {{rid: string, file: string, uploaded: boolean}[]}
+   */
+  readUploadedFiles(runId) {
+    const tempFilePath = this.#getFilePathWithUploadsList(runId);
+
+    debug('Reading file', tempFilePath);
+
+    if (!fs.existsSync(tempFilePath)) {
+      debug('File not found:', tempFilePath);
+      return [];
+    }
+
+    const stats = fs.statSync(tempFilePath);
+    debug('Artifacts file stats:', +stats.mtime);
+    debug('Current time:', +new Date());
+    const diff = +new Date() - +stats.mtime;
+    debug('Diff:', diff);
+    const diffHours = diff / 1000 / 60 / 60;
+    debug('Diff hours:', diffHours);
+    if (diffHours > 3) {
+      console.log(APP_PREFIX, "Artifacts file is too old, can't process artifacts. Please re-run the tests.");
+      return [];
+    }
+
+    const data = fs.readFileSync(tempFilePath, 'utf8');
+    debug('Artifacts file contents:', data);
+    const lines = data.split('\n').filter(Boolean);
+    return lines.map(line => JSON.parse(line));
+  }
+
+  #getFilePathWithUploadsList(runId) {
+    const tempFilePath = path.join(os.tmpdir(), `testomatio.run.${runId}.json`);
+    if (!fs.existsSync(tempFilePath)) {
+      debug('Creating artifacts file:', tempFilePath);
+      fs.writeFileSync(tempFilePath, '');
+    }
+    return tempFilePath;
+  }
+
+  storeUploadedFile(filePath, runId, rid, uploaded = false) {
+    if (!this.storeEnabled) return;
+
+    if (!filePath || !runId || !rid) return;
+
+    const tempFilePath = this.#getFilePathWithUploadsList(runId);
+
+    if (typeof filePath === 'object') {
+      filePath = filePath.path;
+    }
+
+    if (typeof filePath === 'string' && !path.isAbsolute(filePath)) {
+      filePath = path.join(process.cwd(), filePath);
+    }
+
+    const data = { rid, file: filePath, uploaded };
+    const jsonLine = `${JSON.stringify(data)}\n`;
+    fs.appendFileSync(tempFilePath, jsonLine);
+  }
+
+  /**
+   * @param {*} filePath
+   * @param {*} pathInS3 contains runId, rid and filename
+   * @returns
+   */
+  async uploadFileByPath(filePath, pathInS3) {
+    /* WDIO: some artifacts uploading started before createRun function completion
+    probably, the reason is that run is NOT created in adapter (but via cli) */
+    this.isEnabled = this.isEnabled ?? this.checkEnabled();
+
+    const [runId, rid] = pathInS3;
+
+    if (!filePath) return;
+
+    let fileSize = null;
+    let fileSizeInMb = null;
+
+    try {
+      // file may not exist
+      fileSize = fs.statSync(filePath).size || 0;
+      fileSizeInMb = Number((fileSize / (1024 * 1024)).toFixed(2));
+    } catch (e) {
+      debug(`File ${filePath} does not exist`);
+    }
+
+    if (!this.isEnabled) {
+      this.storeUploadedFile(filePath, runId, rid, false);
+      this.skippedUploads.push({ path: filePath, size: fileSize });
+      return;
+    }
+
+    const { S3_BUCKET, TESTOMATIO_ARTIFACT_MAX_SIZE_MB } = this.getConfig();
+
+    debug('Started upload', filePath, 'to', S3_BUCKET);
+
+    const isFileExist = await this.checkArtifactExistsInFileSystem(filePath, 20, 500);
+
+    if (!isFileExist) {
+      console.error(pc.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
+      return;
+    }
+
+    // skipping artifact only if: 1. storing to file is enabled, 2. max size is set and 3. file size exceeds the limit
+    if (
+      this.storeEnabled &&
+      TESTOMATIO_ARTIFACT_MAX_SIZE_MB &&
+      fileSizeInMb > parseFloat(TESTOMATIO_ARTIFACT_MAX_SIZE_MB)
+    ) {
+      const skippedArtifact = { path: filePath, size: fileSize };
+      this.storeUploadedFile(filePath, runId, rid, false);
+      this.skippedUploads.push(skippedArtifact);
+      debug(pc.yellow(`Artifacts file ${JSON.stringify(skippedArtifact)} exceeds the maximum allowed size. Skipping.`));
+      return;
+    }
+    debug('File:', filePath, 'exists, size:', prettyBytes(fileSize));
+
+    const fileStream = fs.createReadStream(filePath);
+    const Key = pathInS3.filter(p => !!p).join('/');
+
+    const link = await this.#uploadToS3(fileStream, Key, { path: filePath, size: fileSize });
+
+    this.storeUploadedFile(filePath, runId, rid, !!link);
+
+    return link;
+  }
+
+  /**
+   * @param {Buffer} buffer
+   * @param {string[]} pathInS3
+   * @returns
+   */
+  async uploadFileAsBuffer(buffer, pathInS3) {
+    /* WDIO: some artifacts uploading started before createRun function completion
+    probably, the reason is that run is NOT created in adapter (but via cli) */
+
+    this.isEnabled = this.isEnabled ?? this.checkEnabled();
+    if (!this.isEnabled) return;
+
+    let Key = pathInS3.filter(p => !!p).join('/');
+    const ext = this.#getFileExtBase64(buffer.toString('base64'));
+
+    if (ext) {
+      Key = `${Key}.${ext}`;
+    }
+
+    return this.#uploadToS3(buffer, Key, { path: Key });
+  }
+
+  async checkArtifactExistsInFileSystem(filePath, attempts = 5, intervalMs = 500) {
+    return promiseRetry(
+      async (retry, number) => {
+        try {
+          fs.accessSync(filePath);
+          return true;
+        } catch (err) {
+          if (number === attempts) {
+            return false;
+          }
+          debug(`File not found, retrying (attempt ${number}/${attempts})`);
+          await new Promise(resolve => {
+            setTimeout(resolve, intervalMs);
+          });
+          retry(err);
+        }
+      },
+      {
+        retries: attempts,
+        minTimeout: intervalMs,
+        maxTimeout: intervalMs,
+      },
+    );
+  }
+
+  async getS3LocationLink(out) {
+    const response = await out.done();
+
+    let s3Location = response?.Location?.trim();
+
+    if (!s3Location) {
+      s3Location = out?.singleUploadResult?.Location;
+      debug('Uploaded singleUploadResult.Location', s3Location);
+
+      if (!s3Location) {
+        throw new Error("Problems getting the S3 artifact's link. Please check S3 permissions!");
+      }
+    }
+
+    // Normalize the URL
+    if (!s3Location.startsWith('http')) {
+      s3Location = `https://${s3Location}`;
+    }
+
+    return s3Location;
+  }
+
+  #getFileExtBase64(str) {
+    const type = str.charAt(0);
+
+    return (
+      {
+        '/': 'jpg',
+        i: 'png',
+        R: 'gif',
+        U: 'webp',
+      }[type] || ''
+    );
+  }
+
+  #getS3Config() {
+    const { S3_REGION, S3_SESSION_TOKEN, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_ENDPOINT } =
+      this.getConfig();
+
+    const cfg = {
+      region: S3_REGION,
+      credentials: {
+        accessKeyId: S3_ACCESS_KEY_ID,
+        secretAccessKey: S3_SECRET_ACCESS_KEY,
+      },
+    };
+
+    if (S3_FORCE_PATH_STYLE) {
+      cfg.forcePathStyle = !['false', '0'].includes(String(S3_FORCE_PATH_STYLE || '').toLowerCase());
+    }
+
+    if (S3_SESSION_TOKEN) {
+      cfg.credentials.sessionToken = S3_SESSION_TOKEN;
+    }
+
+    if (S3_ENDPOINT) {
+      cfg.endpoint = S3_ENDPOINT;
+    }
+
+    return cfg;
+  }
+}

package/src/utils/pipe_utils.js

@@ -0,0 +1,119 @@
+import { APP_PREFIX } from '../constants.js';
+
+/**
+ * Set S3 credentials from the provided artifacts object.
+ * @param {Object} artifacts - The artifacts object containing S3 credentials.
+ */
+function setS3Credentials(artifacts) {
+  if (!Object.keys(artifacts).length) return;
+
+  console.log(APP_PREFIX, 'S3 credentials obtained from Testomat.io...');
+
+  if (artifacts.ACCESS_KEY_ID) process.env.S3_ACCESS_KEY_ID = artifacts.ACCESS_KEY_ID;
+  if (artifacts.SECRET_ACCESS_KEY) process.env.S3_SECRET_ACCESS_KEY = artifacts.SECRET_ACCESS_KEY;
+  if (artifacts.REGION) process.env.S3_REGION = artifacts.REGION;
+  if (artifacts.BUCKET) process.env.S3_BUCKET = artifacts.BUCKET;
+  if (artifacts.SESSION_TOKEN) process.env.S3_SESSION_TOKEN = artifacts.SESSION_TOKEN;
+  if (artifacts.presign) process.env.TESTOMATIO_PRIVATE_ARTIFACTS = '1';
+  // endpoint is not received from the server; and shuld be empty if IAM used (credentails obtained from the testomat)
+  process.env.S3_ENDPOINT = artifacts.ENDPOINT || '';
+}
+
+/**
+ * Generates mode request parameters based on the input params.
+ * @param {{type: string, id?: string, apiKey: string}} params - The input parameters for the request.
+ * @returns {Object|null} - An object containing the generated request parameters, or null if the type is invalid.
+ */
+function generateFilterRequestParams(params) {
+  const { type, id, apiKey } = params;
+
+  if (!type) {
+    return;
+  }
+
+  if (!id) {
+    console.error(APP_PREFIX, `Please make sure your settings "${type.toUpperCase()}"= "${id}" is correct!`);
+    return;
+  }
+
+  return {
+    params: {
+      type,
+      id: encodeURIComponent(id),
+      api_key: apiKey,
+    },
+    responseType: 'json',
+  };
+}
+
+/**
+ * Parse filter parameters from a string in the format "type=id".
+ * @param {string} opts - The input string containing the filter parameters.
+ * @returns {Object} An object containing the parsed filter parameters.
+ *                   The object has properties "type" and "id".
+ */
+function parseFilterParams(opts) {
+  const [type, id] = opts.split('=');
+  const validType = updateFilterType(type);
+
+  return {
+    type: validType,
+    id,
+  };
+}
+
+/**
+ * Update and validate the filter type.
+ * @param {string} type - The original filter type.
+ * @returns {string|undefined} The updated and validated filter type.
+ *                            Returns undefined if the type is not valid.
+ */
+function updateFilterType(type) {
+  const typeLowerCase = type.toLowerCase();
+
+  const filterTypes = ['tag-name', 'plan-id', 'label', 'jira-ticket'];
+
+  const filterApi = [
+    'tag',
+    'plan',
+    'label',
+    'jira',
+    // "ims-issue", //TODO: WIP
+  ];
+
+  if (!filterTypes.includes(typeLowerCase)) {
+    console.log(APP_PREFIX, `❗❗❗ Invalid "filter=${type}" start settings! Available option list: ${filterTypes}`);
+    return;
+  }
+
+  const index = filterTypes.indexOf(typeLowerCase);
+
+  return index !== -1 ? filterApi[index] : undefined;
+}
+
+/**
+ * Return an emoji based on the provided status.
+ * @param {string} status - The status value ('passed', 'failed', or 'skipped').
+ * @returns {string} - An emoji corresponding to the provided status.
+ */
+function statusEmoji(status) {
+  if (status === 'passed') return '🟢';
+  if (status === 'failed') return '🔴';
+  if (status === 'skipped') return '🟡';
+  return '';
+}
+
+/**
+ * Generate a full name string based on the provided test object.
+ * @param {object} t - The test object.
+ * @returns {string} - A formatted full name string for the test object.
+ */
+function fullName(t) {
+  let line = '';
+  if (t.suite_title) line = `${t.suite_title}: `;
+  line += `**${t.title}**`;
+  if (t.example) line += ` \`[${Object.values(t.example)}]\``;
+  return line;
+}
+
+export { updateFilterType, parseFilterParams, generateFilterRequestParams, setS3Credentials, statusEmoji, fullName };