@api-client/core 0.3.5 → 0.3.6

package/src/runtime/node/ProjectParallelRunner.ts

@@ -0,0 +1,234 @@
+import cluster, { Worker } from 'cluster';
+import { cpus } from 'os';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import { HttpProject, IHttpProject } from '../../models/HttpProject.js';
+import { IProjectRunnerOptions } from './ProjectRunner.js';
+import { IProjectExecutionLog, IProjectExecutionIteration } from '../reporters/Reporter.js';
+import { BaseRunner } from './BaseRunner.js';
+
+const numCPUs = cpus().length;
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export type WorkerStatus = 'initializing' | 'ready' | 'running' | 'finished' | 'error';
+
+export interface IWorkerInfo {
+  /**
+   * Whether the worker is online.
+   */
+  online: boolean;
+  /**
+   * The number of iterations the worker is performing.
+   */
+  iterations: number;
+  /**
+   * The current status of the worker.
+   */
+  status: WorkerStatus;
+  /**
+   * Optional error message received from the worker.
+   */
+  message?: string;
+}
+
+interface WorkerInfoInternal extends IWorkerInfo {
+  worker: Worker;
+}
+
+export interface IWorkerMessage {
+  cmd: string;
+  data?: unknown;
+}
+
+export interface IProjectParallelRunnerOptions extends IProjectRunnerOptions {
+}
+
+export interface IProjectParallelWorkerOptions extends IProjectRunnerOptions {
+  project: IHttpProject;
+}
+
+export interface ProjectParallelRunner {
+  /**
+   * Dispatched when a status of a worker change.
+   * This can be used to render the current status.
+   */
+  on(event: 'status', listener: (info: IWorkerInfo[]) => void): this;
+  /**
+   * Dispatched when a status of a worker change.
+   * This can be used to render the current status.
+   */
+  once(event: 'status', listener: (info: IWorkerInfo[]) => void): this;
+}
+
+/**
+ * Runs a project in parallel.
+ * It creates a number of workers determined by the number of CPUs available on the current machine
+ * and the number of iterations defined in the configuration options.
+ * 
+ * When the number of iterations is greater then the number of CPUs then 
+ * the program distributes the remaining iterations among created workers.
+ * 
+ * The program dispatched the `status` event. It is dispatched each time when the worker status
+ * change. This event can be user to refresh the UI to reflect the newest state.
+ */
+export class ProjectParallelRunner extends BaseRunner {
+  project: HttpProject;
+  options: IProjectParallelRunnerOptions;
+  workers: WorkerInfoInternal[] = [];
+  private mainResolver?: (report: IProjectExecutionLog) => void;
+  private mainRejecter?: (err: Error) => void;
+
+  constructor(project: HttpProject, opts: IProjectParallelRunnerOptions = {}) {
+    super();
+    this.project = project;
+    this.options = opts || {};
+
+    this._exitHandler = this._exitHandler.bind(this);
+  }
+
+  execute(): Promise<IProjectExecutionLog> {
+    return new Promise((resolve, reject) => {
+      this.mainResolver = resolve;
+      this.mainRejecter = reject;
+      this._execute();
+    });
+  }
+
+  private _execute(): void {
+    try {
+      cluster.setupPrimary({
+        exec: join(__dirname, 'ProjectRunnerWorker.js'),
+        silent: true,
+      });
+      const { iterations = 1 } = this.options;
+      const poolSize = Math.min(iterations, numCPUs);
+      for (let i = 0; i < poolSize; i++) {
+        const worker = cluster.fork();
+        this.setupWorker(worker);
+      }
+      this.distributeIterations();
+      this.emit('status', this.getStatusWorkers());
+      cluster.on('exit', this._exitHandler);
+    } catch (e) {
+      const cause = e as Error;
+      this.mainRejecter!(cause);
+    }
+  }
+
+  private getStatusWorkers(): IWorkerInfo[] {
+    const { workers } = this;
+    const result: IWorkerInfo[] = [];
+    workers.forEach((info) => {
+      const cp = { ...info } as any;
+      delete cp.worker;
+      result.push(cp);
+    });
+    return result;
+  }
+
+  private distributeIterations(): void {
+    const workers = this.workers.length;
+    const { iterations = 1 } = this.options;
+    let iterationsRemaining = iterations - workers;
+    let currentIndex = 0;
+    while (iterationsRemaining > 0) {
+      this.workers[currentIndex].iterations += 1;
+      iterationsRemaining--;
+      currentIndex++;
+      if (currentIndex + 1 === workers) {
+        currentIndex = 0;
+      }
+    }
+  }
+
+  private setupWorker(worker: Worker): void {
+    this.workers.push({
+      worker,
+      online: false,
+      iterations: 1,
+      status: 'initializing',
+    });
+    worker.on('message', this._messageHandler.bind(this, worker));
+  }
+
+  private _messageHandler(worker: Worker, message: IWorkerMessage): void {
+    switch (message.cmd) {
+      case 'online': this.setOnline(worker); break;
+      case 'result': this.setRunResult(worker, message); break;
+      case 'error': this.setRunError(worker, message); break;
+    }
+  }
+
+  private _exitHandler(worker: Worker): void {
+    const info = this.workers.find(i => i.worker === worker);
+    if (!info) {
+      return;
+    }
+    this.finishWhenReady();
+  }
+
+  private setOnline(worker: Worker): void {
+    const info = this.workers.find(i => i.worker === worker);
+    if (!info) {
+      return;
+    }
+    info.online = true;
+    info.status = 'ready';
+    this.runWhenReady();
+    this.emit('status', this.getStatusWorkers());
+  }
+
+  private setRunResult(worker: Worker, message: IWorkerMessage): void {
+    const reports = message.data as IProjectExecutionIteration[];
+    this.executed = this.executed.concat(reports);
+    worker.destroy();
+    const info = this.workers.find(i => i.worker === worker);
+    if (!info) {
+      return;
+    }
+    info.status = 'finished';
+    this.emit('status', this.getStatusWorkers());
+  }
+
+  private runWhenReady(): void {
+    const waiting = this.workers.some(i => !i.online);
+    if (waiting) {
+      return;
+    }
+    this.startTime = Date.now();
+    this.workers.forEach((info) => {
+      const opts: IProjectParallelWorkerOptions = { ...this.options, project: this.project.toJSON() };
+      opts.iterations = info.iterations;
+      info.status = 'running';
+      info.worker.send({
+        cmd: 'run',
+        data: opts,
+      });
+    });
+  }
+
+  private async finishWhenReady(): Promise<void> {
+    if (this.endTime) {
+      return;
+    }
+    const working = this.workers.some(i => !['finished', 'error'].includes(i.status));
+    if (working || !this.mainResolver) {
+      return;
+    }
+    this.endTime = Date.now();
+    const report = await this.createReport();
+    this.mainResolver(report);
+    cluster.off('exit', this._exitHandler);
+  }
+
+  private setRunError(worker: Worker, message: IWorkerMessage): void {
+    worker.destroy();
+    const info = this.workers.find(i => i.worker === worker);
+    if (!info) {
+      return;
+    }
+    info.status = 'error';
+    info.message = message.data as string;
+    this.emit('status', this.getStatusWorkers());
+  }
+}

package/src/runtime/node/ProjectRequestRunner.ts

@@ -0,0 +1,281 @@
+import { EventEmitter } from 'events';
+import { Environment } from '../../models/Environment.js';
+import { Logger } from '../../lib/logging/Logger.js';
+import { IRequestLog, RequestLog } from '../../models/RequestLog.js';
+import { Property } from '../../models/Property.js';
+import { ProjectFolder, Kind as ProjectFolderKind } from '../../models/ProjectFolder.js';
+import { ProjectRequest } from '../../models/ProjectRequest.js';
+import { IHttpRequest } from '../../models/HttpRequest.js';
+import { HttpProject, IProjectRequestIterator } from '../../models/HttpProject.js';
+import { SentRequest } from '../../models/SentRequest.js';
+import { ErrorResponse } from '../../models/ErrorResponse.js';
+import { VariablesStore } from './VariablesStore.js';
+import { VariablesProcessor } from '../variables/VariablesProcessor.js';
+import { RequestFactory } from './RequestFactory.js';
+import { EventTypes } from '../../events/EventTypes.js';
+
+export interface ProjectRunnerOptions {
+  /**
+   * When provided it overrides any project / folder defined environment.
+   */
+  environment?: Environment;
+  /**
+   * Additional variables to pass to the selected environment.
+   * This can be use to pass system variables, when needed.
+   * 
+   * To use system variables tou can use `init.variables = process.env`;
+   */
+  variables?: Record<string, string>;
+  /**
+   * Overrides the default logger (console).
+   */
+  logger?: Logger;
+  /**
+   * The event target to use.
+   * By default it creates its own target.
+   */
+  eventTarget?: EventTarget;
+}
+
+export interface ProjectRunnerRunOptions extends IProjectRequestIterator {
+}
+
+export interface RunResult {
+  /**
+   * The key of the request from the HttpProject that was executed.
+   */
+  key: string;
+  /**
+   * The key of parent folder of the executed request.
+   */
+  parent?: string;
+  /**
+   * Set when a fatal error occurred so the request couldn't be executed.
+   * This is not the same as error reported during a request. The log's response can still be IResponseError.
+   */
+  error?: boolean;
+  /**
+   * The error message. Always set when the `error` is `true`.
+   */
+  errorMessage?: string;
+  /**
+   * The request log.
+   * Always set when the `error` is `false`.
+   */
+  log?: IRequestLog;
+}
+
+export interface ProjectRequestRunner {
+  /**
+   * The request object is prepared and about to be sent to the HTTP engine
+   */
+  on(event: 'request', listener: (key: string, request: IHttpRequest) => void): this;
+  /**
+   * The response is ready.
+   */
+  on(event: 'response', listener: (key: string, log: IRequestLog) => void): this;
+  /**
+   * There was a general error during the request
+   */
+  on(event: 'error', listener: (key: string, log: IRequestLog, message: string) => void): this;
+  /**
+   * The request object is prepared and about to be sent to the HTTP engine
+   */
+  once(event: 'request', listener: (key: string, request: IHttpRequest) => void): this;
+  /**
+   * The response is ready.
+   */
+  once(event: 'response', listener: (key: string, log: IRequestLog) => void): this;
+  /**
+   * There was a general error during the request
+   */
+  once(event: 'error', listener: (key: string, log: IRequestLog, message: string) => void): this;
+}
+
+/**
+ * Runs requests in a project.
+ * Developers can run the entire project with the `recursive` flag set. They can also 
+ * set the starting point with the `parent` options.
+ * 
+ * Requests are executed in order defined in the folder.
+ */
+export class ProjectRequestRunner extends EventEmitter {
+  eventTarget: EventTarget;
+  logger?: Logger;
+  project: HttpProject;
+
+  protected masterEnvironment?: Environment;
+  protected extraVariables?: Record<string, string>;
+
+  /**
+   * The variables processor instance.
+   */
+  protected variablesProcessor = new VariablesProcessor();
+
+  constructor(project: HttpProject, opts: ProjectRunnerOptions = {}) {
+    super();
+    this.project = project;
+    this.logger = opts.logger;
+    this.eventTarget = opts.eventTarget || new EventTarget();
+    this.masterEnvironment = opts.environment;
+    this.extraVariables = opts.variables;
+  }
+
+  /**
+   * Runs the request from the project root or a specified folder.
+   * @param options Run options.
+   * @returns A promise with the run result.
+   */
+  async run(options?: ProjectRunnerRunOptions): Promise<RunResult[]> {
+    const { project } = this;
+    const executed: RunResult[] = [];
+    for (const request of project.requestIterator(options)) {
+      const parent = request.getParent() || project;
+      let variables: Record<string, string>;
+      if (VariablesStore.has(parent)) {
+        variables = VariablesStore.get(parent);
+      } else {
+        variables = await this.getVariables(parent);
+        VariablesStore.set(parent, variables);
+      }
+      const info = await this.execute(request, variables);
+      executed.push(info);
+    }
+    return executed;
+  }
+
+  protected async execute(request: ProjectRequest, variables: Record<string, string>): Promise<RunResult> {
+    const config = request.getConfig();
+    const factory = new RequestFactory(this.eventTarget);
+
+    factory.variables = variables;
+    if (request.authorization) {
+      factory.authorization = request.authorization.map(i => i.toJSON());
+    }
+    if (request.actions) {
+      factory.actions = request.actions.toJSON();
+    }
+    if (request.clientCertificate) {
+      factory.certificates = [request.clientCertificate];
+    }
+    if (config.enabled !== false) {
+      factory.config = config.toJSON();
+    }
+    if (this.logger) {
+      factory.logger = this.logger;
+    }
+    const info: RunResult = {
+      key: request.key,
+    };
+    const requestData = request.expects.toJSON();
+    requestData.url = this.prepareRequestUrl(requestData.url, variables);
+
+    function variableHandler(e: CustomEvent): void {
+      if (e.defaultPrevented) {
+        return;
+      }
+      const { name, value } = e.detail;
+      variables[name] = value;
+      e.preventDefault();
+      e.detail.result = Promise.resolve();
+    }
+
+    this.eventTarget.addEventListener(EventTypes.Environment.set, variableHandler as any);
+
+    try {
+      // Below replaces the single call to the `run()` function of the factory to 
+      // report via the events a request object that has evaluated with the Jexl library.
+      const requestCopy = await factory.processRequestVariables(requestData);
+      this.emit('request', request.key, { ...requestCopy });
+      await factory.processRequestLogic(requestCopy);
+      const result = await factory.executeRequest(requestCopy);
+      await factory.processResponse(result);
+      request.setLog(result);
+      info.log = result;
+      this.emit('response', request.key, { ...result });
+    } catch (e) {
+      const cause = e as Error;
+      info.error = true;
+      info.errorMessage = cause.message;
+      const sent = new SentRequest({ ...requestData, startTime: 0, endTime: 0, });
+      const response = ErrorResponse.fromError(info.errorMessage);
+      const log = RequestLog.fromRequestResponse(sent.toJSON(), response.toJSON()).toJSON();
+      this.emit('error', request.key, log, info.errorMessage);
+    }
+
+    this.eventTarget.removeEventListener(EventTypes.Environment.set, variableHandler as any);
+    return info;
+  }
+
+  protected async getVariables(parent: HttpProject | ProjectFolder): Promise<Record<string, string>> {
+    if (this.masterEnvironment) {
+      return this.applyVariables([this.masterEnvironment]);
+    }
+    return this.createEnvironment(parent);
+  }
+
+  protected async createEnvironment(parent: HttpProject | ProjectFolder): Promise<Record<string, string>> {
+    const envs = await this.readEnvironments(parent);
+    return this.applyVariables(envs);
+  }
+
+  /**
+   * Reads the list of the environments to apply to this runtime.
+   */
+  protected async readEnvironments(parent: HttpProject | ProjectFolder): Promise<Environment[]> {
+    const folderKey = parent.kind === ProjectFolderKind ? (parent as ProjectFolder).key : undefined;
+    return this.project.readEnvironments({ folderKey });
+  }
+
+  /**
+   * Reads the variables and the base URI from the passed environments.
+   */
+  protected async applyVariables(environments: Environment[]): Promise<Record<string, string>> {
+    let baseUri = '';
+    const variables: Property[] = [];
+    environments.forEach((environment) => {
+      const { server, variables: envVariables } = environment;
+      if (server) {
+        baseUri = server.readUri();
+      }
+      if (envVariables.length) {
+        envVariables.forEach((item) => {
+          const defined = variables.findIndex(i => i.name === item.name);
+          if (defined >= 0) {
+            variables[defined] = item;
+          } else {
+            variables.push(item);
+          }
+        });
+      }
+    });
+    const { extraVariables } = this;
+    const ctx = VariablesProcessor.createContextFromProperties(variables);
+    if (extraVariables) {
+      Object.keys(extraVariables).forEach((key) => {
+        ctx[key] = extraVariables[key];
+      });
+    }
+    // the `baseUri` is reserved and always set to the environment's `baseUri`.
+    ctx.baseUri = baseUri || '';
+    return this.variablesProcessor.buildContext(ctx);
+  }
+
+  /**
+   * When defined it applies the serve's base URI to relative URLs.
+   * @param currentUrl The URL to process.
+   */
+  protected prepareRequestUrl(currentUrl: string, variables: Record<string, string>): string {
+    const { baseUri } = variables;
+    if (!baseUri) {
+      return currentUrl;
+    }
+    if (currentUrl.startsWith('http:') || currentUrl.startsWith('https:')) {
+      return currentUrl;
+    }
+    if (currentUrl.startsWith('/')) {
+      return `${baseUri}${currentUrl}`;
+    }
+    return `${baseUri}/${currentUrl}`;
+  }
+}