@tskmgr/client 1.4.0 → 2.0.2

package/.babelrc

@@ -0,0 +1,10 @@
+{
+  "presets": [
+    [
+      "@nx/js/babel",
+      {
+        "useBuiltIns": "usage"
+      }
+    ]
+  ]
+}

package/.eslintrc.json

@@ -0,0 +1,21 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "parserOptions": {
+        "project": ["libs/client/tsconfig.*?.json"]
+      },
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

package/jest.config.ts

@@ -0,0 +1,17 @@
+/* eslint-disable */
+export default {
+  displayName: 'client',
+  preset: '../../jest.preset.js',
+  globals: {},
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]sx?$': [
+      'ts-jest',
+      {
+        tsconfig: '<rootDir>/tsconfig.spec.json',
+      },
+    ],
+  },
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+  coverageDirectory: '../../coverage/libs/client',
+};

package/package.json

@@ -1,15 +1,13 @@
 {
   "name": "@tskmgr/client",
-  "version": "1.4.0",
+  "version": "2.0.2",
   "license": "MIT",
   "dependencies": {
     "debug": "^4.3.4",
     "node-fetch": "^2.6.7",
-    "systeminformation": "^5.16.6",
-    "@tskmgr/common": "1.4.0",
-    "uuid": "^9.0.0"
-  },
-  "main": "./src/index.js",
-  "types": "./src/index.d.ts",
-  "peerDependencies": {}
+    "@tskmgr/common": "2.0.2",
+    "uuid": "^9.0.1",
+    "@nx/devkit": "18.0.7",
+    "form-data": "2.3.3"
+  }
 }

package/project.json

@@ -0,0 +1,31 @@
+{
+  "name": "client",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "libs/client/src",
+  "projectType": "library",
+  "targets": {
+    "lint": {
+      "executor": "@nx/eslint:lint",
+      "outputs": ["{options.outputFile}"]
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/libs/client"],
+      "options": {
+        "jestConfig": "libs/client/jest.config.ts"
+      }
+    },
+    "build": {
+      "executor": "@nx/js:tsc",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "dist/libs/client",
+        "tsConfig": "libs/client/tsconfig.lib.json",
+        "packageJson": "libs/client/package.json",
+        "main": "libs/client/src/index.ts",
+        "assets": ["libs/client/*.md", "LICENSE"]
+      }
+    }
+  },
+  "tags": []
+}

package/src/{index.d.ts → index.ts}

@@ -1,5 +1,6 @@
 export * from './lib/client';
 export * from './lib/client-options';
 export * from './lib/client-factory';
+
 export * from './lib/run-tasks-result';
 export * from './lib/task-result';

package/src/lib/client-example.ts

@@ -0,0 +1,134 @@
+/**
+ * IntelliJ debug:
+ *  Node parameters: --require ts-node/register --require tsconfig-paths/register
+ *  Environment variables: TS_NODE_PROJECT=libs/client/tsconfig.lib.json
+ *
+ *  Start API:
+ *    nx serve api
+ *  Command:
+ *    DEBUG=tskmgr:* ts-node --project libs/client/tsconfig.lib.json -r tsconfig-paths/register "libs/client/src/lib/client-example.ts"
+ */
+
+import { execSync } from 'child_process';
+import { CreateTaskDto, Run, Task, TaskPriority } from '@tskmgr/common';
+import { ClientOptions } from './client-options';
+import { ClientFactory } from './client-factory';
+import { v4 as uuid } from 'uuid';
+import Debug from 'debug';
+import { readJsonFile } from '@nx/devkit';
+import { unlinkSync } from 'fs';
+const debug = Debug('tskmgr:client-example');
+
+delete process.env.TS_NODE_PROJECT;
+
+const options: ClientOptions = {
+  parallel: 1,
+  dataCallback,
+  errorCallback,
+  spawnOptions: { env: { ...process.env } },
+};
+const client = ClientFactory.createNew('http://localhost:3333', 'RUNNER_1', options);
+
+let completed = false;
+let run: Run;
+
+(async () => {
+  try {
+    // 1. Create the new run
+    run = await client.createRun({
+      name: uuid(),
+      type: '123',
+      prioritization: [TaskPriority.Longest],
+    });
+    debug(run);
+
+    // 2. Leader should create some tasks to run
+    const election = await client.setLeader(run.id);
+    if (election.leader) {
+      const tasks = getNxTasks().map<CreateTaskDto>((nxTask) => {
+        const command = `npx nx run ${nxTask.target.project}:${nxTask.target.target} --configuration=production`;
+        return {
+          name: nxTask.target.project,
+          type: nxTask.target.target,
+          command: command.trim(),
+          options: { shell: true },
+          priority: TaskPriority.Longest,
+        };
+      });
+
+      const createdTasks = await client.createTasks(run.id, { tasks });
+      debug(createdTasks);
+
+      const closeRun = await client.closeRun(run.id);
+      debug(closeRun);
+    }
+
+    // 3. Execute tasks
+    const result = await client.runTasks(run.id);
+    if (result.completed) {
+      // if failFast set to false, runTasks will continue without throwing errors.
+      completed = true;
+    }
+  } catch (e) {
+    console.error(e);
+  } finally {
+    // 4. See results
+    console.log('--------------------------------------------------');
+    console.log(`  tskmgr run: http://localhost:4200/runs/${run.id}`);
+    console.log('--------------------------------------------------');
+    console.log(`${completed ? 'COMPLETED!' : 'FAILED!'}`);
+    process.exit(completed ? 0 : 1);
+  }
+})();
+
+function getNxTasks(): NxTask[] {
+  const graphFileName = uuid() + '.json';
+  // In CI environment use npx nx affected --graph=${graphFileName} --target=lint command. run-many is just for demo purpose.
+  execSync(`npx nx run-many --graph=lint-${graphFileName} --target=lint`);
+  const lintJson = readJsonFile(`lint-${graphFileName}`);
+  unlinkSync(`lint-${graphFileName}`);
+  const lintTasks: NxTask[] = Object.values(lintJson.tasks.tasks);
+
+  execSync(`npx nx run-many --graph=test-${graphFileName} --target=test`);
+  const testJson = readJsonFile(`test-${graphFileName}`);
+  unlinkSync(`test-${graphFileName}`);
+  const testTasks: NxTask[] = Object.values(testJson.tasks.tasks);
+
+  execSync(`npx nx run-many --graph=build-${graphFileName} --target=build`);
+  const buildJson = readJsonFile(`build-${graphFileName}`);
+  unlinkSync(`build-${graphFileName}`);
+  const buildTasks: NxTask[] = Object.values(buildJson.tasks.tasks);
+
+  execSync(`npx nx run-many --graph=e2e-${graphFileName} --target=e2e`);
+  const e2eJson = readJsonFile(`e2e-${graphFileName}`);
+  unlinkSync(`e2e-${graphFileName}`);
+  const e2eTasks: NxTask[] = Object.values(e2eJson.tasks.tasks);
+
+  return [...lintTasks, ...testTasks, ...buildTasks, ...e2eTasks];
+}
+
+function dataCallback(task: Task, data: string, cached: () => void): void {
+  // > nx run frontend:lint  [existing outputs match the cache, left as is]
+  // > nx run client:lint  [local cache]
+  if (
+    (data.startsWith(`> nx run ${task.name}:${task.type}`) &&
+      data.endsWith('[existing outputs match the cache, left as is]')) ||
+    data.endsWith('[local cache]')
+  ) {
+    cached();
+  }
+
+  console.log(`[stdout] ${data}`);
+}
+
+function errorCallback(task: Task, data: string): void {
+  console.log(`[stderr] ${data}`);
+}
+
+interface NxTask {
+  id: string;
+  overrides: any;
+  target: { project: string; target: string };
+  command: string;
+  outputs: string[];
+}

package/src/lib/client-factory.ts

@@ -0,0 +1,17 @@
+import { Client } from './client';
+import { ClientOptions } from './client-options';
+import { ApiUrl } from '@tskmgr/common';
+
+export class ClientFactory {
+  public static createNew(
+    baseUrl: string, //
+    runnerId: string,
+    options?: ClientOptions
+  ): Client {
+    return new Client(
+      ApiUrl.create(baseUrl), //
+      runnerId,
+      { ...Client.DefaultOptions, ...options }
+    );
+  }
+}

package/src/lib/client-options.ts

@@ -0,0 +1,12 @@
+import { Task } from '@tskmgr/common';
+import { SpawnOptionsWithoutStdio } from 'child_process';
+
+export interface ClientOptions {
+  parallel?: number;
+  dataCallback?: (task: Task, data: string, cached: () => void) => void;
+  errorCallback?: (task: Task, data: string) => void;
+  pollingDelayMs?: number;
+  retryDelayMs?: number;
+  retryCount?: number;
+  spawnOptions?: SpawnOptionsWithoutStdio;
+}

package/src/lib/client.spec.ts

@@ -0,0 +1,5 @@
+describe('client', () => {
+  it('should work', () => {
+    //
+  });
+});

package/src/lib/client.ts

@@ -0,0 +1,280 @@
+import { ChildProcess } from 'child_process';
+import {
+  ApiUrl,
+  Run,
+  CompleteTaskDto,
+  CreateRunRequestDto,
+  CreateTasksDto,
+  StartTaskDto,
+  StartTaskResponseDto,
+  Task,
+  File as File_,
+  SetLeaderRequestDto,
+  SetLeaderResponseDto,
+  CreateFileRequestDto,
+} from '@tskmgr/common';
+import fetch from 'node-fetch';
+import * as FormData from 'form-data';
+import { checkStatus, delay, getTaskLogFilename, spawnAsync } from './utils';
+import { RunTasksResult } from './run-tasks-result';
+import { TaskResult } from './task-result';
+import { ClientOptions } from './client-options';
+import { createReadStream, createWriteStream, unlinkSync } from 'fs';
+import Debug from 'debug';
+
+const debug = Debug('tskmgr:client');
+
+export class Client {
+  public static readonly DefaultOptions: ClientOptions = {
+    parallel: 1,
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    dataCallback: (task, data, cached) => {},
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    errorCallback: (task, data) => {},
+    pollingDelayMs: 5000,
+    retryDelayMs: 5000,
+    retryCount: 2,
+  };
+
+  constructor(
+    private readonly apiUrl: ApiUrl,
+    private readonly runnerId: string,
+    private readonly options: ClientOptions
+  ) {}
+
+  public async createRun(params: CreateRunRequestDto): Promise<Run> {
+    const res = await fetch(this.apiUrl.createRunUrl(), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'POST',
+      body: JSON.stringify(params),
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async closeRun(runId: number): Promise<Run> {
+    const res = await fetch(this.apiUrl.closeRunUrl(runId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'PUT',
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async abortRun(runId: number): Promise<Run> {
+    const res = await fetch(this.apiUrl.abortRunUrl(runId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'PUT',
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async failRun(runId: number): Promise<Run> {
+    const res = await fetch(this.apiUrl.failRunUrl(runId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'PUT',
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async setLeader(runId: number): Promise<SetLeaderResponseDto> {
+    const params: SetLeaderRequestDto = { runnerId: this.runnerId };
+    const res = await fetch(this.apiUrl.setLeaderUrl(runId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'PUT',
+      body: JSON.stringify(params),
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async createTasks(runId: number, params: CreateTasksDto): Promise<Task[]> {
+    const res = await fetch(this.apiUrl.createTasksUrl(runId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'POST',
+      body: JSON.stringify(params),
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async startTask(runId: number, params: StartTaskDto): Promise<StartTaskResponseDto> {
+    const res = await fetch(this.apiUrl.startTaskUrl(runId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'PUT',
+      body: JSON.stringify(params),
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async runTasks(runId: number): Promise<RunTasksResult> {
+    const taskRunners: Promise<TaskResult[]>[] = [];
+
+    for (let i = 0; i < this.options.parallel; i++) {
+      taskRunners.push(this.defaultParallelTaskRunner(runId, i));
+    }
+
+    const taskResults = await Promise.all(taskRunners);
+    return new RunTasksResult(taskResults.flatMap((x) => x));
+  }
+
+  public async completeTask(taskId: number, params: CompleteTaskDto): Promise<Task> {
+    const res = await fetch(this.apiUrl.completeTaskUrl(taskId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'PUT',
+      body: JSON.stringify(params),
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async failTask(taskId: number): Promise<Task> {
+    const res = await fetch(this.apiUrl.failTaskUrl(taskId), {
+      headers: { 'Content-Type': 'application/json' },
+      method: 'PUT',
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  public async uploadRunFile(runId: number, path: string, params: CreateFileRequestDto): Promise<File_> {
+    return this.uploadFile(this.apiUrl.createFileRunUrl(runId), path, params);
+  }
+
+  public async uploadTaskFile(taskId: number, path: string, params: CreateFileRequestDto): Promise<File_> {
+    return this.uploadFile(this.apiUrl.createFileTaskUrl(taskId), path, params);
+  }
+
+  private async uploadFile(url: string, path: string, params: CreateFileRequestDto): Promise<File_> {
+    // https://github.com/node-fetch/node-fetch/tree/2.x#post-with-form-data-detect-multipart
+    // https://github.com/form-data/form-data#readme
+
+    const formData = new FormData();
+    formData.append('file', createReadStream(path));
+
+    if (params.type) {
+      formData.append('type', params.type);
+    }
+
+    if (params.description) {
+      formData.append('description', params.description);
+    }
+
+    const res = await fetch(url, {
+      method: 'POST',
+      body: formData,
+    });
+
+    return await checkStatus(res).json();
+  }
+
+  private async defaultParallelTaskRunner(runId: number, parallelId: number): Promise<TaskResult[]> {
+    const logInfo = `[${this.runnerId}:${parallelId}]`;
+    const taskResults: TaskResult[] = [];
+    let _continue = true;
+
+    debug(`${logInfo} parallel task runner started`);
+
+    while (_continue) {
+      const res = await this.startTask(runId, { runnerId: this.runnerId });
+
+      if (!res.continue) {
+        debug(`${logInfo} continue set to false`);
+        _continue = false;
+      }
+
+      if (!res.continue && !res.task) {
+        debug(`${logInfo} continue set to false and no task, exiting`);
+        continue;
+      }
+
+      if (!res.task) {
+        debug(`${logInfo} polling for new tasks in ${this.options.pollingDelayMs} ms`);
+        await delay(this.options.pollingDelayMs);
+        continue;
+      }
+
+      const { run, task } = res;
+      debug(`${logInfo} received task: ${task.id}`);
+
+      let cached = false;
+      let completed = false;
+      let writable = false;
+      let childProcess: ChildProcess;
+      let error: Error;
+
+      const taskLogFilename = getTaskLogFilename(task.id);
+      const writeStream = createWriteStream(taskLogFilename);
+      writeStream.on('open', () => (writable = true));
+
+      const dataHandler = (data: string): void => {
+        this.options.dataCallback(task, data, () => (cached = true));
+        if (writable) {
+          writeStream.write(`[${new Date().toISOString()}] ${data}\n`);
+        }
+      };
+
+      const errorHandler = (data: string): void => {
+        this.options.errorCallback(task, data);
+        if (writable) {
+          writeStream.write(`[${new Date().toISOString()}] ${data}\n`);
+        }
+      };
+
+      debug(`${logInfo} starting task: ${task.id}`);
+      try {
+        childProcess = await spawnAsync(
+          task.command,
+          task.arguments,
+          {
+            ...task.options,
+            ...this.options.spawnOptions,
+          },
+          {
+            dataCallback: dataHandler,
+            errorCallback: errorHandler,
+          }
+        );
+        completed = true;
+        debug(`${logInfo} completed task: ${task.id}`);
+      } catch (err) {
+        console.error(`${logInfo} failed task: ${task.id} with error: ${err}`);
+        error = err;
+
+        if (run.failFast) {
+          // TODO: should abort all running tasks by current client
+          debug(`${logInfo} fail fast enabled, aborting`);
+          throw err;
+        }
+      } finally {
+        taskResults.push({ run, task, childProcess, completed, error });
+
+        if (completed) {
+          await this.completeTask(task.id, { cached });
+          debug(`${logInfo} completed status for task: ${task.id} sent`);
+        } else {
+          await this.failTask(task.id);
+          debug(`${logInfo} failed status for task: ${task.id} sent`);
+        }
+
+        writeStream.end();
+        writeStream.close();
+
+        await this.uploadTaskFile(task.id, taskLogFilename, {
+          type: 'log',
+          description: `Log for task ${task.id}`,
+        });
+
+        debug(`${logInfo} uploaded file for task: ${task.id} sent`);
+        unlinkSync(taskLogFilename);
+      }
+    }
+
+    debug(`${logInfo} parallel task runner completed`);
+
+    return taskResults;
+  }
+}

package/src/lib/run-tasks-result.ts

@@ -0,0 +1,17 @@
+import { TaskResult } from './task-result';
+
+export class RunTasksResult {
+  public readonly completedTasks: TaskResult[];
+  public readonly failedTasks: TaskResult[];
+
+  public readonly completed: boolean;
+  public readonly failed: boolean;
+
+  constructor(public readonly tasks: TaskResult[]) {
+    this.completedTasks = tasks.filter((x) => x.completed);
+    this.failedTasks = tasks.filter((x) => !x.completed);
+
+    this.completed = this.failedTasks.length === 0;
+    this.failed = this.failedTasks.length > 0;
+  }
+}

package/src/lib/task-result.ts

@@ -0,0 +1,10 @@
+import { Run, Task } from '@tskmgr/common';
+import { ChildProcess } from 'child_process';
+
+export interface TaskResult {
+  run: Run;
+  task: Task;
+  childProcess: ChildProcess;
+  completed: boolean;
+  error?: Error;
+}

package/src/lib/utils.ts

@@ -0,0 +1,68 @@
+import { ChildProcess, spawn, SpawnOptionsWithoutStdio } from 'child_process';
+import { createInterface } from 'readline';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+export interface SpawnAsyncOptions {
+  dataCallback?: (data: string) => void;
+  errorCallback?: (data: string) => void;
+}
+
+export async function spawnAsync(
+  command: string,
+  args?: ReadonlyArray<string>,
+  options?: SpawnOptionsWithoutStdio,
+  logging?: SpawnAsyncOptions
+): Promise<ChildProcess> {
+  return new Promise((resolve, reject) => {
+    const childProcess = spawn(command, args, options);
+
+    if (logging.dataCallback) {
+      const readlineStdout = createInterface({ input: childProcess.stdout });
+      readlineStdout.on('line', logging.dataCallback);
+    }
+
+    if (logging.errorCallback) {
+      const readlineStderr = createInterface({ input: childProcess.stderr });
+      readlineStderr.on('line', logging.errorCallback);
+    }
+
+    childProcess.on('close', (code) => {
+      if (code === 0) {
+        resolve(childProcess);
+      } else {
+        reject(childProcess);
+      }
+    });
+
+    childProcess.on('error', (err: Error) => {
+      reject(childProcess); // TODO: try to return error message
+    });
+  });
+}
+
+export function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+export function getTaskLogFilename(taskId: number): string {
+  return join(tmpdir(), `task-${taskId}.log`);
+}
+
+export class HTTPResponseError extends Error {
+  constructor(public readonly response) {
+    super(`HTTP Error Response: ${response.status} ${response.statusText}`);
+    this.response = response;
+  }
+}
+
+export const checkStatus = (response) => {
+  if (response.ok) {
+    // response.status >= 200 && response.status < 300
+    return response;
+  } else {
+    throw new HTTPResponseError(response);
+  }
+};

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

package/tsconfig.lib.json

@@ -0,0 +1,12 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"],
+    "importHelpers": false
+  },
+  "exclude": ["**/*.spec.ts", "**/*.test.ts", "jest.config.ts"],
+  "include": ["**/*.ts"]
+}