@tskmgr/client 2.0.6 → 2.4.0

package/.babelrc

@@ -0,0 +1,10 @@
+{
+  "presets": [
+    [
+      "@nx/js/babel",
+      {
+        "useBuiltIns": "usage"
+      }
+    ]
+  ]
+}

package/README.md

@@ -4,4 +4,4 @@ This library was generated with [Nx](https://nx.dev).
 
 ## Running unit tests
 
-Run `nx test client` to execute the unit tests via [Jest](https://jestjs.io).
+Run `nx test client` to execute the unit tests via [Vitest](https://vitest.dev).

package/eslint.config.mjs

@@ -0,0 +1,28 @@
+import baseConfig from '../../eslint.config.mjs';
+
+export default [
+  {
+    ignores: ['**/dist'],
+  },
+  ...baseConfig,
+  {
+    files: ['**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx'],
+    // Override or add rules here
+    rules: {},
+  },
+  {
+    files: ['**/*.ts', '**/*.tsx'],
+    // Override or add rules here
+    rules: {},
+    languageOptions: {
+      parserOptions: {
+        project: ['libs/client/tsconfig.*?.json'],
+      },
+    },
+  },
+  {
+    files: ['**/*.js', '**/*.jsx'],
+    // Override or add rules here
+    rules: {},
+  },
+];

package/package.json

@@ -1,13 +1,10 @@
 {
   "name": "@tskmgr/client",
-  "version": "2.0.6",
+  "version": "2.4.0",
   "license": "MIT",
   "dependencies": {
     "debug": "^4.3.4",
-    "@tskmgr/common": "2.0.6",
-    "uuid": "^9.0.1",
-    "@nx/devkit": "18.0.7"
-  },
-  "main": "./src/index.js",
-  "type": "commonjs"
+    "p-retry": "^7.1.0",
+    "@tskmgr/common": "2.4.0"
+  }
 }

package/project.json

@@ -0,0 +1,31 @@
+{
+  "name": "client",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "libs/client/src",
+  "projectType": "library",
+  "tags": [],
+  "targets": {
+    "lint": {
+      "executor": "@nx/eslint:lint",
+      "outputs": ["{options.outputFile}"]
+    },
+    "test": {
+      "executor": "@nx/vite:test",
+      "outputs": ["{options.reportsDirectory}"],
+      "options": {
+        "reportsDirectory": "../../coverage/libs/client"
+      }
+    },
+    "build": {
+      "executor": "@nx/js:tsc",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "dist/libs/client",
+        "tsConfig": "libs/client/tsconfig.lib.json",
+        "packageJson": "libs/client/package.json",
+        "main": "libs/client/src/index.ts",
+        "assets": ["libs/client/*.md", "LICENSE"]
+      }
+    }
+  }
+}

package/src/{index.d.ts → index.ts}

@@ -1,5 +1,6 @@
-export * from './lib/client';
-export * from './lib/client-options';
-export * from './lib/client-factory';
-export * from './lib/run-tasks-result';
-export * from './lib/task-result';
+export * from './lib/client';
+export * from './lib/client-options';
+export * from './lib/client-factory';
+
+export * from './lib/run-tasks-result';
+export * from './lib/task-result';

package/src/lib/client-factory.ts

@@ -0,0 +1,17 @@
+import { Client } from './client';
+import { ClientOptions } from './client-options';
+import { ApiUrl } from '@tskmgr/common';
+
+export class ClientFactory {
+  public static createNew(
+    baseUrl: string, //
+    runnerId: string,
+    options?: ClientOptions
+  ): Client {
+    return new Client(
+      ApiUrl.create(baseUrl), //
+      runnerId,
+      { ...Client.DefaultOptions, ...options }
+    );
+  }
+}

package/src/lib/client-options.ts

@@ -0,0 +1,12 @@
+import { Task } from '@tskmgr/common';
+import { SpawnOptionsWithoutStdio } from 'child_process';
+
+export interface ClientOptions {
+  parallel?: number;
+  dataCallback?: (task: Task, data: string, cached: () => void) => void;
+  errorCallback?: (task: Task, data: string) => void;
+  pollingDelayMs?: number;
+  retryDelayMs?: number;
+  retryCount?: number;
+  spawnOptions?: SpawnOptionsWithoutStdio;
+}

package/src/lib/client.spec.ts

@@ -0,0 +1,36 @@
+import { Client } from './client';
+import { ClientFactory } from './client-factory';
+
+describe('client', () => {
+  let client: Client;
+
+  beforeEach(() => {
+    client = ClientFactory.createNew('http://localhost:3000', 'runner-1');
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should retry when rejected promise', async () => {
+    // arrange
+    vi.spyOn(global, 'fetch')
+      .mockRejectedValueOnce(new Error())
+      .mockResolvedValue({ ok: true, json: async () => ({}) } as Response);
+    // act
+    await client.uploadRunFile(1, './README.md', {});
+    // assert
+    expect(global.fetch).toHaveBeenCalledTimes(2);
+  });
+
+  it('should retry when fetch response not ok', async () => {
+    // arrange
+    vi.spyOn(global, 'fetch')
+      .mockResolvedValueOnce({ ok: false, statusText: 'Internal Server Error' } as Response)
+      .mockResolvedValue({ ok: true, json: async () => ({}) } as Response);
+    // act
+    await client.uploadRunFile(1, './README.md', {});
+    // assert
+    expect(global.fetch).toHaveBeenCalledTimes(2);
+  });
+});

package/src/lib/client.ts

@@ -0,0 +1,265 @@
+import { ChildProcess } from 'node:child_process';
+import {
+  ApiUrl,
+  Run,
+  CompleteTaskDto,
+  CreateRunRequestDto,
+  CreateTasksDto,
+  StartTaskDto,
+  StartTaskResponseDto,
+  Task,
+  File as File_,
+  SetLeaderRequestDto,
+  SetLeaderResponseDto,
+  CreateFileRequestDto,
+} from '@tskmgr/common';
+import { delay, fetchWithRetry, getTaskLogFilename, spawnAsync } from './utils';
+import { RunTasksResult } from './run-tasks-result';
+import { TaskResult } from './task-result';
+import { ClientOptions } from './client-options';
+import { createWriteStream, unlinkSync } from 'node:fs';
+import Debug from 'debug';
+import { readFile } from 'node:fs/promises';
+import { SpawnOptionsWithoutStdio } from 'node:child_process';
+
+const debug = Debug('tskmgr:client');
+const headers = { 'Content-Type': 'application/json' };
+
+export class Client {
+  public static readonly DefaultOptions: ClientOptions = {
+    parallel: 1,
+    dataCallback: (task, data, cached) => {
+      // noop
+    },
+    errorCallback: (task, data) => {
+      // noop
+    },
+    pollingDelayMs: 5000,
+    retryDelayMs: 5000,
+    retryCount: 2,
+  };
+
+  constructor(
+    private readonly apiUrl: ApiUrl,
+    private readonly runnerId: string,
+    private readonly clientOptions: ClientOptions,
+  ) {}
+
+  public async createRun(params: CreateRunRequestDto): Promise<Run> {
+    return fetchWithRetry(this.apiUrl.createRunUrl(), {
+      headers,
+      method: 'POST',
+      body: JSON.stringify(params),
+    });
+  }
+
+  public async closeRun(runId: number): Promise<Run> {
+    return fetchWithRetry(this.apiUrl.closeRunUrl(runId), {
+      headers,
+      method: 'PUT',
+    });
+  }
+
+  public async abortRun(runId: number): Promise<Run> {
+    return fetchWithRetry(this.apiUrl.abortRunUrl(runId), {
+      headers,
+      method: 'PUT',
+    });
+  }
+
+  public async failRun(runId: number): Promise<Run> {
+    return fetchWithRetry(this.apiUrl.failRunUrl(runId), {
+      headers,
+      method: 'PUT',
+    });
+  }
+
+  public async setLeader(runId: number): Promise<SetLeaderResponseDto> {
+    const params: SetLeaderRequestDto = { runnerId: this.runnerId };
+    return fetchWithRetry(this.apiUrl.setLeaderUrl(runId), {
+      headers,
+      method: 'PUT',
+      body: JSON.stringify(params),
+    });
+  }
+
+  public async createTasks(runId: number, params: CreateTasksDto): Promise<Task[]> {
+    return fetchWithRetry(this.apiUrl.createTasksUrl(runId), {
+      headers,
+      method: 'POST',
+      body: JSON.stringify(params),
+    });
+  }
+
+  public async startTask(runId: number, params: StartTaskDto): Promise<StartTaskResponseDto> {
+    return fetchWithRetry(this.apiUrl.startTaskUrl(runId), {
+      headers,
+      method: 'PUT',
+      body: JSON.stringify(params),
+    });
+  }
+
+  public async runTasks(runId: number): Promise<RunTasksResult> {
+    const taskRunners: Promise<TaskResult[]>[] = [];
+
+    for (let i = 0; i < this.clientOptions.parallel; i++) {
+      taskRunners.push(this.defaultParallelTaskRunner(runId, i));
+    }
+
+    const taskResults = await Promise.all(taskRunners);
+    return new RunTasksResult(taskResults.flatMap((x) => x));
+  }
+
+  public async completeTask(taskId: number, params: CompleteTaskDto): Promise<Task> {
+    return fetchWithRetry(this.apiUrl.completeTaskUrl(taskId), {
+      headers,
+      method: 'PUT',
+      body: JSON.stringify(params),
+    });
+  }
+
+  public async failTask(taskId: number): Promise<Task> {
+    return fetchWithRetry(this.apiUrl.failTaskUrl(taskId), {
+      headers,
+      method: 'PUT',
+    });
+  }
+
+  public async uploadRunFile(runId: number, path: string, params: CreateFileRequestDto): Promise<File_> {
+    return this.uploadFile(this.apiUrl.createFileRunUrl(runId), path, params);
+  }
+
+  public async uploadTaskFile(taskId: number, path: string, params: CreateFileRequestDto): Promise<File_> {
+    return this.uploadFile(this.apiUrl.createFileTaskUrl(taskId), path, params);
+  }
+
+  private async uploadFile(url: string, path: string, params: CreateFileRequestDto): Promise<File_> {
+    const formData = new FormData();
+
+    const blob = new Blob([await readFile(path)]);
+    formData.append('file', blob, path);
+
+    if (params.type) {
+      formData.append('type', params.type);
+    }
+
+    if (params.description) {
+      formData.append('description', params.description);
+    }
+
+    return fetchWithRetry(url, {
+      method: 'POST',
+      body: formData,
+    });
+  }
+
+  private async defaultParallelTaskRunner(runId: number, parallelId: number): Promise<TaskResult[]> {
+    const logInfo = `[${this.runnerId}:${parallelId}]`;
+    const taskResults: TaskResult[] = [];
+    let _continue = true;
+
+    debug(`${logInfo} parallel task runner started`);
+
+    while (_continue) {
+      const res = await this.startTask(runId, { runnerId: this.runnerId });
+
+      if (!res.continue) {
+        debug(`${logInfo} continue set to false`);
+        _continue = false;
+      }
+
+      if (!res.continue && !res.task) {
+        debug(`${logInfo} continue set to false and no task, exiting`);
+        continue;
+      }
+
+      if (!res.task) {
+        debug(`${logInfo} polling for new tasks in ${this.clientOptions.pollingDelayMs} ms`);
+        await delay(this.clientOptions.pollingDelayMs);
+        continue;
+      }
+
+      const { run, task } = res;
+      debug(`${logInfo} received task: ${task.id}`);
+
+      let cached = false;
+      let completed = false;
+      let writable = false;
+      let childProcess: ChildProcess;
+      let error: Error;
+
+      const taskLogFilename = getTaskLogFilename(task.id);
+      const writeStream = createWriteStream(taskLogFilename);
+      writeStream.on('open', () => (writable = true));
+
+      const dataHandler = (data: string): void => {
+        this.clientOptions.dataCallback(task, data, () => (cached = true));
+        if (writable) {
+          writeStream.write(`[${new Date().toISOString()}] ${data}\n`);
+        }
+      };
+
+      const errorHandler = (data: string): void => {
+        this.clientOptions.errorCallback(task, data);
+        if (writable) {
+          writeStream.write(`[${new Date().toISOString()}] ${data}\n`);
+        }
+      };
+
+      const spawnOptions: SpawnOptionsWithoutStdio = {
+        ...(task.options as SpawnOptionsWithoutStdio),
+        ...this.clientOptions.spawnOptions,
+        env: {
+          ...process.env,
+          ...task.options?.env,
+          ...this.clientOptions.spawnOptions?.env,
+          TSKMGR_PARALLEL_ID: parallelId.toString(),
+        },
+      };
+
+      debug(`${logInfo} starting task: ${task.id}`);
+      try {
+        childProcess = await spawnAsync(task.command, task.arguments, spawnOptions, {
+          dataHandler,
+          errorHandler,
+        });
+        completed = true;
+        debug(`${logInfo} completed task: ${task.id}`);
+      } catch (err) {
+        console.error(`${logInfo} failed task: ${task.id} with error: ${err}`);
+        error = err;
+
+        if (run.failFast) {
+          // TODO: should abort all running tasks by current client
+          debug(`${logInfo} fail fast enabled, aborting`);
+          throw err;
+        }
+      } finally {
+        taskResults.push({ run, task, childProcess, completed, error });
+
+        if (completed) {
+          await this.completeTask(task.id, { cached });
+          debug(`${logInfo} completed status for task: ${task.id} sent`);
+        } else {
+          await this.failTask(task.id);
+          debug(`${logInfo} failed status for task: ${task.id} sent`);
+        }
+
+        writeStream.end();
+        writeStream.close();
+
+        await this.uploadTaskFile(task.id, taskLogFilename, {
+          type: 'log',
+          description: `Log for task ${task.id}`,
+        });
+
+        debug(`${logInfo} uploaded file for task: ${task.id} sent`);
+        unlinkSync(taskLogFilename);
+      }
+    }
+
+    debug(`${logInfo} parallel task runner completed`);
+
+    return taskResults;
+  }
+}

package/src/lib/run-tasks-result.ts

@@ -0,0 +1,17 @@
+import { TaskResult } from './task-result';
+
+export class RunTasksResult {
+  public readonly completedTasks: TaskResult[];
+  public readonly failedTasks: TaskResult[];
+
+  public readonly completed: boolean;
+  public readonly failed: boolean;
+
+  constructor(public readonly tasks: TaskResult[]) {
+    this.completedTasks = tasks.filter((x) => x.completed);
+    this.failedTasks = tasks.filter((x) => !x.completed);
+
+    this.completed = this.failedTasks.length === 0;
+    this.failed = this.failedTasks.length > 0;
+  }
+}

package/src/lib/task-result.ts

@@ -0,0 +1,10 @@
+import { Run, Task } from '@tskmgr/common';
+import { ChildProcess } from 'child_process';
+
+export interface TaskResult {
+  run: Run;
+  task: Task;
+  childProcess: ChildProcess;
+  completed: boolean;
+  error?: Error;
+}

package/src/lib/utils.spec.ts

@@ -0,0 +1,27 @@
+import { spawnAsync } from './utils';
+
+describe('Utils', () => {
+  it('should call dataHandler', async () => {
+    // arrange
+    const dataHandler = vi.fn();
+    const errorHandler = vi.fn();
+    // act
+    const command = `node -e 'console.log("some data"); process.exit(0)'`;
+    await expect(spawnAsync(command, [], { shell: true }, { dataHandler, errorHandler })).resolves.toBeDefined();
+    // assert
+    expect(dataHandler).toHaveBeenCalledWith('some data');
+    expect(errorHandler).not.toHaveBeenCalled();
+  });
+
+  it('should call errorHandler', async () => {
+    // arrange
+    const dataHandler = vi.fn();
+    const errorHandler = vi.fn();
+    // act
+    const command = `node -e 'console.error("some error"); process.exit(1)'`;
+    await expect(spawnAsync(command, [], { shell: true }, { dataHandler, errorHandler })).rejects.toBeDefined();
+    // assert
+    expect(dataHandler).not.toHaveBeenCalled();
+    expect(errorHandler).toHaveBeenCalledWith('some error');
+  });
+});

package/src/lib/utils.ts

@@ -0,0 +1,71 @@
+import { ChildProcess, spawn, SpawnOptionsWithoutStdio } from 'node:child_process';
+import { createInterface } from 'node:readline';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import pRetry from 'p-retry';
+import Debug from 'debug';
+
+const debug = Debug('tskmgr:client');
+
+export interface SpawnAsyncLineHandlers {
+  dataHandler?: (line: string) => void;
+  errorHandler?: (line: string) => void;
+}
+
+export async function spawnAsync(
+  command: string,
+  args?: ReadonlyArray<string>,
+  options?: SpawnOptionsWithoutStdio,
+  lineHandlers?: SpawnAsyncLineHandlers,
+): Promise<ChildProcess> {
+  return new Promise((resolve, reject) => {
+    const childProcess = spawn(command, args, options);
+
+    if (lineHandlers?.dataHandler) {
+      const readlineStdout = createInterface({ input: childProcess.stdout });
+      readlineStdout.on('line', lineHandlers.dataHandler);
+    }
+
+    if (lineHandlers?.errorHandler) {
+      const readlineStderr = createInterface({ input: childProcess.stderr });
+      readlineStderr.on('line', lineHandlers.errorHandler);
+    }
+
+    childProcess.on('close', (code) => {
+      if (code === 0) {
+        resolve(childProcess);
+      } else {
+        reject(childProcess);
+      }
+    });
+
+    childProcess.on('error', (err: Error) => {
+      reject(childProcess); // TODO: try to return error message
+    });
+  });
+}
+
+export function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+export function getTaskLogFilename(taskId: number): string {
+  return join(tmpdir(), `task-${taskId}.log`);
+}
+
+export async function fetchWithRetry<T>(input: string | URL | Request, init?: RequestInit): Promise<T> {
+  const run = async () => {
+    const response = await fetch(input, init);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status} ${response.statusText}: ${input}`);
+    }
+    return response.json();
+  };
+
+  return pRetry(run, {
+    retries: 3,
+    onFailedAttempt: (error) => debug('Retry attempt failed:', error),
+  });
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

package/tsconfig.lib.json

@@ -0,0 +1,27 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"],
+    "importHelpers": false
+  },
+  "exclude": [
+    "**/*.spec.ts",
+    "**/*.test.ts",
+    "vite.config.ts",
+    "vite.config.mts",
+    "vitest.config.ts",
+    "vitest.config.mts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.test.tsx",
+    "src/**/*.spec.tsx",
+    "src/**/*.test.js",
+    "src/**/*.spec.js",
+    "src/**/*.test.jsx",
+    "src/**/*.spec.jsx"
+  ],
+  "include": ["**/*.ts"]
+}

package/tsconfig.spec.json

@@ -0,0 +1,22 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "types": ["vitest/globals", "vitest/importMeta", "vite/client", "node", "vitest"]
+  },
+  "include": [
+    "vite.config.ts",
+    "vite.config.mts",
+    "vitest.config.ts",
+    "vitest.config.mts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.test.tsx",
+    "src/**/*.spec.tsx",
+    "src/**/*.test.js",
+    "src/**/*.spec.js",
+    "src/**/*.test.jsx",
+    "src/**/*.spec.jsx",
+    "src/**/*.d.ts"
+  ]
+}

package/vite.config.ts

@@ -0,0 +1,31 @@
+/// <reference types='vitest' />
+import { defineConfig } from 'vite';
+import { nxViteTsPaths } from '@nx/vite/plugins/nx-tsconfig-paths.plugin';
+import { nxCopyAssetsPlugin } from '@nx/vite/plugins/nx-copy-assets.plugin';
+
+export default defineConfig(() => ({
+  root: __dirname,
+  cacheDir: '../../node_modules/.vite/libs/client',
+  plugins: [nxViteTsPaths(), nxCopyAssetsPlugin(['*.md'])],
+  // Uncomment this if you are using workers.
+  // worker: {
+  //  plugins: [ nxViteTsPaths() ],
+  // },
+  test: {
+    name: 'client',
+    watch: false,
+    globals: true,
+    environment: 'jsdom',
+    include: ['{src,tests}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
+    reporters: ['default'],
+    coverage: {
+      reportsDirectory: '../../coverage/libs/client',
+      provider: 'v8' as const,
+    },
+    typecheck: {
+      enabled: true,
+      tsconfig: 'tsconfig.spec.json',
+      include: ['{src,tests}/**/*.{test,spec}.{ts,mts,tsx}'],
+    },
+  },
+}));