@tramvai/cli 5.53.81 → 5.53.111

package/src/api/benchmark/__integration__/start.test.ts

@@ -6,15 +6,11 @@ const FIXTURES_DIR = resolve(__dirname, '__fixtures__');
 jest.useRealTimers();
 jest.setTimeout(120000);
 
-const matchStats = {
-  samples: expect.any(Array),
-  mean: expect.any(Number),
-  std: expect.any(Number),
-  variance: expect.any(Number),
-};
-const matchFullStats = {
-  client: matchStats,
-  server: matchStats,
+const compilationStats = {
+  // TODO: not working in CI, loaders field in report is empty: TCORE-5363
+  // loaderBuildCosts: expect.any(Object),
+  totalBuildCosts: expect.any(Object),
+  pluginBuildCosts: expect.any(Object),
 };
 
 describe('@tramvai/cli benchmark command', () => {
@@ -31,9 +27,11 @@ describe('@tramvai/cli benchmark command', () => {
       });
 
       expect(result).toMatchObject({
-        cache: matchFullStats,
-        noCache: matchFullStats,
-        rebuild: matchFullStats,
+        clientBuildTime: expect.any(Number),
+        serverBuildTime: expect.any(Number),
+        clientCompilationStats: compilationStats,
+        serverCompilationStats: compilationStats,
+        maxMemoryRss: expect.any(Number),
       });
     });
   });

package/src/api/benchmark/build.ts

@@ -1,21 +1,19 @@
+/* eslint-disable max-statements */
 import type { Container } from '@tinkoff/dippy';
 import type { Params as OriginalBuildParams, Result as OriginalBuildResult } from '../build/index';
 import { COMMAND_PARAMETERS_TOKEN, COMMAND_RUNNER_TOKEN } from '../../di/tokens';
 import type { Params, Result } from './index';
-import type { Samples, RunStats } from './types';
+import type { Samples, RunStats, CompilationStats } from './types';
 import { clearCacheDirectory } from './utils/clearCache';
 import { getResultStats } from './utils/stats';
+import { getClientCompilationTimings, getServerCompilationTimings } from './utils/compilationUtils';
+import { DEFAULT_TIMES } from './const';
 
 export interface BuildParams extends Params {
   command: 'build';
   commandOptions: OriginalBuildParams;
 }
 
-export interface BuildResult extends Result {
-  noCache?: RunStats;
-  cache?: RunStats;
-}
-
 const runBuildCommand = async (
   di: Container,
   {
@@ -26,49 +24,96 @@ const runBuildCommand = async (
     shouldClearCache: boolean;
   }
 ): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
+  const clientBuildTimeSamples: number[] = [];
+  const serverBuildTimeSamples: number[] = [];
+
+  const clientMaxMemoryRssSamples: number[] = [];
+  const serverMaxMemoryRssSamples: number[] = [];
+
+  const serverCompilationTimings: CompilationStats[] = [];
+  const clientCompilationTimings: CompilationStats[] = [];
+
+  const maxMemoryRssSamples: number[] = [];
 
   const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as BuildParams;
+  commandOptions.benchmark = true;
+
+  const buildType = commandOptions.buildType ?? 'all';
+
+  const benchmarkStartTime = new Date().toISOString();
 
   for (let i = 0; i < times; i++) {
     if (shouldClearCache) {
       await clearCacheDirectory(di);
     }
 
-    const { getBuildStats: getStats } = await (di
+    const attemptStartTime = Date.now();
+    const { getBuildStats } = await (di
       .get(COMMAND_RUNNER_TOKEN)
       .run('build', commandOptions) as OriginalBuildResult);
-    const stats = getStats();
+    const stats = getBuildStats();
+
+    // at first attempt do not save metrics if cache is used
+    if (!shouldClearCache && i === 0) {
+      continue;
+    }
+
+    if (buildType === 'all' || buildType === 'server') {
+      serverCompilationTimings.push(
+        await getServerCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      serverBuildTimeSamples.push(stats.server.buildTime);
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
+      if (stats.server.maxMemoryRss) {
+        clientMaxMemoryRssSamples.push(stats.server.maxMemoryRss);
+      }
+    }
+
+    if (buildType === 'all' || buildType === 'client') {
+      clientCompilationTimings.push(
+        await getClientCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      clientBuildTimeSamples.push(stats.client.buildTime);
+
+      if (stats.client.maxMemoryRss) {
+        clientMaxMemoryRssSamples.push(stats.client.maxMemoryRss);
+      }
+    }
+
+    if (stats.maxMemoryRss) {
+      maxMemoryRssSamples.push(stats.maxMemoryRss);
+    }
   }
 
   return {
-    clientSamples,
-    serverSamples,
+    serverCompilationTimings,
+    clientCompilationTimings,
+    clientBuildTimeSamples,
+    serverBuildTimeSamples,
     maxMemoryRssSamples,
+    clientMaxMemoryRssSamples,
+    serverMaxMemoryRssSamples,
   };
 };
 
-export const benchmarkBuild = async (di: Container): Promise<BuildResult> => {
-  const { times = 5 } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
+export const benchmarkBuild = async (di: Container): Promise<Result> => {
+  const { times = DEFAULT_TIMES, commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
+  const noCache = !commandOptions.fileCache;
 
-  const noCache = await runBuildCommand(di, {
-    times: Math.max(Math.floor(times / 2), 2),
-    shouldClearCache: true,
-  });
+  let result;
 
-  const cache = await runBuildCommand(di, {
-    times,
-    shouldClearCache: false,
-  });
+  if (noCache) {
+    result = await runBuildCommand(di, {
+      times,
+      shouldClearCache: true,
+    });
+  } else {
+    result = await runBuildCommand(di, {
+      // additional first attempt for cache warmup
+      times: times + 1,
+      shouldClearCache: false,
+    });
+  }
 
-  return {
-    cache: getResultStats(cache),
-    noCache: getResultStats(noCache),
-  };
+  return getResultStats(result);
 };

package/src/api/benchmark/const.ts

@@ -0,0 +1 @@
+export const DEFAULT_TIMES = 3;

package/src/api/benchmark/index.ts

@@ -1,9 +1,9 @@
 import type { Provider } from '@tinkoff/dippy';
 import { createCommand } from '../../commands/createCommand';
 import { COMMAND_PARAMETERS_TOKEN, WITH_BUILD_STATS_TOKEN } from '../../di/tokens';
-import type { BuildParams, BuildResult } from './build';
+import type { BuildParams } from './build';
 import { benchmarkBuild } from './build';
-import type { StartParams, StartResult } from './start';
+import type { StartParams } from './start';
 import { benchmarkStart } from './start';
 import type { RunStats } from './types';
 
@@ -13,14 +13,12 @@ export interface Params {
   times?: number;
 }
 
-export interface Result {
-  [key: string]: RunStats | undefined;
-}
+export type Result = RunStats;
 
 export type BenchmarkCommand = (
   params: StartParams | BuildParams,
   providers?: Provider[]
-) => Promise<StartResult | BuildResult>;
+) => Promise<Result>;
 
 export default createCommand({
   name: 'benchmark',

package/src/api/benchmark/start.ts

@@ -1,24 +1,19 @@
+/* eslint-disable max-statements */
 import type { Container } from '@tinkoff/dippy';
 import type { Params as OriginalStartParams, Result as OriginalStartResult } from '../start/index';
 import { COMMAND_PARAMETERS_TOKEN, COMMAND_RUNNER_TOKEN } from '../../di/tokens';
 import type { Params, Result } from './index';
-import type { Samples, RunStats } from './types';
+import type { Samples, CompilationStats } from './types';
 import { clearCacheDirectory } from './utils/clearCache';
 import { getResultStats } from './utils/stats';
-
-const REBUILD_WARMUP_TIMES = 3;
+import { getClientCompilationTimings, getServerCompilationTimings } from './utils/compilationUtils';
+import { DEFAULT_TIMES } from './const';
 
 export interface StartParams extends Params {
   command: 'start';
   commandOptions: OriginalStartParams;
 }
 
-export interface StartResult extends Result {
-  noCache?: RunStats;
-  cache?: RunStats;
-  rebuild?: RunStats;
-}
-
 const runStartCommand = async (
   di: Container,
   {
@@ -29,17 +24,29 @@ const runStartCommand = async (
     shouldClearCache: boolean;
   }
 ): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
+  const clientBuildTimeSamples: number[] = [];
+  const serverBuildTimeSamples: number[] = [];
+  const clientMaxMemoryRssSamples: number[] = [];
+  const serverMaxMemoryRssSamples: number[] = [];
+
+  const serverCompilationTimings: CompilationStats[] = [];
+  const clientCompilationTimings: CompilationStats[] = [];
+
+  const maxMemoryRssSamples: number[] = [];
 
   const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as StartParams;
+  commandOptions.benchmark = true;
+
+  const buildType = commandOptions.buildType ?? 'all';
+
+  const benchmarkStartTime = new Date().toISOString();
 
   for (let i = 0; i < times; i++) {
     if (shouldClearCache) {
       await clearCacheDirectory(di);
     }
 
+    const attemptStartTime = Date.now();
     const { close, getBuildStats } = await (di
       .get(COMMAND_RUNNER_TOKEN)
       .run('start', commandOptions) as OriginalStartResult);
@@ -47,70 +54,67 @@ const runStartCommand = async (
 
     await close();
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
-  }
-
-  return {
-    clientSamples,
-    serverSamples,
-    maxMemoryRssSamples,
-  };
-};
-
-const runRebuild = async (di: Container, { times }: { times: number }): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
-  const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as StartParams;
+    // at first attempt do not save metrics if cache is used
+    if (!shouldClearCache && i === 0) {
+      continue;
+    }
 
-  const { close, invalidate, getBuildStats } = await (di
-    .get(COMMAND_RUNNER_TOKEN)
-    .run('start', commandOptions) as OriginalStartResult);
+    if (buildType === 'all' || buildType === 'server') {
+      serverCompilationTimings.push(
+        await getServerCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      serverBuildTimeSamples.push(stats.server.buildTime);
 
-  // warmup rebuild as it usually pretty slow at first runs
-  for (let i = 0; i < REBUILD_WARMUP_TIMES; i++) {
-    await invalidate();
-  }
+      if (stats.server.maxMemoryRss) {
+        serverMaxMemoryRssSamples.push(stats.server.maxMemoryRss);
+      }
+    }
 
-  for (let i = 0; i < times; i++) {
-    await invalidate();
+    if (buildType === 'all' || buildType === 'client') {
+      clientCompilationTimings.push(
+        await getClientCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      clientBuildTimeSamples.push(stats.client.buildTime);
 
-    const stats = getBuildStats();
+      if (stats.client.maxMemoryRss) {
+        clientMaxMemoryRssSamples.push(stats.client.maxMemoryRss);
+      }
+    }
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
+    if (stats.maxMemoryRss) {
+      maxMemoryRssSamples.push(stats.maxMemoryRss);
+    }
   }
 
-  await close();
-
   return {
-    clientSamples,
-    serverSamples,
+    serverCompilationTimings,
+    clientCompilationTimings,
+    clientMaxMemoryRssSamples,
+    serverMaxMemoryRssSamples,
+    clientBuildTimeSamples,
+    serverBuildTimeSamples,
     maxMemoryRssSamples,
   };
 };
 
-export const benchmarkStart = async (di: Container): Promise<StartResult> => {
-  const { times = 5 } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
-
-  const noCache = await runStartCommand(di, {
-    times: Math.max(Math.floor(times / 3), 2),
-    shouldClearCache: true,
-  });
-
-  const cache = await runStartCommand(di, {
-    times: Math.max(Math.floor(times / 2), 2),
-    shouldClearCache: false,
-  });
-
-  const rebuild = await runRebuild(di, { times });
+export const benchmarkStart = async (di: Container): Promise<Result> => {
+  const { times = DEFAULT_TIMES, commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
+  const noCache = !commandOptions.fileCache;
+
+  let result;
+
+  if (noCache) {
+    result = await runStartCommand(di, {
+      times,
+      shouldClearCache: true,
+    });
+  } else {
+    result = await runStartCommand(di, {
+      // additional first attempt for cache warmup
+      times: times + 1,
+      shouldClearCache: false,
+    });
+  }
 
-  return {
-    cache: getResultStats(cache),
-    noCache: getResultStats(noCache),
-    rebuild: getResultStats(rebuild),
-  };
+  return getResultStats(result);
 };

package/src/api/benchmark/types.ts

@@ -1,19 +1,87 @@
-export interface Stats {
-  samples: number[];
-  mean: number;
-  std: number;
-
-  variance: number;
-}
-
 export type Samples = {
-  clientSamples: Stats['samples'];
-  serverSamples: Stats['samples'];
-  maxMemoryRssSamples: Stats['samples'];
+  serverCompilationTimings: CompilationStats[];
+  serverBuildTimeSamples: number[];
+  clientCompilationTimings: CompilationStats[];
+  clientBuildTimeSamples: number[];
+  maxMemoryRssSamples: number[];
+  clientMaxMemoryRssSamples: number[];
+  serverMaxMemoryRssSamples: number[];
+};
+
+export type CompilationStats = {
+  totalBuildCosts: Record<string, number>;
+  loaderBuildCosts: Record<string, number>;
+  pluginBuildCosts: Record<string, number>;
 };
 
 export type RunStats = {
-  client: Stats;
-  server: Stats;
-  maxMemoryRss: Stats;
+  serverCompilationStats: CompilationStats | undefined;
+  clientCompilationStats: CompilationStats | undefined;
+  clientBuildTime: number | undefined;
+  serverBuildTime: number | undefined;
+  clientMaxMemoryRss: number | undefined;
+  serverMaxMemoryRss: number | undefined;
+  maxMemoryRss: number | undefined;
+};
+
+type PlainObject<T = any> = {
+  [key: string]: T;
 };
+
+interface ProcessData {
+  /**
+   * process id
+   */
+  pid: number;
+  /**
+   * parent process id
+   */
+  ppid: number | null;
+}
+
+export interface LoaderTransformData extends ProcessData {
+  /** loader name */
+  loader: string;
+  /**
+   * loader index
+   */
+  loaderIndex: number;
+  /** loader path */
+  path: string;
+  input: string | null | undefined;
+  /**
+   * - isPitch: true: the result of loader.pitch()
+   * - isPitch: false: the code result of loader()
+   */
+  result: string | null | undefined;
+  /** Timestamp when called */
+  startAt: number;
+  endAt: number;
+  /** loader configuration */
+  options: PlainObject;
+  /** pitching loader */
+  isPitch: boolean;
+  /**
+   * is sync
+   */
+  sync: boolean;
+  /** Error during conversion */
+  // errors: DevToolErrorInstance[];
+  /** module layer */
+  layer?: string;
+}
+
+export type MinimalLoaderData = Pick<LoaderTransformData, 'startAt' | 'endAt' | 'pid' | 'loader'>;
+
+export interface PluginData {
+  /** hook tap name */
+  tapName: string;
+  /** hook call time-consuming */
+  costs: number;
+  startAt: number;
+  endAt: number;
+  /** hook function type */
+  type: 'sync' | 'async' | 'promise';
+  /** hook function result */
+  result: any;
+}