@tramvai/cli 5.53.80 → 5.53.94

package/src/api/benchmark/build.ts

@@ -1,21 +1,19 @@
+/* eslint-disable max-statements */
 import type { Container } from '@tinkoff/dippy';
 import type { Params as OriginalBuildParams, Result as OriginalBuildResult } from '../build/index';
 import { COMMAND_PARAMETERS_TOKEN, COMMAND_RUNNER_TOKEN } from '../../di/tokens';
 import type { Params, Result } from './index';
-import type { Samples, RunStats } from './types';
+import type { Samples, RunStats, CompilationStats } from './types';
 import { clearCacheDirectory } from './utils/clearCache';
 import { getResultStats } from './utils/stats';
+import { getClientCompilationTimings, getServerCompilationTimings } from './utils/compilationUtils';
+import { DEFAULT_TIMES } from './const';
 
 export interface BuildParams extends Params {
   command: 'build';
   commandOptions: OriginalBuildParams;
 }
 
-export interface BuildResult extends Result {
-  noCache?: RunStats;
-  cache?: RunStats;
-}
-
 const runBuildCommand = async (
   di: Container,
   {
@@ -26,49 +24,96 @@ const runBuildCommand = async (
     shouldClearCache: boolean;
   }
 ): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
+  const clientBuildTimeSamples: number[] = [];
+  const serverBuildTimeSamples: number[] = [];
+
+  const clientMaxMemoryRssSamples: number[] = [];
+  const serverMaxMemoryRssSamples: number[] = [];
+
+  const serverCompilationTimings: CompilationStats[] = [];
+  const clientCompilationTimings: CompilationStats[] = [];
+
+  const maxMemoryRssSamples: number[] = [];
 
   const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as BuildParams;
+  commandOptions.benchmark = true;
+
+  const buildType = commandOptions.buildType ?? 'all';
+
+  const benchmarkStartTime = new Date().toISOString();
 
   for (let i = 0; i < times; i++) {
     if (shouldClearCache) {
       await clearCacheDirectory(di);
     }
 
-    const { getBuildStats: getStats } = await (di
+    const attemptStartTime = Date.now();
+    const { getBuildStats } = await (di
       .get(COMMAND_RUNNER_TOKEN)
       .run('build', commandOptions) as OriginalBuildResult);
-    const stats = getStats();
+    const stats = getBuildStats();
+
+    // at first attempt do not save metrics if cache is used
+    if (!shouldClearCache && i === 0) {
+      continue;
+    }
+
+    if (buildType === 'all' || buildType === 'server') {
+      serverCompilationTimings.push(
+        await getServerCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      serverBuildTimeSamples.push(stats.server.buildTime);
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
+      if (stats.server.maxMemoryRss) {
+        clientMaxMemoryRssSamples.push(stats.server.maxMemoryRss);
+      }
+    }
+
+    if (buildType === 'all' || buildType === 'client') {
+      clientCompilationTimings.push(
+        await getClientCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      clientBuildTimeSamples.push(stats.client.buildTime);
+
+      if (stats.client.maxMemoryRss) {
+        clientMaxMemoryRssSamples.push(stats.client.maxMemoryRss);
+      }
+    }
+
+    if (stats.maxMemoryRss) {
+      maxMemoryRssSamples.push(stats.maxMemoryRss);
+    }
   }
 
   return {
-    clientSamples,
-    serverSamples,
+    serverCompilationTimings,
+    clientCompilationTimings,
+    clientBuildTimeSamples,
+    serverBuildTimeSamples,
     maxMemoryRssSamples,
+    clientMaxMemoryRssSamples,
+    serverMaxMemoryRssSamples,
   };
 };
 
-export const benchmarkBuild = async (di: Container): Promise<BuildResult> => {
-  const { times = 5 } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
+export const benchmarkBuild = async (di: Container): Promise<Result> => {
+  const { times = DEFAULT_TIMES, commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
+  const noCache = !commandOptions.fileCache;
 
-  const noCache = await runBuildCommand(di, {
-    times: Math.max(Math.floor(times / 2), 2),
-    shouldClearCache: true,
-  });
+  let result;
 
-  const cache = await runBuildCommand(di, {
-    times,
-    shouldClearCache: false,
-  });
+  if (noCache) {
+    result = await runBuildCommand(di, {
+      times,
+      shouldClearCache: true,
+    });
+  } else {
+    result = await runBuildCommand(di, {
+      // additional first attempt for cache warmup
+      times: times + 1,
+      shouldClearCache: false,
+    });
+  }
 
-  return {
-    cache: getResultStats(cache),
-    noCache: getResultStats(noCache),
-  };
+  return getResultStats(result);
 };

package/src/api/benchmark/const.ts

@@ -0,0 +1 @@
+export const DEFAULT_TIMES = 3;

package/src/api/benchmark/index.ts

@@ -1,9 +1,9 @@
 import type { Provider } from '@tinkoff/dippy';
 import { createCommand } from '../../commands/createCommand';
 import { COMMAND_PARAMETERS_TOKEN, WITH_BUILD_STATS_TOKEN } from '../../di/tokens';
-import type { BuildParams, BuildResult } from './build';
+import type { BuildParams } from './build';
 import { benchmarkBuild } from './build';
-import type { StartParams, StartResult } from './start';
+import type { StartParams } from './start';
 import { benchmarkStart } from './start';
 import type { RunStats } from './types';
 
@@ -13,14 +13,12 @@ export interface Params {
   times?: number;
 }
 
-export interface Result {
-  [key: string]: RunStats | undefined;
-}
+export type Result = RunStats;
 
 export type BenchmarkCommand = (
   params: StartParams | BuildParams,
   providers?: Provider[]
-) => Promise<StartResult | BuildResult>;
+) => Promise<Result>;
 
 export default createCommand({
   name: 'benchmark',

package/src/api/benchmark/start.ts

@@ -1,24 +1,19 @@
+/* eslint-disable max-statements */
 import type { Container } from '@tinkoff/dippy';
 import type { Params as OriginalStartParams, Result as OriginalStartResult } from '../start/index';
 import { COMMAND_PARAMETERS_TOKEN, COMMAND_RUNNER_TOKEN } from '../../di/tokens';
 import type { Params, Result } from './index';
-import type { Samples, RunStats } from './types';
+import type { Samples, CompilationStats } from './types';
 import { clearCacheDirectory } from './utils/clearCache';
 import { getResultStats } from './utils/stats';
-
-const REBUILD_WARMUP_TIMES = 3;
+import { getClientCompilationTimings, getServerCompilationTimings } from './utils/compilationUtils';
+import { DEFAULT_TIMES } from './const';
 
 export interface StartParams extends Params {
   command: 'start';
   commandOptions: OriginalStartParams;
 }
 
-export interface StartResult extends Result {
-  noCache?: RunStats;
-  cache?: RunStats;
-  rebuild?: RunStats;
-}
-
 const runStartCommand = async (
   di: Container,
   {
@@ -29,17 +24,29 @@ const runStartCommand = async (
     shouldClearCache: boolean;
   }
 ): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
+  const clientBuildTimeSamples: number[] = [];
+  const serverBuildTimeSamples: number[] = [];
+  const clientMaxMemoryRssSamples: number[] = [];
+  const serverMaxMemoryRssSamples: number[] = [];
+
+  const serverCompilationTimings: CompilationStats[] = [];
+  const clientCompilationTimings: CompilationStats[] = [];
+
+  const maxMemoryRssSamples: number[] = [];
 
   const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as StartParams;
+  commandOptions.benchmark = true;
+
+  const buildType = commandOptions.buildType ?? 'all';
+
+  const benchmarkStartTime = new Date().toISOString();
 
   for (let i = 0; i < times; i++) {
     if (shouldClearCache) {
       await clearCacheDirectory(di);
     }
 
+    const attemptStartTime = Date.now();
     const { close, getBuildStats } = await (di
       .get(COMMAND_RUNNER_TOKEN)
       .run('start', commandOptions) as OriginalStartResult);
@@ -47,70 +54,67 @@ const runStartCommand = async (
 
     await close();
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
-  }
-
-  return {
-    clientSamples,
-    serverSamples,
-    maxMemoryRssSamples,
-  };
-};
-
-const runRebuild = async (di: Container, { times }: { times: number }): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
-  const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as StartParams;
+    // at first attempt do not save metrics if cache is used
+    if (!shouldClearCache && i === 0) {
+      continue;
+    }
 
-  const { close, invalidate, getBuildStats } = await (di
-    .get(COMMAND_RUNNER_TOKEN)
-    .run('start', commandOptions) as OriginalStartResult);
+    if (buildType === 'all' || buildType === 'server') {
+      serverCompilationTimings.push(
+        await getServerCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      serverBuildTimeSamples.push(stats.server.buildTime);
 
-  // warmup rebuild as it usually pretty slow at first runs
-  for (let i = 0; i < REBUILD_WARMUP_TIMES; i++) {
-    await invalidate();
-  }
+      if (stats.server.maxMemoryRss) {
+        serverMaxMemoryRssSamples.push(stats.server.maxMemoryRss);
+      }
+    }
 
-  for (let i = 0; i < times; i++) {
-    await invalidate();
+    if (buildType === 'all' || buildType === 'client') {
+      clientCompilationTimings.push(
+        await getClientCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      clientBuildTimeSamples.push(stats.client.buildTime);
 
-    const stats = getBuildStats();
+      if (stats.client.maxMemoryRss) {
+        clientMaxMemoryRssSamples.push(stats.client.maxMemoryRss);
+      }
+    }
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
+    if (stats.maxMemoryRss) {
+      maxMemoryRssSamples.push(stats.maxMemoryRss);
+    }
   }
 
-  await close();
-
   return {
-    clientSamples,
-    serverSamples,
+    serverCompilationTimings,
+    clientCompilationTimings,
+    clientMaxMemoryRssSamples,
+    serverMaxMemoryRssSamples,
+    clientBuildTimeSamples,
+    serverBuildTimeSamples,
     maxMemoryRssSamples,
   };
 };
 
-export const benchmarkStart = async (di: Container): Promise<StartResult> => {
-  const { times = 5 } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
-
-  const noCache = await runStartCommand(di, {
-    times: Math.max(Math.floor(times / 3), 2),
-    shouldClearCache: true,
-  });
-
-  const cache = await runStartCommand(di, {
-    times: Math.max(Math.floor(times / 2), 2),
-    shouldClearCache: false,
-  });
-
-  const rebuild = await runRebuild(di, { times });
+export const benchmarkStart = async (di: Container): Promise<Result> => {
+  const { times = DEFAULT_TIMES, commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
+  const noCache = !commandOptions.fileCache;
+
+  let result;
+
+  if (noCache) {
+    result = await runStartCommand(di, {
+      times,
+      shouldClearCache: true,
+    });
+  } else {
+    result = await runStartCommand(di, {
+      // additional first attempt for cache warmup
+      times: times + 1,
+      shouldClearCache: false,
+    });
+  }
 
-  return {
-    cache: getResultStats(cache),
-    noCache: getResultStats(noCache),
-    rebuild: getResultStats(rebuild),
-  };
+  return getResultStats(result);
 };

package/src/api/benchmark/types.ts

@@ -1,19 +1,87 @@
-export interface Stats {
-  samples: number[];
-  mean: number;
-  std: number;
-
-  variance: number;
-}
-
 export type Samples = {
-  clientSamples: Stats['samples'];
-  serverSamples: Stats['samples'];
-  maxMemoryRssSamples: Stats['samples'];
+  serverCompilationTimings: CompilationStats[];
+  serverBuildTimeSamples: number[];
+  clientCompilationTimings: CompilationStats[];
+  clientBuildTimeSamples: number[];
+  maxMemoryRssSamples: number[];
+  clientMaxMemoryRssSamples: number[];
+  serverMaxMemoryRssSamples: number[];
+};
+
+export type CompilationStats = {
+  totalBuildCosts: Record<string, number>;
+  loaderBuildCosts: Record<string, number>;
+  pluginBuildCosts: Record<string, number>;
 };
 
 export type RunStats = {
-  client: Stats;
-  server: Stats;
-  maxMemoryRss: Stats;
+  serverCompilationStats: CompilationStats | undefined;
+  clientCompilationStats: CompilationStats | undefined;
+  clientBuildTime: number | undefined;
+  serverBuildTime: number | undefined;
+  clientMaxMemoryRss: number | undefined;
+  serverMaxMemoryRss: number | undefined;
+  maxMemoryRss: number | undefined;
+};
+
+type PlainObject<T = any> = {
+  [key: string]: T;
 };
+
+interface ProcessData {
+  /**
+   * process id
+   */
+  pid: number;
+  /**
+   * parent process id
+   */
+  ppid: number | null;
+}
+
+export interface LoaderTransformData extends ProcessData {
+  /** loader name */
+  loader: string;
+  /**
+   * loader index
+   */
+  loaderIndex: number;
+  /** loader path */
+  path: string;
+  input: string | null | undefined;
+  /**
+   * - isPitch: true: the result of loader.pitch()
+   * - isPitch: false: the code result of loader()
+   */
+  result: string | null | undefined;
+  /** Timestamp when called */
+  startAt: number;
+  endAt: number;
+  /** loader configuration */
+  options: PlainObject;
+  /** pitching loader */
+  isPitch: boolean;
+  /**
+   * is sync
+   */
+  sync: boolean;
+  /** Error during conversion */
+  // errors: DevToolErrorInstance[];
+  /** module layer */
+  layer?: string;
+}
+
+export type MinimalLoaderData = Pick<LoaderTransformData, 'startAt' | 'endAt' | 'pid' | 'loader'>;
+
+export interface PluginData {
+  /** hook tap name */
+  tapName: string;
+  /** hook call time-consuming */
+  costs: number;
+  startAt: number;
+  endAt: number;
+  /** hook function type */
+  type: 'sync' | 'async' | 'promise';
+  /** hook function result */
+  result: any;
+}

package/src/api/benchmark/utils/compilationUtils.ts

@@ -0,0 +1,213 @@
+import fs from 'node:fs/promises';
+
+import { CompilationStats, LoaderTransformData, MinimalLoaderData, PluginData } from '../types';
+
+export async function getServerCompilationTimings(
+  benchmarkStartTime: string,
+  attemptStartTime: number,
+  attempt: number
+) {
+  return getCompilationTimings(benchmarkStartTime, attemptStartTime, attempt, 'server');
+}
+
+export async function getClientCompilationTimings(
+  benchmarkStartTime: string,
+  attemptStartTime: number,
+  attempt: number
+) {
+  return getCompilationTimings(benchmarkStartTime, attemptStartTime, attempt, 'client');
+}
+
+async function getCompilationTimings(
+  benchmarkStartTime: string,
+  attemptStartTime: number,
+  attempt: number,
+  type: 'client' | 'server'
+): Promise<CompilationStats> {
+  try {
+    const rsdoctorData = await getReportData(
+      await processReport(benchmarkStartTime, attempt, type)
+    );
+
+    const { summary, loader, plugin } = rsdoctorData;
+
+    const totalBuildCosts = calculateTotalCosts(summary, attemptStartTime);
+    const loaderBuildCosts = calculateLoadersCosts(loader);
+    const pluginBuildCosts = calculatePluginsCosts(plugin);
+
+    return { totalBuildCosts, loaderBuildCosts, pluginBuildCosts };
+  } catch (err) {
+    throw new Error(`Failed to get rsdoctor report!\n${err}`);
+  }
+}
+
+async function getReportData(reportPath: string) {
+  const rsdoctorRawStats = await fs.readFile(reportPath, 'utf-8');
+  const rsdoctorStats = JSON.parse(rsdoctorRawStats);
+
+  return rsdoctorStats.data;
+}
+
+async function processReport(
+  benchmarkStartTime: string,
+  attempt: number,
+  type: 'client' | 'server'
+) {
+  const reportBasePath = `./.rsdoctor/${benchmarkStartTime}`;
+  const reportPath = `${reportBasePath}/${type}-rsdoctor-data-${attempt + 1}.json`;
+
+  await fs.mkdir(reportBasePath, { recursive: true });
+  await fs.rename(`./.rsdoctor/${type}-rsdoctor-data.json`, reportPath);
+
+  return reportPath;
+}
+
+function mergeIntervals(intervals: [number, number][]) {
+  // Sort from small to large
+  intervals.sort((a, b) => a[0] - b[0]);
+  // The previous interval, the next interval, store the result
+  let previous;
+  let current;
+  const result: [number, number][] = [];
+
+  for (let i = 0; i < intervals.length; i++) {
+    current = intervals[i];
+    // If the first interval or the current interval does not overlap with the previous interval, add the current interval to the result
+    if (!previous || current[0] > previous[1]) {
+      // Assign the current interval to the previous interval
+      previous = current;
+      result.push(current);
+    } else {
+      // Otherwise, the two intervals overlap
+      // Update the end time of the previous interval
+      previous[1] = Math.max(previous[1], current[1]);
+    }
+  }
+
+  return result;
+}
+
+function getLoadersCosts(
+  filter: (loader: MinimalLoaderData) => boolean,
+  loaders: MinimalLoaderData[]
+) {
+  const match: { [pid: number | string]: [start: number, end: number][] } = {};
+  const others: { [pid: number | string]: [start: number, end: number][] } = {};
+
+  loaders.forEach((e) => {
+    if (filter(e)) {
+      if (!match[e.pid]) match[e.pid] = [];
+      match[e.pid].push([e.startAt, e.endAt]);
+    } else {
+      if (!others[e.pid]) others[e.pid] = [];
+      others[e.pid].push([e.startAt, e.endAt]);
+    }
+  });
+
+  let costs = 0;
+
+  const pids = Object.keys(match);
+
+  for (let i = 0; i < pids.length; i++) {
+    const pid = pids[i];
+    const _match = mergeIntervals(match[pid]);
+    // between in loader.startAt and loader.endAt
+    const _others = mergeIntervals(others[pid] || []).filter(([s, e]) =>
+      _match.some((el) => s >= el[0] && e <= el[1])
+    );
+
+    // eslint-disable-next-line no-param-reassign
+    const matchSum = _match.length ? _match.reduce((t, c) => (t += c[1] - c[0]), 0) : 0;
+    // eslint-disable-next-line no-param-reassign
+    const othersSum = _others.length ? _others.reduce((t, c) => (t += c[1] - c[0]), 0) : 0;
+
+    costs += matchSum - othersSum;
+  }
+
+  return costs;
+}
+
+function calculateTotalCosts(
+  summary: { costs: { name: string; cost: string }[] },
+  attemptStartTime: number
+) {
+  const { costs: buildCosts } = summary;
+  const buildCostsMap = buildCosts.reduce((acc, cost) => {
+    acc[cost.name] = cost;
+    return acc;
+  }, {});
+
+  const result: Record<string, number> = {};
+
+  for (const buildCostName in buildCostsMap) {
+    if (buildCostName === 'bootstrap->beforeCompile') {
+      // start of bootstrap is start of first build, so use attemptStarTime instead
+      result[buildCostName] =
+        buildCostsMap['beforeCompile->afterCompile'].startAt - attemptStartTime;
+    } else {
+      result[buildCostName] = buildCostsMap[buildCostName].costs;
+    }
+  }
+
+  return result;
+}
+
+function calculateLoadersCosts(loaders: { loaders: LoaderTransformData[] }[]) {
+  const filteredLoaders: MinimalLoaderData[] = [];
+  const uniqueLoaders = new Map<
+    string,
+    {
+      files: number;
+      path: string;
+    }
+  >();
+
+  loaders.forEach((data) => {
+    data.loaders.forEach((fl) => {
+      const uniqueLoader = uniqueLoaders.get(fl.loader);
+      if (uniqueLoader) {
+        uniqueLoaders.set(fl.loader, {
+          files: uniqueLoader.files + 1,
+          path: fl.path,
+        });
+      } else {
+        uniqueLoaders.set(fl.loader, { files: 1, path: fl.path });
+      }
+
+      return filteredLoaders.push({
+        loader: fl.loader,
+        startAt: fl.startAt,
+        endAt: fl.endAt,
+        pid: fl.pid,
+      });
+    });
+  });
+
+  const costs: Record<string, number> = {};
+  uniqueLoaders.forEach((_, loaderName: string) => {
+    costs[loaderName] = getLoadersCosts((item) => item.loader === loaderName, filteredLoaders);
+  });
+
+  return costs;
+}
+
+function calculatePluginsCosts(plugins: PluginData) {
+  const pluginCosts: Record<string, number> = {};
+
+  for (const hookName in plugins) {
+    const hookCalls = plugins[hookName];
+
+    hookCalls.forEach((plugin) => {
+      const pluginName = plugin.tapName;
+      const pluginCost = plugin.costs;
+
+      if (pluginCosts[pluginName]) {
+        pluginCosts[pluginName] += pluginCost;
+      } else {
+        pluginCosts[pluginName] = pluginCost;
+      }
+    });
+  }
+
+  return pluginCosts;
+}