@tramvai/cli 6.68.0 → 6.68.2

package/src/api/benchmark/start.ts

@@ -1,24 +1,19 @@
+/* eslint-disable max-statements */
 import type { Container } from '@tinkoff/dippy';
 import type { Params as OriginalStartParams, Result as OriginalStartResult } from '../start/index';
 import { COMMAND_PARAMETERS_TOKEN, COMMAND_RUNNER_TOKEN } from '../../di/tokens';
 import type { Params, Result } from './index';
-import type { Samples, RunStats } from './types';
+import type { Samples, CompilationStats } from './types';
 import { clearCacheDirectory } from './utils/clearCache';
 import { getResultStats } from './utils/stats';
-
-const REBUILD_WARMUP_TIMES = 3;
+import { getClientCompilationTimings, getServerCompilationTimings } from './utils/compilationUtils';
+import { DEFAULT_TIMES } from './const';
 
 export interface StartParams extends Params {
   command: 'start';
   commandOptions: OriginalStartParams;
 }
 
-export interface StartResult extends Result {
-  noCache?: RunStats;
-  cache?: RunStats;
-  rebuild?: RunStats;
-}
-
 const runStartCommand = async (
   di: Container,
   {
@@ -29,17 +24,29 @@ const runStartCommand = async (
     shouldClearCache: boolean;
   }
 ): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
+  const clientBuildTimeSamples: number[] = [];
+  const serverBuildTimeSamples: number[] = [];
+  const clientMaxMemoryRssSamples: number[] = [];
+  const serverMaxMemoryRssSamples: number[] = [];
+
+  const serverCompilationTimings: CompilationStats[] = [];
+  const clientCompilationTimings: CompilationStats[] = [];
+
+  const maxMemoryRssSamples: number[] = [];
 
   const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as StartParams;
+  commandOptions.benchmark = true;
+
+  const buildType = commandOptions.buildType ?? 'all';
+
+  const benchmarkStartTime = new Date().toISOString();
 
   for (let i = 0; i < times; i++) {
     if (shouldClearCache) {
       await clearCacheDirectory(di);
     }
 
+    const attemptStartTime = Date.now();
     const { close, getBuildStats } = await (di
       .get(COMMAND_RUNNER_TOKEN)
       .run('start', commandOptions) as OriginalStartResult);
@@ -47,70 +54,67 @@ const runStartCommand = async (
 
     await close();
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
-  }
-
-  return {
-    clientSamples,
-    serverSamples,
-    maxMemoryRssSamples,
-  };
-};
-
-const runRebuild = async (di: Container, { times }: { times: number }): Promise<Samples> => {
-  const clientSamples: number[] = Array(times);
-  const serverSamples: number[] = Array(times);
-  const maxMemoryRssSamples: number[] = Array(times);
-  const { commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as StartParams;
+    // at first attempt do not save metrics if cache is used
+    if (!shouldClearCache && i === 0) {
+      continue;
+    }
 
-  const { close, invalidate, getBuildStats } = await (di
-    .get(COMMAND_RUNNER_TOKEN)
-    .run('start', commandOptions) as OriginalStartResult);
+    if (buildType === 'all' || buildType === 'server') {
+      serverCompilationTimings.push(
+        await getServerCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      serverBuildTimeSamples.push(stats.server.buildTime);
 
-  // warmup rebuild as it usually pretty slow at first runs
-  for (let i = 0; i < REBUILD_WARMUP_TIMES; i++) {
-    await invalidate();
-  }
+      if (stats.server.maxMemoryRss) {
+        serverMaxMemoryRssSamples.push(stats.server.maxMemoryRss);
+      }
+    }
 
-  for (let i = 0; i < times; i++) {
-    await invalidate();
+    if (buildType === 'all' || buildType === 'client') {
+      clientCompilationTimings.push(
+        await getClientCompilationTimings(benchmarkStartTime, attemptStartTime, i)
+      );
+      clientBuildTimeSamples.push(stats.client.buildTime);
 
-    const stats = getBuildStats();
+      if (stats.client.maxMemoryRss) {
+        clientMaxMemoryRssSamples.push(stats.client.maxMemoryRss);
+      }
+    }
 
-    clientSamples[i] = stats.clientBuildTime;
-    serverSamples[i] = stats.serverBuildTime;
-    maxMemoryRssSamples[i] = stats.maxMemoryRss;
+    if (stats.maxMemoryRss) {
+      maxMemoryRssSamples.push(stats.maxMemoryRss);
+    }
   }
 
-  await close();
-
   return {
-    clientSamples,
-    serverSamples,
+    serverCompilationTimings,
+    clientCompilationTimings,
+    clientMaxMemoryRssSamples,
+    serverMaxMemoryRssSamples,
+    clientBuildTimeSamples,
+    serverBuildTimeSamples,
     maxMemoryRssSamples,
   };
 };
 
-export const benchmarkStart = async (di: Container): Promise<StartResult> => {
-  const { times = 5 } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
-
-  const noCache = await runStartCommand(di, {
-    times: Math.max(Math.floor(times / 3), 2),
-    shouldClearCache: true,
-  });
-
-  const cache = await runStartCommand(di, {
-    times: Math.max(Math.floor(times / 2), 2),
-    shouldClearCache: false,
-  });
-
-  const rebuild = await runRebuild(di, { times });
+export const benchmarkStart = async (di: Container): Promise<Result> => {
+  const { times = DEFAULT_TIMES, commandOptions } = di.get(COMMAND_PARAMETERS_TOKEN) as Params;
+  const noCache = !commandOptions.fileCache;
+
+  let result;
+
+  if (noCache) {
+    result = await runStartCommand(di, {
+      times,
+      shouldClearCache: true,
+    });
+  } else {
+    result = await runStartCommand(di, {
+      // additional first attempt for cache warmup
+      times: times + 1,
+      shouldClearCache: false,
+    });
+  }
 
-  return {
-    cache: getResultStats(cache),
-    noCache: getResultStats(noCache),
-    rebuild: getResultStats(rebuild),
-  };
+  return getResultStats(result);
 };

package/src/api/benchmark/types.ts

@@ -1,19 +1,87 @@
-export interface Stats {
-  samples: number[];
-  mean: number;
-  std: number;
-
-  variance: number;
-}
-
 export type Samples = {
-  clientSamples: Stats['samples'];
-  serverSamples: Stats['samples'];
-  maxMemoryRssSamples: Stats['samples'];
+  serverCompilationTimings: CompilationStats[];
+  serverBuildTimeSamples: number[];
+  clientCompilationTimings: CompilationStats[];
+  clientBuildTimeSamples: number[];
+  maxMemoryRssSamples: number[];
+  clientMaxMemoryRssSamples: number[];
+  serverMaxMemoryRssSamples: number[];
+};
+
+export type CompilationStats = {
+  totalBuildCosts: Record<string, number>;
+  loaderBuildCosts: Record<string, number>;
+  pluginBuildCosts: Record<string, number>;
 };
 
 export type RunStats = {
-  client: Stats;
-  server: Stats;
-  maxMemoryRss: Stats;
+  serverCompilationStats: CompilationStats | undefined;
+  clientCompilationStats: CompilationStats | undefined;
+  clientBuildTime: number | undefined;
+  serverBuildTime: number | undefined;
+  clientMaxMemoryRss: number | undefined;
+  serverMaxMemoryRss: number | undefined;
+  maxMemoryRss: number | undefined;
+};
+
+type PlainObject<T = any> = {
+  [key: string]: T;
 };
+
+interface ProcessData {
+  /**
+   * process id
+   */
+  pid: number;
+  /**
+   * parent process id
+   */
+  ppid: number | null;
+}
+
+export interface LoaderTransformData extends ProcessData {
+  /** loader name */
+  loader: string;
+  /**
+   * loader index
+   */
+  loaderIndex: number;
+  /** loader path */
+  path: string;
+  input: string | null | undefined;
+  /**
+   * - isPitch: true: the result of loader.pitch()
+   * - isPitch: false: the code result of loader()
+   */
+  result: string | null | undefined;
+  /** Timestamp when called */
+  startAt: number;
+  endAt: number;
+  /** loader configuration */
+  options: PlainObject;
+  /** pitching loader */
+  isPitch: boolean;
+  /**
+   * is sync
+   */
+  sync: boolean;
+  /** Error during conversion */
+  // errors: DevToolErrorInstance[];
+  /** module layer */
+  layer?: string;
+}
+
+export type MinimalLoaderData = Pick<LoaderTransformData, 'startAt' | 'endAt' | 'pid' | 'loader'>;
+
+export interface PluginData {
+  /** hook tap name */
+  tapName: string;
+  /** hook call time-consuming */
+  costs: number;
+  startAt: number;
+  endAt: number;
+  /** hook function type */
+  type: 'sync' | 'async' | 'promise';
+  /** hook function result */
+  result: any;
+}

package/src/api/benchmark/utils/compilationUtils.ts

@@ -0,0 +1,213 @@
+import fs from 'node:fs/promises';
+
+import { CompilationStats, LoaderTransformData, MinimalLoaderData, PluginData } from '../types';
+
+export async function getServerCompilationTimings(
+  benchmarkStartTime: string,
+  attemptStartTime: number,
+  attempt: number
+) {
+  return getCompilationTimings(benchmarkStartTime, attemptStartTime, attempt, 'server');
+}
+
+export async function getClientCompilationTimings(
+  benchmarkStartTime: string,
+  attemptStartTime: number,
+  attempt: number
+) {
+  return getCompilationTimings(benchmarkStartTime, attemptStartTime, attempt, 'client');
+}
+
+async function getCompilationTimings(
+  benchmarkStartTime: string,
+  attemptStartTime: number,
+  attempt: number,
+  type: 'client' | 'server'
+): Promise<CompilationStats> {
+  try {
+    const rsdoctorData = await getReportData(
+      await processReport(benchmarkStartTime, attempt, type)
+    );
+
+    const { summary, loader, plugin } = rsdoctorData;
+
+    const totalBuildCosts = calculateTotalCosts(summary, attemptStartTime);
+    const loaderBuildCosts = calculateLoadersCosts(loader);
+    const pluginBuildCosts = calculatePluginsCosts(plugin);
+
+    return { totalBuildCosts, loaderBuildCosts, pluginBuildCosts };
+  } catch (err) {
+    throw new Error(`Failed to get rsdoctor report!\n${err}`);
+  }
+}
+
+async function getReportData(reportPath: string) {
+  const rsdoctorRawStats = await fs.readFile(reportPath, 'utf-8');
+  const rsdoctorStats = JSON.parse(rsdoctorRawStats);
+
+  return rsdoctorStats.data;
+}
+
+async function processReport(
+  benchmarkStartTime: string,
+  attempt: number,
+  type: 'client' | 'server'
+) {
+  const reportBasePath = `./.rsdoctor/${benchmarkStartTime}`;
+  const reportPath = `${reportBasePath}/${type}-rsdoctor-data-${attempt + 1}.json`;
+
+  await fs.mkdir(reportBasePath, { recursive: true });
+  await fs.rename(`./.rsdoctor/${type}-rsdoctor-data.json`, reportPath);
+
+  return reportPath;
+}
+
+function mergeIntervals(intervals: [number, number][]) {
+  // Sort from small to large
+  intervals.sort((a, b) => a[0] - b[0]);
+  // The previous interval, the next interval, store the result
+  let previous;
+  let current;
+  const result: [number, number][] = [];
+
+  for (let i = 0; i < intervals.length; i++) {
+    current = intervals[i];
+    // If the first interval or the current interval does not overlap with the previous interval, add the current interval to the result
+    if (!previous || current[0] > previous[1]) {
+      // Assign the current interval to the previous interval
+      previous = current;
+      result.push(current);
+    } else {
+      // Otherwise, the two intervals overlap
+      // Update the end time of the previous interval
+      previous[1] = Math.max(previous[1], current[1]);
+    }
+  }
+
+  return result;
+}
+
+function getLoadersCosts(
+  filter: (loader: MinimalLoaderData) => boolean,
+  loaders: MinimalLoaderData[]
+) {
+  const match: { [pid: number | string]: [start: number, end: number][] } = {};
+  const others: { [pid: number | string]: [start: number, end: number][] } = {};
+
+  loaders.forEach((e) => {
+    if (filter(e)) {
+      if (!match[e.pid]) match[e.pid] = [];
+      match[e.pid].push([e.startAt, e.endAt]);
+    } else {
+      if (!others[e.pid]) others[e.pid] = [];
+      others[e.pid].push([e.startAt, e.endAt]);
+    }
+  });
+
+  let costs = 0;
+
+  const pids = Object.keys(match);
+
+  for (let i = 0; i < pids.length; i++) {
+    const pid = pids[i];
+    const _match = mergeIntervals(match[pid]);
+    // between in loader.startAt and loader.endAt
+    const _others = mergeIntervals(others[pid] || []).filter(([s, e]) =>
+      _match.some((el) => s >= el[0] && e <= el[1])
+    );
+
+    // eslint-disable-next-line no-param-reassign
+    const matchSum = _match.length ? _match.reduce((t, c) => (t += c[1] - c[0]), 0) : 0;
+    // eslint-disable-next-line no-param-reassign
+    const othersSum = _others.length ? _others.reduce((t, c) => (t += c[1] - c[0]), 0) : 0;
+
+    costs += matchSum - othersSum;
+  }
+
+  return costs;
+}
+
+function calculateTotalCosts(
+  summary: { costs: { name: string; cost: string }[] },
+  attemptStartTime: number
+) {
+  const { costs: buildCosts } = summary;
+  const buildCostsMap = buildCosts.reduce((acc, cost) => {
+    acc[cost.name] = cost;
+    return acc;
+  }, {});
+
+  const result: Record<string, number> = {};
+
+  for (const buildCostName in buildCostsMap) {
+    if (buildCostName === 'bootstrap->beforeCompile') {
+      // start of bootstrap is start of first build, so use attemptStarTime instead
+      result[buildCostName] =
+        buildCostsMap['beforeCompile->afterCompile'].startAt - attemptStartTime;
+    } else {
+      result[buildCostName] = buildCostsMap[buildCostName].costs;
+    }
+  }
+
+  return result;
+}
+
+function calculateLoadersCosts(loaders: { loaders: LoaderTransformData[] }[]) {
+  const filteredLoaders: MinimalLoaderData[] = [];
+  const uniqueLoaders = new Map<
+    string,
+    {
+      files: number;
+      path: string;
+    }
+  >();
+
+  loaders.forEach((data) => {
+    data.loaders.forEach((fl) => {
+      const uniqueLoader = uniqueLoaders.get(fl.loader);
+      if (uniqueLoader) {
+        uniqueLoaders.set(fl.loader, {
+          files: uniqueLoader.files + 1,
+          path: fl.path,
+        });
+      } else {
+        uniqueLoaders.set(fl.loader, { files: 1, path: fl.path });
+      }
+
+      return filteredLoaders.push({
+        loader: fl.loader,
+        startAt: fl.startAt,
+        endAt: fl.endAt,
+        pid: fl.pid,
+      });
+    });
+  });
+
+  const costs: Record<string, number> = {};
+  uniqueLoaders.forEach((_, loaderName: string) => {
+    costs[loaderName] = getLoadersCosts((item) => item.loader === loaderName, filteredLoaders);
+  });
+
+  return costs;
+}
+
+function calculatePluginsCosts(plugins: PluginData) {
+  const pluginCosts: Record<string, number> = {};
+
+  for (const hookName in plugins) {
+    const hookCalls = plugins[hookName];
+
+    hookCalls.forEach((plugin) => {
+      const pluginName = plugin.tapName;
+      const pluginCost = plugin.costs;
+
+      if (pluginCosts[pluginName]) {
+        pluginCosts[pluginName] += pluginCost;
+      } else {
+        pluginCosts[pluginName] = pluginCost;
+      }
+    });
+  }
+
+  return pluginCosts;
+}

package/src/api/benchmark/utils/stats.ts

@@ -1,45 +1,62 @@
-import type { RunStats, Stats } from '../types';
+import type { RunStats, Samples, CompilationStats } from '../types';
 
-export const getSamplesStats = (samples: number[]): Stats => {
-  const n = samples.length;
+function getMeanValue(samples: number[]) {
+  if (samples.length === 0) {
+    return;
+  }
 
   let sum = 0;
 
-  for (let i = 0; i < n; i++) {
+  for (let i = 0; i < samples.length; i++) {
     sum += samples[i];
   }
 
-  const mean = sum / n;
+  return sum / samples.length;
+}
 
-  sum = 0;
+function getCompilationMeanStats(allStats) {
+  const result = {} as CompilationStats;
 
-  for (let i = 0; i < n; i++) {
-    sum += (samples[i] - mean) * (samples[i] - mean);
-  }
+  const stats: CompilationStats = allStats[0];
 
-  const std = Math.sqrt(sum / n);
-  const variance = (100 * std) / mean;
+  for (const section in stats) {
+    for (const metricName in stats[section]) {
+      const samples = allStats.reduce((acc, item) => {
+        acc.push(item[section][metricName]);
+        return acc;
+      }, []);
 
-  return {
-    samples,
-    mean,
-    std,
-    variance,
-  };
-};
+      if (!result[section]) {
+        result[section] = {};
+      }
+      const meanValue = getMeanValue(samples);
+
+      // ignore values less than 10ms
+      if (meanValue > 10) {
+        result[section][metricName] = meanValue;
+      }
+    }
+  }
+
+  return result;
+}
 
 export const getResultStats = ({
-  clientSamples,
-  serverSamples,
+  clientBuildTimeSamples,
+  clientCompilationTimings,
+  clientMaxMemoryRssSamples,
+  serverBuildTimeSamples,
+  serverCompilationTimings,
+  serverMaxMemoryRssSamples,
   maxMemoryRssSamples,
-}: {
-  clientSamples: number[];
-  serverSamples: number[];
-  maxMemoryRssSamples: number[];
-}): RunStats => {
+}: Samples): RunStats => {
   return {
-    client: getSamplesStats(clientSamples),
-    server: getSamplesStats(serverSamples),
-    maxMemoryRss: getSamplesStats(maxMemoryRssSamples),
+    serverBuildTime: getMeanValue(serverBuildTimeSamples),
+    serverCompilationStats: getCompilationMeanStats(serverCompilationTimings),
+    clientBuildTime: getMeanValue(clientBuildTimeSamples),
+    clientCompilationStats: getCompilationMeanStats(clientCompilationTimings),
+    maxMemoryRss: getMeanValue(maxMemoryRssSamples),
+    clientMaxMemoryRss: getMeanValue(clientMaxMemoryRssSamples),
+    serverMaxMemoryRss: getMeanValue(serverMaxMemoryRssSamples),
   };
 };

package/src/api/build/index.ts

@@ -21,6 +21,7 @@ export type Params = WithConfig<{
   env?: Record<string, string>;
   fileCache?: boolean;
   withBuildStats?: boolean;
+  benchmark?: boolean;
   withModulesStats?: boolean;
 
   // `package` target parameters

package/src/api/start/application.experimental.ts

@@ -24,6 +24,7 @@ export const startApplicationExperimental = async (di: Container): Result => {
   const inputParameters: StartParameters = {
     name: configEntry.name,
     mode: 'development',
+    benchmark: options.benchmark,
     buildType: options.buildType,
     port: options.port,
     host: options.host,
@@ -101,7 +102,7 @@ export const startApplicationExperimental = async (di: Container): Result => {
       await devServer.invalidate();
     },
     getBuildStats: () => {
-      return {};
+      return devServer.getStats();
     },
     builder: {
       name: '@tramvai/plugin-webpack-builder',
@@ -114,7 +115,7 @@ export const startApplicationExperimental = async (di: Container): Result => {
             await devServer.invalidate();
           },
           getBuildStats: () => {
-            return {};
+            return devServer.getStats();
           },
         };
       },
@@ -210,7 +211,7 @@ function mapTramvaiJsonToNewTsConfig({ rootDir }: { rootDir: string }) {
       }
       if (applicationProject.polyfill) {
         try {
-          if (require.resolve(mappedProject.polyfill).includes('node_modules')) {
+          if (require.resolve(applicationProject.polyfill).includes('node_modules')) {
             mappedProject.polyfill = require.resolve(applicationProject.polyfill);
           } else {
             mappedProject.polyfill = path.resolve(applicationProject.polyfill);

package/src/api/start/index.ts

@@ -24,6 +24,7 @@ export type Params = WithConfig<{
   noClientRebuild?: boolean;
   resolveSymlinks?: boolean;
   showConfig?: boolean;
+  benchmark?: boolean;
   withBuildStats?: boolean;
   // @todo: not working?
   env?: Record<string, string>;

package/src/builder/webpack/analyzePlugins/rsdoctor.ts

@@ -1,26 +1,14 @@
-import type Config from 'webpack-chain';
 import { RsdoctorWebpackPlugin } from '@rsdoctor/webpack-plugin';
 import { AnalyzePlugin } from '../types';
 
+type options = ConstructorParameters<typeof RsdoctorWebpackPlugin<[]>>;
 export class RsdoctorAnalyzePlugin extends AnalyzePlugin {
   requireDeps = [];
 
-  options: [RsdoctorWebpackPlugin<[]>['options']] = [
-    {
-      features: ['loader', 'plugins', 'resolver'],
-    },
-  ];
+  options: options = [];
 
   plugin = RsdoctorWebpackPlugin;
 
-  patchConfig = (config: Config): Config => {
-    super.patchConfigInternal(config);
-    // https://github.com/web-infra-dev/rsdoctor/issues/717
-    config.set('experiments', { backCompat: true });
-
-    return config;
-  };
-
   // rsdoctor поднимает dev server
   afterBuild = () => new Promise(() => null);
 }