i18n-tasks 1.1.2 → 1.2.0

data/benchmarks/fixtures/generator.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "yaml"
+
+# Generates synthetic but realistic i18n-tasks fixtures at three scales.
+#
+# Each scale produces:
+#   - config/locales/<locale>.yml for each locale
+#   - app/controllers/*.rb with I18n.t() calls
+#   - app/views/**/*.html.erb with t() calls (relative and absolute)
+#   - app/helpers/*.rb with mixed calls
+#
+# ~10% of keys in base locale are intentionally left out of other locales (missing).
+# ~10% of defined keys are never referenced in source (unused).
+module BenchmarkFixtures
+  SCALES = {
+    small: {num_keys: 200, num_rb_files: 15, num_erb_files: 15, num_locales: 2},
+    medium: {num_keys: 2_000, num_rb_files: 100, num_erb_files: 100, num_locales: 5},
+    large: {num_keys: 8_000, num_rb_files: 300, num_erb_files: 300, num_locales: 8}
+  }.freeze
+
+  LOCALES = %w[en es fr de ja zh pt it ko nl].freeze
+
+  CONTROLLERS = %w[users posts comments orders products invoices sessions registrations
+    profiles settings dashboards reports notifications search tags categories].freeze
+
+  HELPERS = %w[application_helper users_helper posts_helper form_helper date_helper].freeze
+
+  # Key sections that mirror realistic Rails apps
+  SECTIONS = %w[
+    users.index users.show users.new users.edit users.form
+    posts.index posts.show posts.create posts.update posts.destroy
+    comments.index comments.new comments.form
+    orders.index orders.show orders.confirm orders.complete
+    shared.errors shared.validations shared.navigation shared.buttons shared.labels
+    mailers.welcome mailers.reset_password mailers.confirmation mailers.notification
+    admin.users.index admin.users.show admin.posts.index admin.dashboard
+    errors.not_found errors.unauthorized errors.server_error
+    auth.login auth.logout auth.register auth.forgot_password
+  ].freeze
+
+  class << self
+    def root
+      File.expand_path("../..", __dir__)
+    end
+
+    def generated_root
+      File.join(root, "benchmarks", "fixtures", "generated")
+    end
+
+    # Generate all fixtures.
+    # @param force [Boolean] regenerate even if files already exist
+    def generate_all(force: false)
+      SCALES.each_key do |scale|
+        generate(scale, force: force)
+      end
+    end
+
+    # Generate fixtures for a single scale.
+    # @param scale [:small, :medium, :large]
+    # @param force [Boolean] regenerate even if files already exist
+    # @return [String] path to the generated fixture directory
+    def generate(scale, force: false)
+      config = SCALES.fetch(scale)
+      dir = File.join(generated_root, scale.to_s)
+
+      sentinel = File.join(dir, ".generated")
+      if !force && File.exist?(sentinel)
+        return dir
+      end
+
+      FileUtils.rm_rf(dir)
+      FileUtils.mkdir_p(dir)
+
+      locales = LOCALES.first(config[:num_locales])
+      base_locale = locales.first
+      keys = generate_keys(config[:num_keys])
+
+      write_config_file(dir, locales, base_locale)
+      write_locale_files(dir, locales, base_locale, keys)
+      write_ruby_files(dir, keys, config[:num_rb_files])
+      write_erb_files(dir, keys, config[:num_erb_files], base_locale)
+
+      FileUtils.touch(sentinel)
+      dir
+    end
+
+    private
+
+    # Build a flat list of dot-separated keys distributed across realistic sections.
+    def generate_keys(count)
+      keys = []
+      per_section = [count / SECTIONS.size, 1].max
+      SECTIONS.cycle do |section|
+        break if keys.size >= count
+
+        per_section.times do |i|
+          break if keys.size >= count
+
+          keys << "#{section}.key_#{i + 1}"
+        end
+      end
+      keys.first(count)
+    end
+
+    def write_config_file(dir, locales, base_locale)
+      FileUtils.mkdir_p(File.join(dir, "config"))
+
+      config = {
+        "base_locale" => base_locale,
+        "locales" => locales,
+        "data" => {
+          "read" => ["config/locales/%{locale}.yml"],
+          "write" => ["config/locales/%{locale}.yml"]
+        },
+        "search" => {
+          "paths" => ["app/"],
+          "relative_roots" => ["app/controllers", "app/helpers", "app/views"]
+        }
+      }
+
+      File.write(File.join(dir, "config", "i18n-tasks.yml"), YAML.dump(config))
+    end
+
+    def write_locale_files(dir, locales, base_locale, keys)
+      locale_dir = File.join(dir, "config", "locales")
+      FileUtils.mkdir_p(locale_dir)
+
+      locales.each do |locale|
+        # Skip ~10% of keys for non-base locales to simulate missing translations.
+        locale_keys = if locale == base_locale
+          keys
+        else
+          keys.reject.with_index { |_, i| (i % 10).zero? }
+        end
+
+        # Mark ~10% of keys as "unused" by never referencing them in source.
+        # We track unused indices and write them in the locale files but skip them in source.
+        tree = keys_to_nested_hash(locale, locale_keys) { |key, i| "#{locale}.#{key}.value_#{i}" }
+        File.write(File.join(locale_dir, "#{locale}.yml"), YAML.dump(tree))
+      end
+    end
+
+    def keys_to_nested_hash(locale, keys)
+      root = {locale => {}}
+      keys.each_with_index do |key, i|
+        parts = key.split(".")
+        node = root[locale]
+        parts[0..-2].each do |part|
+          node[part] ||= {}
+          node = node[part]
+        end
+        node[parts.last] = block_given? ? yield(key, i) : "#{locale}.#{key}"
+      end
+      root
+    end
+
+    # Write Ruby controller/helper files referencing ~90% of keys (leaving ~10% unused).
+    def write_ruby_files(dir, keys, num_files)
+      app_dir = File.join(dir, "app")
+      FileUtils.mkdir_p(File.join(app_dir, "controllers"))
+      FileUtils.mkdir_p(File.join(app_dir, "helpers"))
+
+      # Keys that will be referenced from Ruby (skip every 10th = ~10% unused in source)
+      source_keys = keys.reject.with_index { |_, i| (i % 10).zero? }
+      keys_per_file = [source_keys.size / num_files, 1].max
+
+      num_files.times do |n|
+        file_keys = source_keys.slice(n * keys_per_file, keys_per_file) || []
+        controller_name = "#{CONTROLLERS[n % CONTROLLERS.size]}_#{n}_controller"
+        content = generate_ruby_file_content(controller_name, file_keys)
+
+        if n < num_files / 2
+          File.write(File.join(app_dir, "controllers", "#{controller_name}.rb"), content)
+        else
+          helper_name = "helper_#{n}"
+          File.write(File.join(app_dir, "helpers", "#{helper_name}.rb"), content)
+        end
+      end
+    end
+
+    def generate_ruby_file_content(class_name, keys)
+      lines = ["class #{camelize(class_name)} < ApplicationController"]
+      lines << "  def index"
+      keys.each_slice(3).with_index do |slice, idx|
+        case idx % 4
+        when 0
+          lines << "    I18n.t(#{slice[0].inspect})"
+          lines << "    I18n.t(#{slice[1].inspect})" if slice[1]
+        when 1
+          scope_parts = slice[0].split(".")
+          lines << "    t(#{scope_parts.last.inspect}, scope: #{scope_parts[0..-2].inspect})"
+        when 2
+          lines << "    I18n.translate(#{slice[0].inspect})"
+        else
+          lines << "    t #{slice[0].inspect}"
+          lines << "    t #{slice[1].inspect}" if slice[1]
+        end
+        lines << "    t #{slice[2].inspect}" if slice[2]
+      end
+      lines << "  end"
+
+      # Add a method with dynamic key (exercises expr_key_re)
+      if keys.any?
+        section = keys.first.split(".").first(2).join(".")
+        lines << "  def show"
+        lines << "    # dynamic key - should not count as unused"
+        lines << "    t(\"#{section}.\#{some_dynamic_value}\")"
+        lines << "  end"
+      end
+
+      lines << "end"
+      lines.join("\n")
+    end
+
+    # Write ERB view files with relative t() calls and some absolute ones.
+    def write_erb_files(dir, keys, num_files, _base_locale)
+      views_dir = File.join(dir, "app", "views")
+
+      # Use only non-"unused" keys in views to keep ~10% unused
+      source_keys = keys.reject.with_index { |_, i| (i % 10).zero? }
+      keys_per_file = [source_keys.size / [num_files, 1].max, 1].max
+
+      num_files.times do |n|
+        file_keys = source_keys.slice(n * keys_per_file, keys_per_file) || []
+        section = file_keys.first&.split(".")&.first(2)&.join("/") || "shared"
+        FileUtils.mkdir_p(File.join(views_dir, section))
+
+        content = generate_erb_file_content(file_keys)
+        File.write(File.join(views_dir, section, "view_#{n}.html.erb"), content)
+      end
+    end
+
+    def generate_erb_file_content(keys)
+      lines = ["<div>"]
+      keys.each_with_index do |key, i|
+        case i % 3
+        when 0
+          lines << "  <%= t(#{key.inspect}) %>"
+        when 1
+          # relative key - use just the last segment
+          leaf = key.split(".").last
+          lines << "  <%= t(#{".#{leaf}".inspect}) %>"
+        else
+          lines << "  <%= I18n.t(#{key.inspect}) %>"
+        end
+      end
+      lines << "</div>"
+      lines.join("\n")
+    end
+
+    def camelize(str)
+      str.split(/[_\/]/).map(&:capitalize).join
+    end
+  end
+end

data/benchmarks/results/baseline.json

@@ -0,0 +1,250 @@
+{
+  "scanning/small": {
+    "RubyScanner/parser (small)": {
+      "ips": 85.14,
+      "ips_sd": 1,
+      "microseconds": 11745.33
+    },
+    "RubyScanner/prism  (small)": {
+      "ips": 520.55,
+      "ips_sd": 5,
+      "microseconds": 1921.04
+    },
+    "ErbAstScanner      (small)": {
+      "ips": 80.25,
+      "ips_sd": 1,
+      "microseconds": 12461.32
+    },
+    "PatternScanner     (small)": {
+      "ips": 366.24,
+      "ips_sd": 19,
+      "microseconds": 2730.47
+    }
+  },
+  "scanning/medium": {
+    "RubyScanner/parser (medium)": {
+      "ips": 9.34,
+      "ips_sd": 0,
+      "microseconds": 107064.63
+    },
+    "RubyScanner/prism  (medium)": {
+      "ips": 53.15,
+      "ips_sd": 2,
+      "microseconds": 18816.42
+    },
+    "ErbAstScanner      (medium)": {
+      "ips": 7.82,
+      "ips_sd": 0,
+      "microseconds": 127814.44
+    },
+    "PatternScanner     (medium)": {
+      "ips": 31.25,
+      "ips_sd": 1,
+      "microseconds": 32003.1
+    }
+  },
+  "tree/construction": {
+    "from_key_names (200 keys)": {
+      "ips": 336.11,
+      "ips_sd": 5,
+      "microseconds": 2975.22
+    },
+    "from_key_names (2k keys)": {
+      "ips": 28.24,
+      "ips_sd": 2,
+      "microseconds": 35410.79
+    },
+    "from_key_names (8k keys)": {
+      "ips": 6.51,
+      "ips_sd": 0,
+      "microseconds": 153567.77
+    },
+    "from_nested_hash (200 keys)": {
+      "ips": 286.18,
+      "ips_sd": 7,
+      "microseconds": 3494.35
+    },
+    "from_nested_hash (2k keys)": {
+      "ips": 23.43,
+      "ips_sd": 2,
+      "microseconds": 42674.17
+    }
+  },
+  "tree/merge": {
+    "merge! (200 into 200 keys)": {
+      "ips": 963.35,
+      "ips_sd": 23,
+      "microseconds": 1038.04
+    },
+    "merge! (2k into 2k keys)": {
+      "ips": 96.47,
+      "ips_sd": 3,
+      "microseconds": 10365.85
+    }
+  },
+  "tree/traversal": {
+    "leaves (2k keys)": {
+      "ips": 409.96,
+      "ips_sd": 5,
+      "microseconds": 2439.29
+    },
+    "key_names (2k keys)": {
+      "ips": 382.42,
+      "ips_sd": 7,
+      "microseconds": 2614.92
+    },
+    "key_names (8k keys)": {
+      "ips": 92.16,
+      "ips_sd": 4,
+      "microseconds": 10850.73
+    },
+    "select_keys (2k, match half)": {
+      "ips": 53.54,
+      "ips_sd": 1,
+      "microseconds": 18678.8
+    },
+    "nodes block iteration (2k keys)": {
+      "ips": 450.94,
+      "ips_sd": 5,
+      "microseconds": 2217.57
+    }
+  },
+  "tree/subtract": {
+    "subtract_by_key! (200 keys)": {
+      "ips": 374.7,
+      "ips_sd": 6,
+      "microseconds": 2668.77
+    },
+    "subtract_by_key! (2k keys)": {
+      "ips": 35.83,
+      "ips_sd": 1,
+      "microseconds": 27912.64
+    }
+  },
+  "data/yaml_parse": {
+    "YAML parse small  (~9KB)": {
+      "ips": 1933.54,
+      "ips_sd": 54,
+      "microseconds": 517.19
+    },
+    "YAML parse medium (~97KB)": {
+      "ips": 223.54,
+      "ips_sd": 5,
+      "microseconds": 4473.41
+    },
+    "YAML parse large  (~400KB)": {
+      "ips": 56.35,
+      "ips_sd": 2,
+      "microseconds": 17747.21
+    }
+  },
+  "data/yaml_dump": {
+    "YAML dump small  tree": {
+      "ips": 1366.38,
+      "ips_sd": 33,
+      "microseconds": 731.86
+    },
+    "YAML dump medium tree": {
+      "ips": 153.11,
+      "ips_sd": 3,
+      "microseconds": 6531.45
+    },
+    "YAML dump large  tree": {
+      "ips": 37.9,
+      "ips_sd": 1,
+      "microseconds": 26386.17
+    }
+  },
+  "data/tree_load": {
+    "from_nested_hash small": {
+      "ips": 1325.81,
+      "ips_sd": 55,
+      "microseconds": 754.26
+    },
+    "from_nested_hash medium": {
+      "ips": 190.78,
+      "ips_sd": 7,
+      "microseconds": 5241.64
+    },
+    "from_nested_hash large": {
+      "ips": 44.98,
+      "ips_sd": 2,
+      "microseconds": 22232.79
+    }
+  },
+  "data/full_load": {
+    "data['en'] load (small)": {
+      "ips": 557.68,
+      "ips_sd": 18,
+      "microseconds": 1793.15
+    },
+    "data['en'] load (medium)": {
+      "ips": 80.26,
+      "ips_sd": 2,
+      "microseconds": 12459.21
+    }
+  },
+  "end_to_end/small": {
+    "used_tree     (small)": {
+      "ips": 21.36,
+      "ips_sd": 1,
+      "microseconds": 46813.31
+    },
+    "missing_keys  (small)": {
+      "ips": 15.77,
+      "ips_sd": 1,
+      "microseconds": 63412.33
+    },
+    "unused_keys   (small)": {
+      "ips": 11.4,
+      "ips_sd": 1,
+      "microseconds": 87687.22
+    }
+  },
+  "end_to_end/medium": {
+    "used_tree     (medium)": {
+      "ips": 2.39,
+      "ips_sd": 0,
+      "microseconds": 419130.05
+    },
+    "missing_keys  (medium)": {
+      "ips": 1.14,
+      "ips_sd": 0,
+      "microseconds": 877165.79
+    },
+    "unused_keys   (medium)": {
+      "ips": 1.05,
+      "ips_sd": 0,
+      "microseconds": 948448.83
+    }
+  },
+  "end_to_end/large": {
+    "used_tree     (large)": {
+      "ips": 0.54,
+      "ips_sd": 0,
+      "microseconds": 1856802.63
+    },
+    "missing_keys  (large)": {
+      "ips": 0.18,
+      "ips_sd": 0,
+      "microseconds": 5652702.62
+    },
+    "unused_keys   (large)": {
+      "ips": 0.18,
+      "ips_sd": 0,
+      "microseconds": 5463508.26
+    }
+  },
+  "end_to_end/backend_comparison": {
+    "used_tree — parser AST (medium)": {
+      "ips": 2.4,
+      "ips_sd": 0,
+      "microseconds": 415864.89
+    },
+    "used_tree — prism (medium)": {
+      "ips": 3.1,
+      "ips_sd": 0,
+      "microseconds": 322536.32
+    }
+  }
+}

data/benchmarks/run_all.rb

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rbconfig"
+
+# Runs all benchmarks and optionally saves / compares against a baseline.
+#
+# Usage:
+#   bundle exec ruby benchmarks/run_all.rb              # run everything
+#   bundle exec ruby benchmarks/run_all.rb --save       # run + save as new baseline
+#   bundle exec ruby benchmarks/run_all.rb --compare    # run + compare against baseline (exits 1 on regression)
+#   bundle exec ruby benchmarks/run_all.rb --memory     # include memory profiles
+#   bundle exec ruby benchmarks/run_all.rb --only=tree  # run only tree benchmarks
+#
+# The --only flag accepts comma-separated values: tree, data, scanning, e2e
+
+bench_dir = File.dirname(__FILE__)
+
+save = ARGV.include?("--save")
+compare = ARGV.include?("--compare")
+memory = ARGV.include?("--memory")
+
+only_arg = ARGV.grep(/\A--only=/).first&.sub("--only=", "")
+only_filter = only_arg&.split(",") || []
+
+all_benches = {
+  "scanning" => File.join(bench_dir, "scanning_bench.rb"),
+  "tree" => File.join(bench_dir, "tree_bench.rb"),
+  "data" => File.join(bench_dir, "data_bench.rb"),
+  "e2e" => File.join(bench_dir, "end_to_end_bench.rb")
+}
+
+if only_filter.any?
+  unknown = only_filter - all_benches.keys
+  if unknown.any?
+    warn "Unknown --only values: #{unknown.join(", ")}. Available: #{all_benches.keys.join(", ")}"
+    exit 1
+  end
+  benches = all_benches.slice(*only_filter)
+else
+  benches = all_benches
+end
+
+flags = []
+flags << "--save" if save
+flags << "--compare" if compare
+flags << "--memory" if memory
+
+failed = false
+benches.each do |name, path|
+  puts
+  puts "-" * 70
+  puts "  Running: #{name}"
+  puts "-" * 70
+
+  result = system(RbConfig.ruby, path, *flags)
+  failed = true unless result
+end
+
+exit(failed ? 1 : 0)

data/benchmarks/scanning_bench.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+# Benchmarks for the source-code scanning hot path.
+#
+# Measures per-scanner throughput (IPS) for Ruby (parser AST and Prism backends)
+# and ERB scanning across fixture scales.
+#
+# Usage:
+#   bundle exec ruby benchmarks/scanning_bench.rb
+#   bundle exec ruby benchmarks/scanning_bench.rb --save    # save as new baseline
+#   bundle exec ruby benchmarks/scanning_bench.rb --compare # compare against baseline
+#   bundle exec ruby benchmarks/scanning_bench.rb --memory  # include memory profiles
+
+require_relative "bench_helper"
+
+require "i18n/tasks/scanners/ruby_scanner"
+require "i18n/tasks/scanners/erb_ast_scanner"
+require "i18n/tasks/scanners/pattern_with_scope_scanner"
+require "i18n/tasks/scanners/files/caching_file_finder_provider"
+require "i18n/tasks/scanners/files/caching_file_reader"
+
+SAVE_RESULTS = ARGV.include?("--save")
+COMPARE_RESULTS = ARGV.include?("--compare")
+MEMORY_PROFILE = ARGV.include?("--memory")
+
+def build_scanner(scanner_class, app_dir, only_pattern:, shared_provider:, shared_reader:, **extra_config)
+  config = {
+    paths: [app_dir],
+    only: Array(only_pattern),
+    exclude: [],
+    relative_roots: [
+      File.join(app_dir, "controllers"),
+      File.join(app_dir, "helpers"),
+      File.join(app_dir, "views")
+    ],
+    relative_exclude_method_name_paths: [],
+    **extra_config
+  }
+  scanner_class.new(
+    config: config,
+    file_finder_provider: shared_provider,
+    file_reader: shared_reader
+  )
+end
+
+all_suites = []
+
+[:small, :medium].each do |scale|
+  dir = BenchmarkFixtures.generate(scale)
+  app_dir = File.join(dir, "app")
+
+  BenchHelper.header("Scanning — #{scale} fixture")
+
+  # Build shared provider/reader and warm their caches for every pattern used below,
+  # so the timed block measures only parse throughput, not file-system traversal or I/O.
+  shared_provider = I18n::Tasks::Scanners::Files::CachingFileFinderProvider.new
+  shared_reader = I18n::Tasks::Scanners::Files::CachingFileReader.new
+  build_scanner(I18n::Tasks::Scanners::RubyScanner, app_dir,
+    only_pattern: ["*.rb"], shared_provider: shared_provider,
+    shared_reader: shared_reader, prism: nil).keys
+  build_scanner(I18n::Tasks::Scanners::RubyScanner, app_dir,
+    only_pattern: ["*.rb"], shared_provider: shared_provider,
+    shared_reader: shared_reader, prism: "rails").keys
+  build_scanner(I18n::Tasks::Scanners::ErbAstScanner, app_dir,
+    only_pattern: ["*.erb"], shared_provider: shared_provider,
+    shared_reader: shared_reader).keys
+  build_scanner(I18n::Tasks::Scanners::PatternWithScopeScanner, app_dir,
+    only_pattern: ["*.rb", "*.erb"], shared_provider: shared_provider,
+    shared_reader: shared_reader).keys
+
+  suite = Benchmark.ips do |x|
+    x.config(warmup: 3, time: 10)
+
+    x.report("RubyScanner/parser (#{scale})") do
+      build_scanner(I18n::Tasks::Scanners::RubyScanner, app_dir,
+        only_pattern: ["*.rb"], shared_provider: shared_provider,
+        shared_reader: shared_reader, prism: nil).keys
+    end
+
+    x.report("RubyScanner/prism  (#{scale})") do
+      build_scanner(I18n::Tasks::Scanners::RubyScanner, app_dir,
+        only_pattern: ["*.rb"], shared_provider: shared_provider,
+        shared_reader: shared_reader, prism: "rails").keys
+    end
+
+    x.report("ErbAstScanner      (#{scale})") do
+      build_scanner(I18n::Tasks::Scanners::ErbAstScanner, app_dir,
+        only_pattern: ["*.erb"], shared_provider: shared_provider,
+        shared_reader: shared_reader).keys
+    end
+
+    x.report("PatternScanner     (#{scale})") do
+      build_scanner(I18n::Tasks::Scanners::PatternWithScopeScanner, app_dir,
+        only_pattern: ["*.rb", "*.erb"], shared_provider: shared_provider,
+        shared_reader: shared_reader).keys
+    end
+
+    x.compare!
+  end
+
+  all_suites << [suite, "scanning/#{scale}"]
+end
+
+if MEMORY_PROFILE
+  dir = BenchmarkFixtures.generate(:medium)
+  app_dir = File.join(dir, "app")
+  shared_provider = I18n::Tasks::Scanners::Files::CachingFileFinderProvider.new
+  shared_reader = I18n::Tasks::Scanners::Files::CachingFileReader.new
+  # Warm caches so memory report reflects only parse allocations
+  build_scanner(I18n::Tasks::Scanners::RubyScanner, app_dir,
+    only_pattern: ["*.rb"], shared_provider: shared_provider,
+    shared_reader: shared_reader, prism: nil).keys
+
+  BenchHelper.header("Memory profile — RubyScanner/parser (medium)")
+  MemoryProfiler.report do
+    build_scanner(I18n::Tasks::Scanners::RubyScanner, app_dir,
+      only_pattern: ["*.rb"], shared_provider: shared_provider,
+      shared_reader: shared_reader, prism: nil).keys
+  end.pretty_print(scale_bytes: true, detailed_report: false)
+
+  BenchHelper.header("Memory profile — RubyScanner/prism (medium)")
+  MemoryProfiler.report do
+    build_scanner(I18n::Tasks::Scanners::RubyScanner, app_dir,
+      only_pattern: ["*.rb"], shared_provider: shared_provider,
+      shared_reader: shared_reader, prism: "rails").keys
+  end.pretty_print(scale_bytes: true, detailed_report: false)
+end
+
+passed = true
+all_suites.each do |(suite, label)|
+  BenchHelper.save_results(suite, label) if SAVE_RESULTS
+  passed = false if COMPARE_RESULTS && !BenchHelper.compare_baseline(suite, label)
+end
+
+exit(1) if COMPARE_RESULTS && !passed