gitlab-crystalball 0.7.1

data/lib/crystalball/rails/map_generator/action_view_strategy.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "crystalball/map_generator/base_strategy"
+require "crystalball/map_generator/helpers/path_filter"
+require "crystalball/rails/map_generator/action_view_strategy/patch"
+
+module Crystalball
+  module Rails
+    class MapGenerator
+      # Map generator strategy to build map of views affected by an example.
+      # It patches `ActionView::Template#compile!` to get original name of compiled views.
+      class ActionViewStrategy
+        include ::Crystalball::MapGenerator::BaseStrategy
+        include ::Crystalball::MapGenerator::Helpers::PathFilter
+
+        class << self
+          # List of views affected by current example
+          #
+          # @return [Array<String>]
+          attr_reader :views
+
+          # Reset cached list of views
+          def reset_views
+            @views = []
+          end
+        end
+
+        def after_start
+          Patch.apply!
+        end
+
+        def before_finalize
+          Patch.revert!
+        end
+
+        # Adds views related to the spec to the example group map
+        # @param [Crystalball::ExampleGroupMap] example_group_map - object holding example metadata and used files
+        def call(example_group_map, _)
+          self.class.reset_views
+          yield example_group_map
+          example_group_map.push(*filter(self.class.views))
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/map_generator/i18n_strategy/simple_patch.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "i18n"
+
+module Crystalball
+  module Rails
+    class MapGenerator
+      class I18nStrategy
+        # Module to add new patched `load_file`, `store_translations` and `lookup`
+        # methods to `I18n::Backend::Simple`.
+        module SimplePatch
+          class << self
+            # Patches `I18n::Backend::Simple`.
+            def apply!
+              ::I18n::Backend::Simple.class_eval do
+                include SimplePatch
+
+                %i[load_file store_translations lookup].each do |method|
+                  alias_method :"cb_original_#{method}", method
+                  alias_method method, :"cb_patched_#{method}"
+                end
+              end
+            end
+
+            # Reverts original behavior of `I18n::Backend::Simple`
+            def revert!
+              ::I18n::Backend::Simple.class_eval do
+                %i[load_file store_translations lookup].each do |method|
+                  alias_method method, :"cb_original_#{method}"
+                  undef_method :"cb_patched_#{method}"
+                end
+              end
+              ::I18n.reload!
+            end
+          end
+
+          # Will replace original `I18n::Backend::Simple#load_file`.
+          # Stores filename in current thread
+          def cb_patched_load_file(filename, *args)
+            Thread.current[:cb_locale_file_name] = filename
+            cb_original_load_file(filename, *args)
+          end
+
+          # Will replace original `I18n::Backend::Simple#store_translations`.
+          # Adds filename for each value
+          def cb_patched_store_translations(locale, data, *args)
+            cb_add_filename_to_values(data, Thread.current[:cb_locale_file_name])
+            cb_original_store_translations(locale, data, *args)
+          end
+
+          # Will replace original `I18n::Backend::Simple#lookup`.
+          # Records origin filename of each value used.
+          def cb_patched_lookup(*args)
+            value = cb_original_lookup(*args)
+            cb_remove_and_track_filename_from_values(value)
+          end
+
+          private
+
+          def cb_add_filename_to_values(data, filename)
+            data.each do |key, value|
+              case value
+              when Hash
+                next if value.frozen?
+
+                cb_add_filename_to_values(value, filename)
+              else
+                data[key] = {cb_filename: filename, cb_value: value}.freeze
+              end
+            end
+          end
+
+          def cb_remove_and_track_filename_from_values(data)
+            return data unless data.is_a?(Hash)
+
+            if data.key?(:cb_filename)
+              ::Crystalball::Rails::MapGenerator::I18nStrategy.locale_files << data[:cb_filename]
+              return data[:cb_value]
+            end
+
+            data.each.with_object({}) do |(key, value), collector|
+              collector[key] = cb_remove_and_track_filename_from_values(value)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/map_generator/i18n_strategy.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "crystalball/map_generator/base_strategy"
+require "crystalball/map_generator/helpers/path_filter"
+require "crystalball/rails/map_generator/i18n_strategy/simple_patch"
+
+module Crystalball
+  module Rails
+    class MapGenerator
+      # Map generator strategy to build map of locale files used by an example.
+      class I18nStrategy
+        include ::Crystalball::MapGenerator::BaseStrategy
+        include ::Crystalball::MapGenerator::Helpers::PathFilter
+
+        class << self
+          # List of locale files affected by current example
+          #
+          # @return [Array<String>]
+          def locale_files
+            @locale_files ||= []
+          end
+
+          # Reset cached list of locale files
+          def reset_locale_files
+            @locale_files = []
+          end
+        end
+
+        def after_register
+          SimplePatch.apply!
+        end
+
+        def before_finalize
+          SimplePatch.revert!
+        end
+
+        # Adds to the example group map the locale files used by the example
+        # @param [Crystalball::ExampleGroupMap] example_group_map - object holding example metadata and used files
+        def call(example_group_map, _)
+          self.class.reset_locale_files
+          yield example_group_map
+          example_group_map.push(*filter(self.class.locale_files.compact))
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/predictor/modified_schema.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "crystalball/rails/helpers/schema_definition_parser"
+require "crystalball/predictor/helpers/affected_example_groups_detector"
+
+module Crystalball
+  module Rails
+    class Predictor
+      # Used with `predictor.use Crystalball::Rails::Predictor::ModifiedSchema.new(tables_map_path:)`.
+      # When used will check db/schema.rb for changes and add specs which depend on files affected
+      # by changed tables
+      class ModifiedSchema
+        include ::Crystalball::Predictor::Helpers::AffectedExampleGroupsDetector
+        SCHEMA_PATH = "db/schema.rb"
+
+        attr_reader :tables_map_path
+
+        # @param [String] tables_map_path - path to generated TablesMap
+        def initialize(tables_map_path:)
+          @tables_map_path = tables_map_path
+        end
+
+        # @param [Crystalball::SourceDiff] diff - the diff from which to predict
+        #   which specs should run
+        # @param [Crystalball::ExecutionMap] map - the map with the relations of
+        #   examples and used files
+        # @return [Array<String>] the spec paths associated with the changes
+        def call(diff, map)
+          return [] if schema_diff(diff).nil?
+
+          old_schema = old_schema(diff)
+          new_schema = new_schema(diff)
+
+          changed_tables = changed_tables(old_schema, new_schema)
+
+          files = changed_tables.flat_map do |table_name|
+            files = tables_map[table_name]
+            Crystalball.log :warn, "There are no model files for changed table `#{table_name}`. Check https://github.com/toptal/crystalball#warning for detailed description" unless files&.any?
+            files
+          end.compact
+          detect_examples(files, map)
+        end
+
+        # @return [Crystalball::Rails::TablesMap]
+        def tables_map
+          @tables_map ||= MapStorage::YAMLStorage.load(Pathname(tables_map_path))
+        end
+
+        private
+
+        def schema_diff(diff)
+          diff.find { |file_diff| [SCHEMA_PATH, "./#{SCHEMA_PATH}"].include? file_diff.relative_path }
+        end
+
+        def old_schema(diff)
+          old_schema_contents = schema_content(diff.repository, diff.from)
+          Crystalball::Rails::Helpers::SchemaDefinitionParser.parse(old_schema_contents)
+        end
+
+        def new_schema(diff)
+          new_schema_contents = schema_content(diff.repository, diff.to)
+          Crystalball::Rails::Helpers::SchemaDefinitionParser.parse(new_schema_contents)
+        end
+
+        def schema_content(repository, revision)
+          if revision
+            repository.lib.show(revision, SCHEMA_PATH)
+          else
+            File.read(File.join(repository.dir.path, SCHEMA_PATH))
+          end
+        end
+
+        def changed_tables(schema1, schema2)
+          schema1.map do |table_name, body|
+            table_name if schema2[table_name] != body
+          end.compact
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/tables_map.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Crystalball
+  module Rails
+    # Storage for tables map
+    class TablesMap
+      extend Forwardable
+
+      # Simple data object for map metadata information
+      class Metadata
+        attr_reader :commit, :version
+
+        # @param [String] commit - SHA of commit
+        # @param [Numeric] version - map generator version number
+        def initialize(commit: nil, version: nil, **_)
+          @commit = commit
+          @version = version
+        end
+
+        def to_h
+          {type: TablesMap.name, commit: commit, version: version}
+        end
+      end
+
+      attr_reader :example_groups, :metadata
+
+      delegate %i[commit version] => :metadata
+      delegate %i[size [] []=] => :example_groups
+
+      # @param [Hash] metadata - add or override metadata of execution map
+      # @param [Hash] example_groups - initial list of tables
+      def initialize(metadata: {}, example_groups: {})
+        @metadata = Metadata.new(**metadata)
+        @example_groups = example_groups
+      end
+
+      # Remove all example_groups
+      def clear!
+        self.example_groups = {}
+      end
+
+      def add(files:, for_table:)
+        example_groups[for_table] ||= []
+        example_groups[for_table] += files
+        example_groups[for_table].uniq!
+      end
+
+      private
+
+      attr_writer :example_groups, :metadata
+    end
+  end
+end

data/lib/crystalball/rails/tables_map_generator/configuration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "crystalball/map_generator/object_sources_detector"
+
+module Crystalball
+  module Rails
+    class TablesMapGenerator
+      # Configuration of tables map generator. Is can be accessed as a first argument inside
+      # `Crystalball::Rails::TablesMapGenerator.start! { |config| config } block.
+      class Configuration
+        attr_writer :map_storage
+        attr_accessor :commit
+        attr_accessor :version
+        attr_writer :root_path
+        attr_writer :object_sources_detector
+
+        def map_storage_path
+          @map_storage_path ||= Pathname("tables_map.yml")
+        end
+
+        def map_storage_path=(value)
+          @map_storage_path = Pathname(value)
+        end
+
+        def map_storage
+          @map_storage ||= MapStorage::YAMLStorage.new(map_storage_path)
+        end
+
+        def root_path
+          @root_path ||= Dir.pwd
+        end
+
+        def object_sources_detector
+          @object_sources_detector ||= ::Crystalball::MapGenerator::ObjectSourcesDetector.new(root_path: root_path)
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/tables_map_generator.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "crystalball/rails/tables_map"
+require "crystalball/rails/tables_map_generator/configuration"
+
+module Crystalball
+  module Rails
+    # Class to generate tables to files map during RSpec build execution
+    class TablesMapGenerator
+      extend Forwardable
+
+      attr_reader :configuration
+      delegate %i[map_storage object_sources_detector] => :configuration
+
+      class << self
+        # Registers Crystalball handlers to generate execution map during specs execution
+        #
+        # @param [Proc] block to configure MapGenerator and Register strategies
+        def start!(&block)
+          generator = new(&block)
+
+          ::RSpec.configure do |c|
+            c.before(:suite) { generator.start! }
+            c.after(:suite) { generator.finalize! }
+          end
+        end
+      end
+
+      def initialize
+        @configuration = Configuration.new
+        @configuration.commit = repo.object("HEAD").sha if repo
+        yield @configuration if block_given?
+        object_sources_detector.after_register
+      end
+
+      # Prepares metadata for execution map
+      def start!
+        self.map = nil
+        map_storage.clear!
+
+        map_storage.dump(map.metadata.to_h)
+
+        self.started = true
+      end
+
+      # Finalizes and saves map
+      def finalize!
+        return unless started
+
+        collect_tables_info
+
+        object_sources_detector.before_finalize
+        map_storage.dump(map.example_groups) if map.size.positive?
+      end
+
+      # @return [Crystalball::Rails::TablesMap]
+      def map
+        @map ||= TablesMap.new(metadata: {commit: configuration.commit, version: configuration.version})
+      end
+
+      private
+
+      def repo
+        @repo = GitRepo.open(".") unless defined?(@repo)
+        @repo
+      end
+
+      def collect_tables_info
+        ActiveRecord::Base.descendants.each do |descendant|
+          table_name = descendant.table_name
+
+          next if table_name.nil?
+
+          files = object_sources_detector.detect([descendant])
+
+          map.add(files: files, for_table: table_name)
+        end
+      end
+
+      attr_writer :map
+      attr_accessor :started
+    end
+  end
+end

data/lib/crystalball/rails.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "crystalball/rails/map_generator/action_view_strategy"
+require "crystalball/rails/map_generator/i18n_strategy"
+require "crystalball/active_record"
+
+module Crystalball
+  # Module containting Rails-specific stuff for Crystalball
+  module Rails
+  end
+end

data/lib/crystalball/rspec/filtering.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Crystalball
+  module RSpec
+    # This class is meant to remove the example filtering options
+    # for example_groups when a prediction contains a file path and the same file
+    # example id.
+    #
+    # For example, if a prediction contains `./spec/foo_spec.rb[1:1] ./spec/foo_spec.rb`,
+    # only `./spec/foo_spec.rb[1:1]` would run, because of the way RSpec
+    # filters are designed.
+    #
+    # Therefore, we need to manually remove the filters from such example_groups.
+    class Filtering
+      # @param [RSpec::Core::Configuration] config
+      # @param [Array<String>] paths
+      def self.remove_unnecessary_filters(config, paths)
+        new(config).remove_unnecessary_filters(paths)
+      end
+
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      def remove_unnecessary_filters(files_or_directories)
+        directories, files = files_or_directories.partition { |f| File.directory?(f) }
+        remove_unecessary_filters_from_files(files)
+        remove_unecessary_filters_from_directories(directories)
+      end
+
+      def remove_unecessary_filters_from_directories(directories)
+        directories.each do |dir|
+          files = configuration.__send__(:gather_directories, dir)
+          remove_unecessary_filters_from_files(files)
+        end
+      end
+
+      def remove_unecessary_filters_from_files(files)
+        files.select { |f| ::RSpec::Core::Example.parse_id(f).last.nil? }.each do |file|
+          next remove_unecessary_filters(fd) if File.directory?(file)
+
+          path = ::RSpec::Core::Metadata.relative_path(File.expand_path(file))
+          configuration.filter_manager.inclusions[:ids]&.delete(path)
+        end
+      end
+
+      private
+
+      attr_reader :configuration
+    end
+  end
+end

data/lib/crystalball/rspec/prediction_builder.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Crystalball
+  module RSpec
+    # Class for building a prediction for RSpec runner.
+    # Accepts configuration hash and builds a prediction according to configuration.
+    class PredictionBuilder
+      attr_reader :config
+
+      def initialize(config = {})
+        @config = config
+      end
+
+      def prediction
+        predictor.prediction
+      end
+
+      def expired_map?
+        expiration_period = config["map_expiration_period"].to_i
+        return false unless expiration_period.positive?
+
+        execution_map.timestamp.to_i <= Time.now.to_i - config["map_expiration_period"]
+      end
+
+      def execution_map
+        @execution_map ||= Crystalball::MapStorage::YAMLStorage.load(config["execution_map_path"])
+      end
+
+      def repo
+        @repo ||= Crystalball::GitRepo.open(config["repo_path"])
+      end
+
+      private
+
+      # This method should be overridden in ancestor. Example:
+      #
+      # def predictor
+      #   super do |p|
+      #     p.use Crystalball::Predictor::ModifiedExecutionPaths.new
+      #     p.use Crystalball::Predictor::ModifiedSpecs.new
+      #   end
+      # end
+      #
+      def predictor(&block)
+        raise NotImplementedError, "Configure `prediction_builder_class_name` in `crystalball.yml` and override `predictor` method" unless block_given?
+
+        @predictor ||= Crystalball::Predictor.new(execution_map, repo, from: config["diff_from"], to: config["diff_to"], &block)
+      end
+    end
+  end
+end

data/lib/crystalball/rspec/prediction_pruning/examples_pruner.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Crystalball
+  module RSpec
+    module PredictionPruning
+      # A class to prune given world example groups to fit the limit.
+      class ExamplesPruner
+        # Simple data object for holding context ids array with total examples size
+        class ContextIdsSet
+          attr_reader :ids, :size
+          alias to_a ids
+
+          def initialize
+            @size = 0
+            @ids = []
+          end
+
+          def add(id, size = 1)
+            @size += size
+            @ids << id
+          end
+        end
+
+        attr_reader :world, :limit
+
+        # @param [RSpec::Core::World] rspec_world RSpec world instance
+        # @param [Integer] to upper bound limit for prediction.
+        def initialize(rspec_world, to:)
+          @world = rspec_world
+          @limit = to
+        end
+
+        # @return [Array<String>] set of example and context ids to run
+        def pruned_set
+          resulting_set = ContextIdsSet.new
+          world.ordered_example_groups.each { |g| prune_to_limit(g, resulting_set) }
+          resulting_set.to_a
+        end
+
+        private
+
+        def prune_to_limit(group, resulting_set)
+          return if resulting_set.size >= limit
+
+          group_size = world.example_count([group])
+
+          if resulting_set.size + group_size > limit
+            (group.descendants - [group]).each do |g|
+              prune_to_limit(g, resulting_set)
+            end
+
+            add_examples(group, resulting_set)
+          else
+            resulting_set.add(group.id, group_size)
+          end
+        end
+
+        def add_examples(group, resulting_set)
+          limit_diff = limit - resulting_set.size
+
+          return unless limit_diff.positive?
+
+          group.filtered_examples.first(limit_diff).each do |example|
+            resulting_set.add(example.id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rspec/prediction_pruning.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "crystalball/rspec/prediction_pruning/examples_pruner"
+
+module Crystalball
+  module RSpec
+    # Module contains logic related to examples_limit configuration option for our runner.
+    module PredictionPruning
+      def self.included(base)
+        base.extend ClassMethods
+      end
+
+      # Class methods for prediction pruning logic
+      module ClassMethods
+        def examples_limit
+          config["examples_limit"].to_i
+        end
+
+        private
+
+        def prune_prediction_to_limit(prediction)
+          return prediction if !examples_limit.positive? || prediction.size <= examples_limit
+
+          Crystalball.log :warn, "Prediction size #{prediction.size} is over the limit (#{examples_limit})"
+          Crystalball.log :warn, "Prediction is pruned to fit the limit!"
+
+          # Actual examples size is not less than prediction size.
+          prediction.first(examples_limit)
+        end
+      end
+
+      private
+
+      def examples_limit
+        self.class.examples_limit
+      end
+
+      def reconfiguration_needed?
+        examples_limit.positive? && @world.example_count > examples_limit
+      end
+
+      def reconfigure_to_limit
+        pruner = ExamplesPruner.new(@world, to: examples_limit)
+
+        @options = ::RSpec::Core::ConfigurationOptions.new(pruner.pruned_set)
+        @world.reset
+        @world.filtered_examples.clear
+        @world.instance_variable_get(:@example_group_counts_by_spec_file).clear
+        @configuration.reset
+        @configuration.reset_filters
+
+        @options.configure(@configuration)
+      end
+    end
+  end
+end