gitlab-crystalball 0.7.1

data/lib/crystalball/map_generator.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Crystalball
+  # Class to generate execution map during RSpec build execution
+  class MapGenerator
+    extend Forwardable
+
+    attr_reader :configuration
+    delegate %i[map_storage strategies dump_threshold map_class] => :configuration
+
+    class << self
+      # Registers Crystalball handlers to generate execution map during specs execution
+      #
+      # @param [Proc] block to configure MapGenerator and Register strategies
+      def start!(&block)
+        generator = new(&block)
+
+        ::RSpec.configure do |c|
+          c.before(:suite) { generator.start! }
+
+          c.around(:each) { |e| generator.refresh_for_case(e) }
+
+          c.after(:suite) { generator.finalize! }
+        end
+      end
+    end
+
+    def initialize
+      @configuration = Configuration.new
+      @configuration.commit = repo.gcommit("HEAD") if repo
+      yield @configuration if block_given?
+    end
+
+    # Registers strategies and prepares metadata for execution map
+    def start!
+      self.map = nil
+      map_storage.clear!
+      map_storage.dump(map.metadata.to_h)
+
+      strategies.reverse.each(&:after_start)
+      self.started = true
+    end
+
+    # Runs example and collects execution map for it
+    def refresh_for_case(example)
+      map << strategies.run(ExampleGroupMap.new(example), example) { example.run }
+      check_dump_threshold
+    end
+
+    # Finalizes strategies and saves map
+    def finalize!
+      return unless started
+
+      strategies.each(&:before_finalize)
+
+      return unless map.size.positive?
+
+      example_groups = (configuration.compact_map? ? MapCompactor.compact_map!(map) : map).example_groups
+      map_storage.dump(example_groups)
+    end
+
+    def map
+      @map ||= map_class.new(metadata: {commit: configuration.commit&.sha, timestamp: configuration.commit&.date&.to_i, version: configuration.version})
+    end
+
+    private
+
+    attr_writer :map
+    attr_accessor :started
+
+    def repo
+      @repo = GitRepo.open(".") unless defined?(@repo)
+      @repo
+    end
+
+    def check_dump_threshold
+      return if configuration.compact_map
+      return unless dump_threshold.positive? && map.size >= dump_threshold
+
+      map_storage.dump(map.example_groups)
+      map.clear!
+    end
+  end
+end

data/lib/crystalball/map_storage/yaml_storage.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Crystalball
+  class MapStorage
+    # Exception class for missing map files
+    class NoFilesFoundError < StandardError; end
+
+    # YAML persistence adapter for execution map storage
+    class YAMLStorage
+      attr_reader :path
+
+      class << self
+        # Loads map from given path
+        #
+        # @param [String] path to map
+        # @return [Crystalball::ExecutionMap]
+        def load(path)
+          meta, example_groups = *read_files(path).transpose
+
+          guard_metadata_consistency(meta)
+
+          Object.const_get(meta.first[:type]).new(metadata: meta.first, example_groups: example_groups.compact.inject(&:merge!))
+        end
+
+        private
+
+        def read_files(path)
+          paths = path.directory? ? path.each_child.select(&:file?) : [path]
+
+          raise NoFilesFoundError, "No files or folder exists #{path}" unless paths.any?(&:exist?)
+
+          paths.map do |file|
+            metadata, *example_groups = file.read.split("---\n").reject(&:empty?).map do |yaml|
+              YAML.safe_load(yaml, permitted_classes: [Symbol])
+            end
+            example_groups = example_groups.inject(&:merge!)
+
+            [metadata, example_groups]
+          end
+        end
+
+        def guard_metadata_consistency(metadata)
+          uniq = metadata.uniq
+          raise "Can't load execution maps with different metadata. Metadata: #{uniq}" if uniq.size > 1
+        end
+      end
+
+      # @param [String] path to store execution map
+      def initialize(path)
+        @path = path
+      end
+
+      # Removes storage file
+      def clear!
+        path.delete if path.exist?
+      end
+
+      # Writes data to storage file
+      #
+      # @param [Hash] data to write to storage file
+      def dump(data)
+        path.dirname.mkpath
+        path.open("a") { |f| f.write YAML.dump(data) }
+      end
+    end
+  end
+end

data/lib/crystalball/prediction.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Crystalball
+  # Class for Crystalball prediction results
+  class Prediction
+    def initialize(records)
+      @records = records
+    end
+
+    # When the records are something like:
+    #   ./spec/foo ./spec/foo/bar_spec.rb
+    # this returns just ./spec/foo
+    def compact
+      sort_by(&:length).each_with_object([]) do |c, result|
+        result << c unless result.any? { |r| c.start_with?(r) }
+      end.compact
+    end
+
+    def to_a
+      records
+    end
+
+    def method_missing(*args, &block)
+      records.respond_to?(*args) ? records.public_send(*args, &block) : super
+    end
+
+    def respond_to_missing?(*args)
+      records.respond_to?(*args)
+    end
+
+    private
+
+    attr_reader :records
+  end
+end

data/lib/crystalball/predictor/associated_specs.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "crystalball/predictor/strategy"
+
+module Crystalball
+  class Predictor
+    # Used with `predictor.use Crystalball::Predictor::AssociatedSpecs.new(from: %r{models/(.*).rb}, to: "./spec/models/%s_spec.rb")`.
+    # When used will look for files matched to `from` regex and use captures to fill `to` string to
+    # get paths of proper specs
+    class AssociatedSpecs
+      include Strategy
+
+      # @param [Regexp] from - regular expression to match specific files and get proper captures
+      # @param [String] to - string in sprintf format to get proper files using captures of regexp
+      def initialize(from:, to:)
+        @from = from
+        @to = to
+      end
+
+      # This strategy does not depend on a previously generated example group map.
+      # It uses the defined regex rules to infer which specs to run.
+      # @param [Crystalball::SourceDiff] diff - the diff from which to predict
+      #   which specs should run
+      # @return [Array<String>] the spec paths associated with the changes
+      def call(diff, _map)
+        super do
+          diff.map(&:relative_path).grep(from).map { |source_file_path| to % captures(source_file_path) }
+        end
+      end
+
+      private
+
+      attr_reader :from, :to
+
+      def captures(file_path)
+        match = file_path.match(from)
+        if match.names.any?
+          match.names.map(&:to_sym).zip(match.captures).to_h
+        else
+          match.captures
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/predictor/helpers/affected_example_groups_detector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Crystalball
+  class Predictor
+    module Helpers
+      # Helper module to fetch example groups affected by given list of changed files
+      module AffectedExampleGroupsDetector
+        # Fetch examples affected by given list of files
+        # @param [Array<String>] files - list of files
+        # @param [Crystalball::ExecutionMap] map - execution map with examples
+        # @return [Array<String>] list of affected examples
+        def detect_examples(files, map)
+          map.example_groups.map do |uid, example_group_map|
+            uid if files.any? { |file| example_group_map.include?(file) }
+          end.compact
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/predictor/helpers/path_formatter.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Crystalball
+  class Predictor
+    module Helpers
+      # Helper module for converting relative path to RSpec format
+      module PathFormatter
+        def format_paths(paths)
+          paths.map { |path| format_path(path) }
+        end
+
+        def format_path(path)
+          path.start_with?("./") ? path : "./#{path}"
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/predictor/modified_execution_paths.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "crystalball/predictor/strategy"
+require "crystalball/predictor/helpers/affected_example_groups_detector"
+
+module Crystalball
+  class Predictor
+    # Used with `predictor.use Crystalball::Predictor::ModifiedExecutionPaths.new`. When used will check the map which
+    # specs depend on which files and will return only those specs which depend on files modified since last time map
+    # was generated.
+    class ModifiedExecutionPaths
+      include Helpers::AffectedExampleGroupsDetector
+      include Strategy
+
+      # @param [Crystalball::SourceDiff] diff - the diff from which to predict
+      #   which specs should run
+      # @param [Crystalball::ExampleGroupMap] map - the map with the relations of
+      #   examples and used files
+      # @return [Array<String>] the spec paths associated with the changes
+      def call(diff, map)
+        super do
+          detect_examples(diff.map(&:relative_path), map)
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/predictor/modified_specs.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "crystalball/predictor/strategy"
+
+module Crystalball
+  class Predictor
+    # Used with `predictor.use Crystalball::Predictor::ModifiedSpecs.new`. Will find files that match spec regexp and
+    # return all new or modified files. You can specify spec regexp using first parameter to `#initialize`.
+    class ModifiedSpecs
+      include Strategy
+
+      # @param [Regexp] spec_pattern - regexp to filter specs files
+      def initialize(spec_pattern = %r{spec/.*_spec\.rb\z})
+        @spec_pattern = spec_pattern
+      end
+
+      # This strategy does not depend on a previously generated example group map.
+      # It uses the spec pattern to determine which specs should run.
+      # @param [Crystalball::SourceDiff] diff - the diff from which to predict
+      #   which specs should run
+      # @return [Array<String>] the spec paths associated with the changes
+      def call(diff, _)
+        super do
+          diff.reject(&:deleted?).map(&:new_relative_path).grep(spec_pattern)
+        end
+      end
+
+      private
+
+      attr_reader :spec_pattern
+    end
+  end
+end

data/lib/crystalball/predictor/modified_support_specs.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "crystalball/predictor/strategy"
+require "crystalball/predictor/helpers/affected_example_groups_detector"
+
+module Crystalball
+  class Predictor
+    # Used with `predictor.use Crystalball::Predictor::ModifiedSupportSpecs.new`. Will find files that match passed regexp and
+    # return full spec files which uses matched support spec files. Perfectly works for shared_context and shared_examples.
+    class ModifiedSupportSpecs
+      include Strategy
+      include Helpers::AffectedExampleGroupsDetector
+
+      # @param [Regexp] support_spec_pattern - regexp to filter support specs files
+      def initialize(support_spec_pattern = %r{spec/support/.*\.rb\z})
+        @support_spec_pattern = support_spec_pattern
+      end
+
+      # @param [Crystalball::SourceDiff] diff - the diff from which to predict
+      #   which specs should run
+      # @param [Crystalball::ExampleGroupMap] map - the map with the relations of
+      #   examples and used files
+      # @return [Array<String>] the spec paths associated with the changes
+      def call(diff, map)
+        super do
+          changed_support_files = diff.map(&:relative_path).grep(support_spec_pattern)
+
+          examples = detect_examples(changed_support_files, map)
+
+          examples.map { |e| e.to_s.split("[").first }.uniq
+        end
+      end
+
+      private
+
+      attr_reader :support_spec_pattern
+    end
+  end
+end

data/lib/crystalball/predictor/strategy.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "crystalball/predictor/helpers/path_formatter"
+
+module Crystalball
+  class Predictor
+    # Base module to include in any strategy. Provides output formatting similar to RSpec
+    module Strategy
+      include Helpers::PathFormatter
+
+      def call(*)
+        format_paths(yield)
+      end
+    end
+  end
+end

data/lib/crystalball/predictor.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Crystalball
+  # Class to predict test failures with given execution map and sources diff
+  class Predictor
+    attr_reader :map, :from, :to, :prediction_strategies
+
+    # @param [Crystalball::ExecutionMap] map execution map
+    # @param [Crystalball::GitRepo] repo to build execution list on
+    # @param [String] from starting commit for diff. Default: HEAD
+    # @param [String] to ending commit for diff. Default: nil
+    def initialize(map, repo, from: "HEAD", to: nil)
+      @map = map
+      @repo = repo
+      @from = from
+      @to = to
+      @prediction_strategies = []
+      yield self if block_given?
+    end
+
+    # Adds additional predictor to use
+    #
+    # @param [#call] strategy - the strategy can be any object that responds to #call
+    def use(strategy)
+      prediction_strategies << strategy
+    end
+
+    # @return [Crystalball::Prediction] list of examples which may fail
+    def prediction
+      Prediction.new(filter(raw_prediction(diff)))
+    end
+
+    def diff
+      @diff ||= begin
+                  ancestor = repo.merge_base(from, to || "HEAD").sha
+                  repo.diff(ancestor, to)
+                end
+    end
+
+    private
+
+    # TODO: check if it would be better to call predictors with one case instead of passing the whole map.
+    def predict!(current_diff)
+      prediction_strategies.flat_map { |strategy| strategy.call(current_diff, map) }
+    end
+    alias raw_prediction predict!
+
+    attr_reader :repo
+
+    def filter(example_groups)
+      example_groups.compact.select { |example_group| extract_file_path(example_group).exist? }.uniq
+    end
+
+    def extract_file_path(example)
+      repo.repo_path.join(example.split("[").first).expand_path
+    end
+  end
+end

data/lib/crystalball/predictor_evaluator.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Crystalball
+  # Class to collect statistics about prediction quality
+  class PredictorEvaluator
+    attr_reader :predictor, :actual_failures
+
+    # @param [Crystalball::Predictor] predictor - configured predictor to fetch list of examples which might fail
+    # @param [Array<String>] actual_failures - list of actual failed examples
+    def initialize(predictor, actual_failures:)
+      @predictor = predictor
+      @actual_failures = actual_failures
+    end
+
+    def predicted_failures
+      @predicted_failures ||= actual_failures.select do |failure|
+        prediction.any? { |p| failure.include?(p) }
+      end
+    end
+
+    def unpredicted_failures
+      actual_failures - predicted_failures
+    end
+
+    def diff_size
+      predictor.diff.lines
+    end
+
+    def prediction_to_diff_ratio
+      prediction_size.to_f / diff_size
+    end
+
+    def prediction_scale
+      prediction_size.to_f / map_size
+    end
+
+    def prediction_rate
+      actual_failures.empty? ? 1.0 : predicted_failures.size.to_f / actual_failures.size
+    end
+
+    def prediction_size
+      @prediction_size ||= predictor.map.example_groups.keys.select { |example| prediction.any? { |p| example.include?(p) } }.size
+    end
+
+    def map_size
+      predictor.map.size
+    end
+
+    private
+
+    def prediction
+      @prediction ||= predictor.prediction
+    end
+  end
+end

data/lib/crystalball/rails/helpers/base_schema_parser.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Crystalball
+  module Rails
+    module Helpers
+      # Interface for schema parsers
+      module BaseSchemaParser
+        def self.parse(*_)
+          raise NotImplementedError
+        end
+
+        # @return [Hash] stored info about all method calls which ended in #method_missing
+        attr_accessor :hash
+
+        def initialize
+          @hash = {}
+        end
+
+        private
+
+        # Store info about call in hash. First argument of method call used as a key
+        def method_missing(method_name, *args, &block)
+          name = args.shift
+          add_to_hash(name, options: [method_name] + args)
+
+          new_parser = self.class.new
+          add_to_hash(name, content: new_parser.instance_exec(&block)) if block
+          add_to_hash(name, content: new_parser.hash)
+          new_parser
+        end
+
+        def respond_to_missing?(*_)
+          true
+        end
+
+        def add_to_hash(name, options: nil, content: nil)
+          hash[name] ||= {}
+          add_optional(name, :options, options)
+          add_optional(name, :content, content)
+        end
+
+        def add_optional(name, key, value)
+          return unless value
+
+          hash[name][key] ||= []
+          hash[name][key] << value
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/helpers/schema_definition_parser/active_record.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Crystalball
+  module Rails
+    module Helpers
+      class SchemaDefinitionParser
+        # This mock will be used during SchemaDefinitionParser.parse
+        module ActiveRecord
+          # A simple mock to read definition of schema
+          class Schema
+            def self.define(*_args, &block)
+              collector = SchemaDefinitionParser.new
+              collector.instance_exec(collector, &block)
+              collector.hash
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/helpers/schema_definition_parser/table_content_parser.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "crystalball/rails/helpers/base_schema_parser"
+
+module Crystalball
+  module Rails
+    module Helpers
+      class SchemaDefinitionParser
+        # Class used to parse ActiveRecord::Schema create_table definition and provide hash representation
+        class TableContentParser
+          include BaseSchemaParser
+
+          # Parse create_table definition of schema
+          # @param [Proc] block - block for create_table definition
+          # @return [Hash] hash representation of table definition
+          def self.parse(&block)
+            return {} unless block
+
+            collector = new
+            collector.instance_exec(collector, &block)
+            collector.hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/helpers/schema_definition_parser.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "crystalball/rails/helpers/base_schema_parser"
+require "crystalball/rails/helpers/schema_definition_parser/active_record"
+require "crystalball/rails/helpers/schema_definition_parser/table_content_parser"
+
+module Crystalball
+  module Rails
+    module Helpers
+      # Class used to parse ActiveRecord::Schema definition and provide hash representation
+      class SchemaDefinitionParser
+        include BaseSchemaParser
+
+        # Parse schema content
+        # @param [String] schema - schema file content
+        # @return [Hash] hash representation of schema
+        def self.parse(schema)
+          return {} if schema&.empty?
+
+          new.instance_eval(schema)
+        end
+
+        private
+
+        def create_table(table_name, *options, &block)
+          add_to_hash(table_name, options: ["create_table"] + options, content: TableContentParser.parse(&block))
+        end
+
+        def add_foreign_key(table1, table2, *options)
+          add_to_hash(table1, options: ["add_foreign_key", table2] + options)
+          add_to_hash(table2, options: ["add_foreign_key", table1] + options) if table1 != table2
+        end
+      end
+    end
+  end
+end

data/lib/crystalball/rails/map_generator/action_view_strategy/patch.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "action_view"
+
+module Crystalball
+  module Rails
+    class MapGenerator
+      class ActionViewStrategy
+        # Module to add new patched `compile!` method to ActionView::Template
+        module Patch
+          class << self
+            # Patches `ActionView::Template#compile!`. Renames original `compile!` to `cb_original_compile!` and
+            # replaces it with custom one
+            def apply!
+              ::ActionView::Template.class_eval do
+                include Patch
+
+                alias_method :cb_original_compile!, :compile!
+                alias_method :compile!, :cb_patched_compile!
+              end
+            end
+
+            # Reverts original behavior of `ActionView::Template#compile!`
+            def revert!
+              ::ActionView::Template.class_eval do
+                alias_method :compile!, :cb_original_compile! # rubocop:disable Lint/DuplicateMethods
+                undef_method :cb_patched_compile!
+              end
+            end
+          end
+
+          # Will replace original `ActionView::Template#compile!`. Pushes path of a view to
+          # `ActionViewStrategy.views` and calls original `compile!`
+          def cb_patched_compile!(*args)
+            ActionViewStrategy.views.push identifier
+            cb_original_compile!(*args)
+          end
+        end
+      end
+    end
+  end
+end