ossert 0.1.0

data/lib/ossert/classifiers/growing/classifier.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+module Ossert
+  module Classifiers
+    class Growing
+      class Classifier
+        GRADES = %w(
+          ClassA
+          ClassB
+          ClassC
+          ClassD
+          ClassE
+        ).freeze
+        REVERSED_GRADE = {
+          'ClassA' => 'ClassE',
+          'ClassB' => 'ClassD',
+          'ClassC' => 'ClassC',
+          'ClassD' => 'ClassB',
+          'ClassE' => 'ClassA'
+        }.freeze
+
+        attr_reader :classifier, :config
+        def initialize(classifier, config)
+          @classifier = classifier
+          @config = config
+        end
+
+        def train
+          run_aggregation
+          run_syntetics
+          run_values_to_ranges
+          run_reverse
+
+          classifier
+        end
+
+        def run_aggregation
+          GRADES.each_with_index do |grade, idx|
+            classifier[grade].each_pair do |metric, values|
+              sibling_class_values = []
+              sibling_class_values = classifier[GRADES[idx + 1]][metric] if (idx + 1) < GRADES.count
+
+              all_values = sibling_class_values + values
+              (classifier[grade][metric] = (values.max || 0)) && next if all_values.count <= 2
+              classifier[grade][metric] = (all_values.sum / all_values.count).round(2)
+            end
+          end
+        end
+
+        def run_syntetics
+          config['syntetics'].each do |synt_metric, values_range|
+            real_values = classifier.values.map { |metrics| metrics[synt_metric] }.compact
+            next if real_values.empty?
+
+            values_range = Array.wrap(values_range)
+            values_range = values_range.reverse if reversed_metrics.include? synt_metric
+            step_threshold = syntetic_step_threshold(values_range)
+
+            GRADES.each_with_index do |grade, idx|
+              classifier[grade][synt_metric] = step_threshold.call(idx)
+            end
+          end
+        end
+
+        def syntetic_step_threshold(values_range)
+          return ->(idx) { values_range[idx] } if values_range.count == GRADES.count
+          max_value, min_value = values_range
+          min_value = min_value.to_i
+          max_value = max_value.to_i
+          step = ((max_value - min_value) / GRADES.count.to_f).round(2)
+          ->(idx) { max_value - (step * idx).round(2) }
+        end
+
+        def run_values_to_ranges
+          GRADES.each do |grade|
+            classifier[grade].each_pair do |metric, value|
+              classifier[grade][metric] = {
+                threshold: value,
+                range: ThresholdToRange.range_for(metric, value, grade)
+              }
+            end
+          end
+        end
+
+        class ThresholdToRange
+          def self.range_for(metric, value, grade)
+            new(metric, value, grade).range
+          end
+
+          def initialize(metric, value, grade)
+            @metric = metric
+            @value = value
+            @grade = grade
+          end
+
+          def range
+            if reversed_metrics.include?(@metric)
+              Reversed.new(@value, @grade).range
+            else
+              Base.new(@value, @grade).range
+            end
+          end
+
+          def reversed_metrics
+            @reversed_metrics ||= Ossert::Classifiers::Growing.config['reversed']
+          end
+
+          class Base
+            def initialize(value, grade)
+              @value = value
+              @full_range = (grade == last_grade)
+            end
+
+            def range
+              return full_range if full_range?
+              start_value...end_value
+            end
+
+            private
+
+            def full_range?
+              @full_range
+            end
+
+            def last_grade
+              GRADES.last
+            end
+
+            def full_range
+              -Float::INFINITY...Float::INFINITY
+            end
+
+            def start_value
+              @value
+            end
+
+            def end_value
+              Float::INFINITY
+            end
+          end
+
+          class Reversed < Base
+            def last_grade
+              GRADES.first
+            end
+
+            def start_value
+              -Float::INFINITY
+            end
+
+            def end_value
+              @value
+            end
+          end
+        end
+
+        def reversed_metrics
+          @reversed_metrics ||= config['reversed']
+        end
+
+        def run_reverse
+          reversed_metrics.each do |reversed_metric|
+            GRADES.first(GRADES.count / 2).each do |grade|
+              grade_metrics = classifier[grade]
+              next unless grade_metrics[reversed_metric].present?
+              reversed_grade_metrics = classifier[REVERSED_GRADE[grade]]
+
+              reversed_grade_metrics[reversed_metric], grade_metrics[reversed_metric] =
+                grade_metrics[reversed_metric], reversed_grade_metrics[reversed_metric]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ossert/classifiers/growing.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+require 'ossert/classifiers/growing/classifier'
+require 'ossert/classifiers/growing/check'
+
+module Ossert
+  module Classifiers
+    class Growing
+      GRADES = %w(
+        ClassA
+        ClassB
+        ClassC
+        ClassD
+        ClassE
+      ).freeze
+
+      class << self
+        attr_accessor :all
+
+        def current
+          all.last
+        end
+
+        def config
+          @config ||= Settings['classifiers']['growth']
+        end
+      end
+
+      def initialize
+        (self.class.all ||= []) << self
+      end
+
+      attr_reader :train_group
+      attr_reader :agility_total_classifier, :community_total_classifier,
+                  :agility_last_year_classifier, :community_last_year_classifier
+
+      def ready?
+        agility_total_classifier.keys == GRADES && community_total_classifier.keys == GRADES
+      end
+
+      def reference_values_per_grade
+        {
+          agility_total: classifier_to_metrics_per_grade(agility_total_classifier),
+          agility_quarter: classifier_to_metrics_per_grade(agility_last_year_classifier),
+          agility_year: classifier_to_metrics_per_grade(agility_last_year_classifier),
+          community_total: classifier_to_metrics_per_grade(community_total_classifier),
+          community_quarter: classifier_to_metrics_per_grade(community_last_year_classifier),
+          community_year: classifier_to_metrics_per_grade(community_last_year_classifier)
+        }
+      end
+
+      def classifier_to_metrics_per_grade(classifier)
+        classifier.each_with_object({}) do |(grade, metrics), res|
+          metrics.each do |metric, value|
+            (res[metric] ||= {})[grade] = value
+          end
+        end
+      end
+
+      def process_using(action, project, last_year_offset = 1)
+        Check.send(action,
+                   self.class.config,
+                   project,
+                   {
+                     agility_total: agility_total_classifier,
+                     community_total: community_total_classifier,
+                     agility_last_year: agility_last_year_classifier,
+                     community_last_year: community_last_year_classifier
+                   },
+                   last_year_offset)
+      end
+
+      def grade(*args)
+        process_using(*args.unshift(:grade))
+      end
+
+      def check(*args)
+        process_using(*args.unshift(:check))
+      end
+
+      def train
+        classifiers_initializer = ClassifiersInitializer.load_or_create
+        classifiers_initializer.run
+        classifiers_initializer.classifiers.each do |name, classifier|
+          instance_variable_set(
+            "@#{name}_classifier",
+            Classifier.new(classifier, self.class.config).train
+          )
+        end
+      end
+
+      class ClassifiersInitializer
+        CLASSIFIERS_METRICS = {
+          agility_total: ->(project) { project.agility.total.metrics_to_hash },
+          agility_last_year: ->(project) { project.agility.quarters.last_year_as_hash },
+          community_total: ->(project) { project.community.total.metrics_to_hash },
+          community_last_year: ->(project) { project.community.quarters.last_year_as_hash }
+        }.freeze
+
+        def self.load_or_create
+          if ::Classifier.actual?
+            new.load
+          else
+            new(Project.projects_by_reference)
+          end
+        end
+
+        attr_reader :classifiers
+
+        def initialize(grouped_projects = nil)
+          @projects = grouped_projects
+          @classifiers = []
+        end
+
+        def load
+          @classifiers = {}
+          CLASSIFIERS_METRICS.keys.each do |section|
+            @classifiers[section] = JSON.parse(::Classifier[section.to_s].reference_values)
+          end
+          self
+        end
+
+        def merge_metrics(storage, metrics)
+          metrics.each do |metric, value|
+            storage.store(
+              metric.to_s,
+              storage[metric.to_s].to_a << value.to_f
+            )
+          end
+          storage
+        end
+
+        def save
+          ::Classifier.dataset.delete
+
+          @classifiers.each do |section, reference_values|
+            ::Classifier.create(
+              section: section.to_s,
+              reference_values: JSON.generate(reference_values)
+            )
+          end
+        end
+
+        def new_classifiers
+          CLASSIFIERS_METRICS.keys.map { |type| [type, {}] }.to_h
+        end
+
+        def run
+          return if @classifiers.present?
+
+          @classifiers = GRADES.each_with_object(new_classifiers) do |grade, classifiers|
+            @projects[grade].each do |project|
+              CLASSIFIERS_METRICS.each do |type, metrics|
+                classifiers[type][grade] = merge_metrics(classifiers[type][grade].to_h, metrics.call(project))
+              end
+            end
+          end
+
+          save
+        end
+      end
+    end
+  end
+end

data/lib/ossert/classifiers.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+require 'ossert/classifiers/decision_tree'
+require 'ossert/classifiers/growing'
+
+module Ossert
+  module Classifiers
+    def train
+      Growing.new.train
+      # Stale. Very untrusty
+      # DecisionTree.new(projects_by_reference).train
+    end
+    module_function :train
+  end
+end

data/lib/ossert/config.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+require 'yaml'
+
+module Ossert
+  class Config
+    CONFIG_ROOT = File.join(File.dirname(__FILE__), '..', '..', 'config')
+    CONST_NAME = 'Settings'
+
+    # TODO: use ERB, needed for Time syntetics
+    def self.load(*configs)
+      config_data = configs.each_with_object({}) do |config, result|
+        if (path = File.join(CONFIG_ROOT, "#{config}.yml")) && File.exist?(path.to_s)
+          result.merge! YAML.load(IO.read(path.to_s))
+        end
+      end
+
+      Kernel.send(:remove_const, CONST_NAME) if Kernel.const_defined?(CONST_NAME)
+      Kernel.const_set(CONST_NAME, config_data)
+    rescue Psych::SyntaxError => e
+      raise "YAML syntax error occurred while parsing #{path}. " \
+            "Error: #{e.message}"
+    end
+  end
+end

data/lib/ossert/fetch/bestgems.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+module Ossert
+  module Fetch
+    class BestgemsBase
+      def self.process_page(page = nil)
+        doc = Nokogiri::HTML(open("http://bestgems.org/#{endpoint}#{page ? "?page=#{page}" : ''}"))
+        doc.css('table').xpath('//tr//td').each_slice(4) do |rank, downloads, name, _|
+          rank = rank.text.delete(',').to_i
+          downloads = downloads.text.delete(',').to_i
+          yield(rank, downloads, name.text)
+        end
+      end
+    end
+
+    class BestgemsDailyStat < BestgemsBase
+      def self.endpoint
+        :daily
+      end
+    end
+
+    class BestgemsTotalStat < BestgemsBase
+      def self.endpoint
+        :total
+      end
+    end
+
+    class Bestgems
+      attr_reader :client, :project
+
+      extend Forwardable
+      def_delegators :project, :agility, :community, :meta
+
+      def initialize(project)
+        @client = SimpleClient.new('http://bestgems.org/api/v1/')
+        @project = project
+      end
+
+      def total_downloads
+        client.get("gems/#{project.rubygems_alias}/total_downloads.json")
+      end
+
+      def daily_downloads
+        client.get("gems/#{project.rubygems_alias}/daily_downloads.json")
+      end
+
+      def total_ranking
+        client.get("gems/#{project.rubygems_alias}/total_ranking.json")
+      end
+
+      def daily_ranking
+        client.get("gems/#{project.rubygems_alias}/daily_ranking.json")
+      end
+
+      def process
+        process_total_downloads
+        process_daily_downloads
+        process_delta_downloads
+      end
+
+      def process_total_downloads
+        @downloads_till_now = nil
+        total_downloads.each do |total|
+          @downloads_till_now = total unless @downloads_till_now
+          quarter_downloads = community.quarters[total['date']]
+
+          quarter_downloads.total_downloads = [
+            quarter_downloads.total_downloads.to_i,
+            total['total_downloads']
+          ].max
+        end
+        community.total.total_downloads = @downloads_till_now['total_downloads']
+      end
+
+      def process_daily_downloads
+        daily_downloads.each do |daily|
+          downloads_saved = community.quarters[daily['date']].delta_downloads.to_i
+          community.quarters[daily['date']].delta_downloads = downloads_saved + daily['daily_downloads']
+        end
+      end
+
+      def process_delta_downloads
+        prev_downloads_delta = 0
+        community.quarters.each_sorted do |start_date, stat|
+          prev_downloads_delta = stat.delta_downloads.to_i - prev_downloads_delta
+          community.quarters[start_date].download_divergence = divergence(
+            prev_downloads_delta, @downloads_till_now['total_downloads']
+          )
+        end
+      end
+
+      private
+
+      def divergence(delta, total)
+        (delta.to_f / total.to_f * 100.0).round(2)
+      end
+    end
+  end
+end