ossert 0.1.0

data/lib/ossert/fetch.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+require 'nokogiri'
+require 'open-uri'
+require 'benchmark'
+
+require 'ossert/fetch/github'
+require 'ossert/fetch/rubygems'
+require 'ossert/fetch/bestgems'
+
+# TODO: Add logging
+
+module Ossert
+  # Public: Various classes and methods for fetching data from different sources.
+  # Such as GitHub, Rubygems, Bestgems. Also provides simple functionality for
+  # fetching HTTP API.
+  module Fetch
+    ALL_FETCHERS = [Rubygems, Bestgems, GitHub].freeze
+
+    # Public: Fetch data for project using all fetchers by default process method
+    #
+    # project  - The Ossert::Project instance to fill using fetchers
+    #
+    # Examples
+    #
+    #   project = Ossert::Project.new('ramaze')
+    #   Ossert::Fetch.all(project)
+    #   project.dump
+    #
+    # Returns nothing.
+    def all(project)
+      ALL_FETCHERS.each do |fetcher|
+        puts "======> with #{fetcher}..."
+        time = Benchmark.realtime do
+          fetcher.new(project).process
+        end
+        puts "<====== Finished in #{time.round(3)} sec."
+        sleep(1)
+      end
+      nil
+    rescue => e
+      puts "Fetching Failed for '#{name}' with error: #{e.inspect}"
+      puts e.backtrace
+    end
+    module_function :all
+
+    # Public: Fetch data for project using given fetchers by process method
+    #
+    # fetchers - The Array or one of Ossert::Fetch::GitHub, Ossert::Fetch::Bestgems, Ossert::Fetch::Rubygems to
+    #            use for processing
+    # project  - The Ossert::Project instance to fill using fetchers
+    # process  - The Symbol method name used for processing by fetchers (default: :process)
+    #
+    # Examples
+    #
+    #   project = Ossert::Project.new('ramaze')
+    #   Ossert::Fetch.only(Ossert::Fetch::Rubygems, project, :process_meta)
+    #   project.dump_attribute :meta_data
+    #
+    # Returns nothing.
+    def only(fetchers, project, process = :process)
+      fetchers = Array.wrap(fetchers)
+      puts "Fetching project '#{project.name}'..."
+      (ALL_FETCHERS & fetchers).each do |fetcher|
+        puts "======> with #{fetcher}..."
+        time = Benchmark.realtime do
+          fetcher.new(project).send(process)
+        end
+        puts "<====== Finished in #{time.round(3)} sec."
+        sleep(1)
+      end
+    end
+    module_function :only
+
+    # Public: Simple client for fetching HTTP API
+    #
+    # Examples
+    #
+    #   client = SimpleClient.new("http://bestgems.org/api/v1/")
+    #   client.get("gems/#{project.rubygems_alias}/total_downloads.json")
+    #   # => Some JSON from api
+    class SimpleClient
+      attr_reader :api_endpoint, :type
+
+      # Public: Instantiate client for fetching API for given api_endpoint and response type
+      #
+      # path - The String describes path of endpoint to access the data
+      # type - The String describes type of response data, e.g. 'json'
+      #
+      # Examples
+      #
+      #   client = SimpleClient.new("http://bestgems.org/api/v1/")
+      #   client.get("gems/#{project.rubygems_alias}/total_downloads.json")
+      #   # => Some JSON from api
+      #
+      # Returns nothing.
+      def initialize(api_endpoint, type = nil)
+        raise ArgumentError if !api_endpoint.start_with?('http') || !api_endpoint.end_with?('/')
+        @api_endpoint = api_endpoint
+        @type = type || 'json'
+      end
+
+      class NotFound < StandardError; end
+      class UnexpectedResponseError < StandardError; end
+
+      # Public: Get data via HTTP GET for given API path
+      #
+      # path - The String describes path of endpoint to access the data
+      #
+      # Examples
+      #
+      #   client = SimpleClient.new("http://bestgems.org/api/v1/")
+      #   client.get("gems/#{project.rubygems_alias}/total_downloads.json")
+      #   # => Some JSON from api
+      #
+      # Returns nothing.
+      def get(path)
+        raise ArgumentError unless path.end_with? type
+        response = agent.get("#{api_endpoint}#{path}")
+        case response.status
+        when 404
+          raise NotFound
+        when 200
+          JSON.parse(response.body)
+        else
+          raise UnexpectedResponseError
+        end
+      end
+
+      private
+
+      # Internal: Initialize Faraday agent for processing requests
+      #
+      # Returns Faraday::Connection instance.
+      def agent
+        @agent ||= ::Faraday.new do |http|
+          http.request :url_encoded
+          http.adapter :net_http
+        end
+      end
+    end
+  end
+end

data/lib/ossert/presenters/project.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+require 'ossert/presenters/project_v2'
+
+module Ossert
+  module Presenters
+    class Project
+      include Ossert::Presenters::ProjectV2
+
+      attr_reader :project, :decorator
+
+      def initialize(project)
+        @project = project
+        @reference = Ossert::Classifiers::Growing.current.reference_values_per_grade
+        @decorator = Decorator.new(@reference)
+      end
+
+      def self.with_presenter(project)
+        presenter = new(project)
+        presenter.prepare!
+        yield(presenter)
+        presenter.cleanup_references!
+      end
+
+      class Decorator
+        def initialize(reference)
+          @reference = reference
+        end
+
+        # value, Float !
+        def with_reference(text, value, metric, type)
+          return (text.to_i.positive? ? "+#{text}" : text).to_s if type == :delta
+          metric_by_ref = @reference[type][metric.to_s]
+          reference = Project::CLASSES.inject('NaN') do |acc, ref|
+            metric_by_ref[ref][:range].cover?(value) ? ref : acc
+          end
+
+          { text: "#{text} #{Project::KLASS_2_GRADE[reference]}",
+            mark: Project::KLASS_2_GRADE[reference].downcase }
+        rescue => e
+          puts "NO DATA FOR METRIC: '#{metric}'"
+          raise e
+        end
+
+        TOO_LONG_AGO = 20.years.ago
+        METRICS_DECORATIONS = {
+          /(percent|divergence)/ => ->(value) { "#{value.ceil}%" },
+          /(date|changed)/ => (lambda do |value|
+            date = Time.at(value)
+            return 'N/A' if date < TOO_LONG_AGO
+            date.strftime('%d/%m/%y')
+          end),
+          /processed_in/ => (lambda do |value|
+            return 'N/A' if value >= Ossert::Stats::PER_YEAR_TOO_LONG || value.zero?
+            case value
+            when 1
+              "~#{value.ceil} day"
+            when 2..30
+              "~#{value.ceil} days"
+            when 31..61
+              "~#{(value / 31).to_i} month"
+            else
+              "~#{(value / 31).to_i} months"
+            end
+          end),
+          /period/ => (lambda do |value|
+            if (years = value.to_i / 365).positive?
+              "#{years}+ years"
+              'Less than a year'
+            else
+            end
+          end),
+          /downloads/ => ->(value) { value.ceil.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse }
+        }.freeze
+
+        def value(metric, value)
+          value = value.to_f
+          METRICS_DECORATIONS.each { |check, decorator| return decorator.call(value) if metric =~ check }
+          value.to_i
+        end
+
+        def metric(metric, value, type)
+          with_reference(value(metric, value), value.to_f, metric, type)
+        end
+
+        def quarter_with_diff(project, time, section)
+          section_type = "#{section}_quarter".to_sym
+          quarter_data = project.send(section).quarters[quarter_start(time)].metrics_to_hash
+          diff = quarter_values(project.send(section).quarters[quarter_start(time) - 1.day].metrics_to_hash)
+
+          quarter_data.each_with_object({}) do |(metric, value), result|
+            decorated_metric = metric(metric, value, section_type)
+            result[Ossert.t(metric)] = <<-TEXT
+  #{decorated_metric[:text]} <> #{metric(metric, value.to_i - diff[metric], :delta)}
+            TEXT
+          end
+        end
+
+        def metrics(metrics_data, section_type)
+          metrics_data.each_with_object({}) do |(metric, value), result|
+            result[metric] = metric(metric, value.to_i, section_type)
+          end
+        end
+
+        def quarter_values(quarter_data)
+          quarter_data.each_with_object({}) do |(metric, value), res|
+            res[metric] = value.to_i
+          end
+        end
+
+        private
+
+        def quarter_start(time)
+          Time.at(time).to_date.to_time(:utc).beginning_of_quarter
+        end
+      end
+
+      def agility_quarter(time)
+        decorator.quarter_with_diff(@project, time, :agility)
+      end
+
+      def community_quarter(time)
+        decorator.quarter_with_diff(@project, time, :community)
+      end
+
+      def agility_quarter_values(time)
+        decorator.quarter_values @project.agility.quarters[time].metrics_to_hash
+      end
+
+      def community_quarter_values(time)
+        decorator.quarter_values @project.community.quarters[time].metrics_to_hash
+      end
+
+      def agility_total
+        @agility_total ||= decorator.metrics @project.agility.total.metrics_to_hash, :agility_total
+      end
+
+      def community_total
+        @community_total ||= decorator.metrics @project.community.total.metrics_to_hash, :community_total
+      end
+
+      def community_last_year
+        @community_last_year ||= decorator.metrics @project.community.quarters.last_year_as_hash, :community_year
+      end
+
+      def agility_last_year
+        @agility_last_year ||= decorator.metrics @project.agility.quarters.last_year_as_hash, :agility_year
+      end
+
+      def metric_preview(metric)
+        preview = {}
+        return(preview) if (section = Ossert::Stats.guess_section_by_metric(metric)) == :not_found
+
+        preview[:tooltip] = MultiJson.dump(tooltip_data(metric))
+        preview[:translation] = Ossert.t(metric)
+
+        preview.merge!(last_year_section(metric, section))
+        preview.merge!(total_section(metric, section))
+        preview
+      end
+
+      def last_year_section(metric, section)
+        section_last_year = public_send("#{section}_last_year")[metric]
+        {
+          last_year_mark: section_last_year.try(:[], :mark),
+          last_year_val: section_last_year.try(:[], :text) || 'N/A'
+        }
+      end
+
+      def total_section(metric, section)
+        section_total = public_send("#{section}_total")[metric]
+        {
+          total_mark: section_total.try(:[], :mark),
+          total_val: section_total.try(:[], :text) || 'N/A'
+        }
+      end
+
+      def prepare!
+        agility_total
+        agility_last_year
+        community_total
+        community_last_year
+
+        lookback = 5
+        check_results = (lookback - 1).downto(0).map do |last_year_offset|
+          Ossert::Classifiers::Growing.current.check(@project, last_year_offset)
+        end
+
+        @grade = check_results.last(2).first.map { |k, v| [k, v[:mark].downcase] }.to_h
+        @fast_preview_graph = fast_preview_graph_data(lookback, check_results)
+      end
+      attr_reader :grade, :fast_preview_graph
+
+      def cleanup_references!
+        @reference = nil
+        @project = nil
+        @fast_preview_graph_data = nil
+        @grade = nil
+        @decorator = nil
+      end
+    end
+  end
+end

data/lib/ossert/presenters/project_v2.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+module Ossert
+  module Presenters
+    module ProjectV2
+      CLASSES = %w(
+        ClassE
+        ClassD
+        ClassC
+        ClassB
+        ClassA
+      ).freeze
+      KLASS_2_GRADE = {
+        'ClassA' => 'A',
+        'ClassB' => 'B',
+        'ClassC' => 'C',
+        'ClassD' => 'D',
+        'ClassE' => 'E'
+      }.freeze
+
+      def preview_reference_values_for(metric, section) # maybe automatically find section?
+        metric_by_grades = @reference[section][metric.to_s]
+        grades = CLASSES.reverse
+        sign = reversed_metrics.include?(metric) ? '<' : '>'
+
+        grades.each_with_object({}) do |klass, preview|
+          preview[KLASS_2_GRADE[klass]] = "#{sign} #{metric_by_grades[klass][:threshold].to_i}"
+        end
+      end
+
+      REFERENCES_STUB = {
+        'ClassA' => { threshold: '0', range: [] },
+        'ClassB' => { threshold: '0', range: [] },
+        'ClassC' => { threshold: '0', range: [] },
+        'ClassD' => { threshold: '0', range: [] },
+        'ClassE' => { threshold: '0', range: [] }
+      }.freeze
+
+      # Tooltip data:
+      # {
+      #   title: '',
+      #   description: '',
+      #   ranks: [
+      #     {"type":"a","year":100,"total":300},
+      #     {"type":"b","year":80,"total":240},
+      #     {"type":"c","year":60,"total":120},
+      #     {"type":"d","year":40,"total":100},
+      #     {"type":"e","year":20,"total":80}
+      #   ]
+      # }
+      def tooltip_data(metric)
+        classes = CLASSES.reverse
+        section = Ossert::Stats.guess_section_by_metric(metric)
+        ranks = classes.inject([]) do |preview, klass|
+          base = { type: KLASS_2_GRADE[klass].downcase, year: ' N/A ', total: ' N/A ' }
+          preview << [:year, :total].each_with_object(base) do |section_type, result|
+            next unless (metric_data = metric_tooltip_data(metric, section, section_type, klass)).present?
+            result[section_type] = metric_data
+          end
+        end
+
+        { title: Ossert.t(metric), description: Ossert.descr(metric), ranks: ranks }
+      end
+
+      def metric_tooltip_data(metric, section, section_type, klass)
+        return if section == :not_found # this should not happen
+        reference_section = [section, section_type].join('_')
+        return unless (metric_by_grades = @reference[reference_section.to_sym][metric.to_s])
+
+        [
+          reversed_metrics.include?(metric) ? '&lt;&nbsp;' : '&gt;&nbsp;',
+          decorator.value(metric, metric_by_grades[klass][:threshold])
+        ].join(' ')
+      end
+
+      def reversed_metrics
+        @reversed_metrics ||= Ossert::Classifiers::Growing.config['reversed']
+      end
+
+      # Fast preview graph
+      # [
+      #   {"title":"Jan - Mar 2016","type":"a","values":[10,20]},
+      #   {"title":"Apr - Jun 2016","type":"b","values":[20,25]},
+      #   {"title":"Jul - Sep 2016","type":"c","values":[25,35]},
+      #   {"title":"Oct - Dec 2016","type":"d","values":[35,50]},
+      #   {"title":"Next year","type":"e","values":[50,10]}
+      # ]
+      def fast_preview_graph_data(lookback = 5, check_results = nil)
+        graph_data = { popularity: [], maintenance: [], maturity: [] } # replace with config
+
+        check_results.each_with_index do |check_result, index|
+          check_result.each do |check, results|
+            sum_up_checks(graph_data, check, results, index, lookback - index)
+          end
+        end
+        graph_data.map { |k, v| [k, MultiJson.dump(v)] }.to_h
+      end
+
+      def sum_up_checks(graph_data, check, results, index, offset)
+        gain = results[:gain]
+        graph_data[check] << {
+          title: last_quarters_bounds(offset),
+          type: results[:mark].downcase,
+          values: [gain, gain]
+        }
+
+        graph_data[check][index - 1][:values][1] = gain if index.positive?
+      end
+
+      def last_quarters_bounds(last_year_offset)
+        date = Time.current.utc - ((last_year_offset - 1) * 3).months
+
+        [date.beginning_of_quarter.strftime('%b'),
+         date.end_of_quarter.strftime('%b %Y')].join(' - ')
+      end
+    end
+  end
+end

data/lib/ossert/presenters.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+require 'ossert/presenters/project'
+
+module Ossert
+  # Public: Various classes and methods for presenting data.
+  module Presenters
+  end
+end

data/lib/ossert/project.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+module Ossert
+  class Project
+    include Ossert::Saveable
+
+    attr_accessor :name, :github_alias, :rubygems_alias,
+                  :community, :agility, :reference,
+                  :meta, :created_at, :updated_at
+
+    META_STUB = {
+      homepage_url: nil,
+      docs_url: nil,
+      wiki_url: nil,
+      source_url: nil,
+      issue_tracker_url: nil,
+      mailing_list_url: nil,
+      authors: nil,
+      top_10_contributors: [],
+      description: nil,
+      current_version: nil,
+      rubygems_url: nil,
+      github_url: nil
+    }.freeze
+
+    class << self
+      def fetch_all(name, reference = Ossert::Saveable::UNUSED_REFERENCE)
+        project = find_by_name(name, reference)
+
+        Ossert::Fetch.all project
+        project.prepare_time_bounds!
+        project.dump
+      end
+
+      def projects_by_reference
+        load_referenced.group_by(&:reference)
+      end
+    end
+
+    def grade_by_growing_classifier
+      raise unless Classifiers::Growing.current.ready?
+      Classifiers::Growing.current.grade(self)
+    end
+    alias grade_by_classifier grade_by_growing_classifier
+
+    def analyze_by_decisision_tree
+      raise unless Classifiers::DecisionTree.current.ready?
+      Classifiers::DecisionTree.current.check(self)
+    end
+
+    def initialize(name, github_alias = nil, rubygems_alias = nil, reference = nil)
+      @name = name.dup
+      @github_alias = github_alias
+      @rubygems_alias = (rubygems_alias || name).dup
+      @reference = reference.dup
+
+      @agility = Agility.new
+      @community = Community.new
+      @meta = META_STUB.dup
+    end
+
+    def assign_data(meta:, agility:, community:, created_at:, updated_at:)
+      @agility = agility
+      @community = community
+      @meta = meta
+      @created_at = created_at
+      @updated_at = updated_at
+    end
+
+    def decorated
+      @decorated ||= Ossert::Presenters::Project.new(self)
+    end
+
+    TIME_BOUNDS_CONFIG = {
+      base_value: {
+        start: nil,
+        end: nil
+      },
+      aggregation: {
+        start: :min,
+        end: :max
+      },
+      extended: {
+        start: nil,
+        end: nil
+      }
+    }.freeze
+
+    def prepare_time_bounds!(extended_start: nil, extended_end: nil)
+      config = TIME_BOUNDS_CONFIG.dup
+      config[:base_value][:start] = Time.now.utc
+      config[:base_value][:end] = 20.years.ago
+      config[:extended][:start] = (extended_start || Time.now.utc).to_datetime
+      config[:extended][:end] = (extended_end || 20.years.ago).to_datetime
+
+      agility.quarters.fullfill! && community.quarters.fullfill!
+
+      [:start, :end].map { |time_bound| time_bound_values(time_bound, config).to_date }
+    end
+
+    def time_bound_values(time_bound, config)
+      [
+        config[:base_value][time_bound], config[:extended][time_bound],
+        agility.quarters.send("#{time_bound}_date"), community.quarters.send("#{time_bound}_date")
+      ].send(config[:aggregation][time_bound])
+    end
+
+    def meta_to_json
+      MultiJson.dump(meta)
+    end
+
+    class BaseStore
+      attr_accessor :quarters, :total, :total_prediction, :quarter_prediction
+
+      def initialize(quarters: nil, total: nil)
+        @quarters = quarters || QuartersStore.new(self.class.quarter_stats_klass_name)
+        @total = total || ::Kernel.const_get(self.class.total_stats_klass_name).new
+      end
+    end
+
+    class Agility < BaseStore
+      class << self
+        def quarter_stats_klass_name
+          'Ossert::Stats::AgilityQuarter'
+        end
+
+        def total_stats_klass_name
+          'Ossert::Stats::AgilityTotal'
+        end
+      end
+    end
+
+    class Community < BaseStore
+      class << self
+        def quarter_stats_klass_name
+          'Ossert::Stats::CommunityQuarter'
+        end
+
+        def total_stats_klass_name
+          'Ossert::Stats::CommunityTotal'
+        end
+      end
+    end
+  end
+end