ossert 0.1.0

data/lib/ossert/quarters_store.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+module Ossert
+  # Public: Class for data divided by quarters. Each quarter instantiates some statistics class.
+  # Contains methods for quarters calculations, such as grouping, preview and other.
+  class QuartersStore
+    attr_reader :quarters, :data_klass, :start_date, :end_date
+
+    # Public: Instantiate QuarterStore
+    #
+    # data_klass - the Object for quarter data storage, to be compatable it
+    #              should implement:
+    #                - class method #metrics returns Array of metric names;
+    #                - instance method #metric_values returns values of metrics
+    #                  in same order.
+    #
+    # Returns nothing.
+    def initialize(data_klass_name)
+      @data_klass_name = data_klass_name
+      @quarters = {}
+      @start_date = Time.now
+      @end_date = Time.now
+    end
+
+    def data_klass
+      @data_klass ||= Kernel.const_get(@data_klass_name)
+    end
+
+    # Public: Strict fetch of quarter for given date
+    #
+    # date - the String, Numeric or DateTime to seek begining of quarter for.
+    #
+    # Returns quarter Object or KeyError will be raised.
+    def fetch(date)
+      quarters.fetch date_to_start(date)
+    end
+
+    # Public: Find or create quarter for given date.
+    #
+    # date - the String, Numeric or DateTime to seek begining of quarter for.
+    #
+    # Returns quarter Object.
+    def find_or_create(date)
+      quarters[date_to_start(date)] ||= data_klass.new
+    end
+    alias [] find_or_create
+
+    # Public: Find closest begining of quarter for given date.
+    #
+    # date - the String, Numeric or DateTime to seek begining of quarter for.
+    #
+    # Returns begining of quarter DateTime.
+    def date_to_start(date)
+      if date.is_a? String
+        # Alternative, but more expensive: DateTime.parse(value).beginning_of_quarter.to_i
+        DateTime.new(*date.split('-').map(&:to_i)).beginning_of_quarter.to_i
+      else
+        Time.at(date).to_date.to_time(:utc).beginning_of_quarter.to_i
+      end
+    end
+
+    # Public: Prepare quarters to preview.
+    #
+    # Returns sorted Hash of quarter date and its data.
+    def preview
+      quarters.sort.map { |unix_timestamp, quarter| [Time.at(unix_timestamp), quarter] }.to_h
+    end
+
+    # Public: Get quarters metric values aggregated for last year.
+    #
+    # offset - the Numeric (default: 1) in quarters for offset of when "last year" should ends
+    #
+    # Returns Array of quarter metric values aggregated for last year.
+    def last_year_data(offset = 1)
+      last_year_as_hash(offset).values
+    end
+
+    # Public: Get quarters metric values aggregated for last year.
+    #
+    # offset - the Numeric (default: 1) in quarters for offset of when "last year" should ends
+    #
+    # Returns Hash of quarter metrics and its values aggregated for last year.
+    def last_year_as_hash(offset = 1)
+      data_klass.metrics.zip(aggregated_quarter(offset).metric_values).to_h
+    end
+
+    # Public: Generate aggregated quarter object for last year.
+    #
+    # offset - the Numeric (default: 1) in quarters for offset of when "last year" should ends
+    #
+    # Returns quarter Object with attributes aggregated for last year.
+    def aggregated_quarter(offset = 1)
+      last_quarters = quarters.sort.last(4 + offset).take(4)
+      last_quarters.inject(data_klass.new) do |acc, (_, quarter)|
+        acc << quarter
+      end
+    end
+
+    # Public: Fill quarter bounds and wholes in periods from first to last quarter.
+    # It will assign @start_date and @end_date of QuarterStore instance.
+    # Should be called after all data is gathered and we ready for data presentation.
+    #
+    # Returns nothing.
+    def fullfill!
+      return if quarters.empty?
+
+      periods_range = with_quarters_dates do |period|
+        find_or_create Time.at(period)
+      end
+
+      @start_date = Time.at(periods_range.first)
+      @end_date = Time.at(periods_range.last)
+    end
+
+    # Public: Iterate (and yields) through quarter dates in ascending order
+    #
+    # Yields the Numeric UNIX-timestamp inside of quarter
+    #
+    # Returns Range of quarters dates
+    def with_quarters_dates
+      sorted_quarters = quarters.keys.sort
+      (sorted_quarters.first..sorted_quarters.last).step(93.days) { |period| yield(period) }
+    end
+
+    # Public: Iterate (and yields) through quarters in descending order
+    #
+    # Yields the Numeric UNIX-timestamp beginning of quarter
+    #        the Object for quarter data
+    #
+    # Returns Array of sorted pairs of time and quarter object.
+    def reverse_each_sorted
+      quarters.sort.reverse.map { |time, quarter| yield(time, quarter) }
+    end
+
+    # Public: Iterate (and yields) through quarters in ascending order
+    #
+    # Yields the Numeric UNIX-timestamp beginning of quarter
+    #        the Object for quarter data
+    #
+    # Returns Array of sorted pairs of time and quarter object.
+    def each_sorted
+      quarters.sort.map { |time, quarter| yield(time, quarter) }
+    end
+
+    # Public: Generate Hash for current data structure.
+    #         Keys are UNIX-timestamps (beginning of each quarter),
+    #         values are quarter objects explicitly converted to Hash.
+    #
+    # Returns Hash.
+    def to_hash
+      quarters.each_with_object({}) do |(time, quarter), result|
+        result[time] = quarter.to_hash
+      end
+    end
+
+    # Public: Generate JSON for current data structure.
+    #         Keys are UNIX-timestamps (beginning of each quarter),
+    #         values are quarter objects explicitly converted to Hash.
+    #
+    # Returns String contains valid JSON.
+    def to_json
+      MultiJson.dump(self)
+    end
+  end
+end

data/lib/ossert/rake_tasks.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require 'rake'
+require 'sequel'
+
+load 'ossert/tasks/database.rake'
+load 'ossert/tasks/ossert.rake'

data/lib/ossert/reference.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+module Ossert
+  module Reference
+    def prepare_projects!
+      references = %w(A B C D E).map { |e| Kernel.const_get("Ossert::Reference::Class#{e}").new }
+      references.each(&:prepare_projects!)
+      references
+    end
+    module_function :prepare_projects!
+
+    def process_references(references)
+      require './config/sidekiq'
+      Array(references).each do |reference|
+        reference.project_names.each_with_object(reference.class.name.demodulize) do |project_name, klass|
+          if Ossert::Project.exist?(project_name)
+            project = Ossert::Project.load_by_name(project_name)
+            project.reference = klass
+            project.dump
+          else
+            Ossert::Workers::Fetch.perform_async(project_name, klass)
+          end
+        end
+      end
+    end
+    module_function :process_references
+
+    class Base
+      CLASSES = %w(ClassA ClassB ClassC ClassD ClassE).freeze
+
+      attr_reader :representative, :pages, :project_names
+
+      def initialize(representative, pages)
+        @representative = representative
+        @pages = pages
+        @project_names = Set.new
+        # 20 each page, total 5907 pages
+      end
+
+      PER_PAGE = 20
+
+      def prepare_projects!
+        puts "Processing #{self.class.name}"
+        all_pages = pages.to_a.shuffle
+        all_projects = {}
+        (representative / PER_PAGE).times do
+          current_page = all_pages.pop
+          Fetch::BestgemsDailyStat.process_page(current_page) do |rank, downloads, name|
+            all_projects[name] = { rank: rank, downloads: downloads }
+          end
+        end
+
+        # @project_names.merge all_projects.keys.shuffle.first(representative)
+        @project_names.merge all_projects.sort_by { |_, info| info[:downloads] }.to_h.keys.last(representative)
+      end
+    end
+
+    class ClassA < Base
+      def initialize
+        super(50, 1..10)
+      end
+    end
+
+    class ClassB < Base
+      def initialize
+        super(50, 11..100)
+      end
+    end
+
+    class ClassC < Base
+      def initialize
+        super(50, 101..250)
+      end
+    end
+
+    class ClassD < Base
+      def initialize
+        super(50, 251..500)
+      end
+    end
+
+    class ClassE < Base
+      def initialize
+        super(50, 501..2500)
+      end
+    end
+  end
+end

data/lib/ossert/repositories.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+require 'oj'
+require 'multi_json'
+
+class NameException < Sequel::Model(:exceptions)
+  set_primary_key [:name]
+  class << self
+    alias by_name []
+  end
+end
+::NameException.unrestrict_primary_key
+
+class Classifier < Sequel::Model
+  set_primary_key [:section]
+  def self.actual?
+    where('updated_at > ?', 1.month.ago).count.positive?
+  end
+end
+::Classifier.unrestrict_primary_key
+
+class Project < Sequel::Model
+  set_primary_key [:name]
+
+  def_dataset_method(:random) do |count|
+    where('random() < ?', count * 0.05).limit(count)
+  end
+
+  class << self
+    def later_than(id)
+      where('id >= ?', id)
+    end
+
+    def referenced
+      where('reference <> ?', Ossert::Saveable::UNUSED_REFERENCE)
+    end
+  end
+
+  class Unpacker
+    def initialize(stored_project)
+      @stored_project = stored_project
+    end
+
+    def self.process(stored_project)
+      new(stored_project).process
+    end
+
+    def process
+      [:agility, :community].each_with_object(process_meta) do |stats_type, result|
+        result[stats_type] = factory_project_stats(stats_type).new(
+          [Total, Quarter].each_with_object({}) do |unpacker_type, stats_result|
+            section_unpacker = unpacker_type.new(@stored_project, stats_type)
+            stats_result[section_unpacker.section] = section_unpacker.process
+          end
+        )
+      end
+    ensure
+      @stored_project = nil
+    end
+
+    private
+
+    def process_meta(result = {})
+      result = {
+        created_at: @stored_project.created_at,
+        updated_at: @stored_project.updated_at
+      }
+      result[:meta] = if @stored_project.meta_data.present?
+                        MultiJson.load(@stored_project.meta_data)
+                      else
+                        {}
+                      end
+
+      result
+    end
+
+    def factory_project_stats(stats_type)
+      Kernel.const_get "Ossert::Project::#{stats_type.to_s.capitalize}"
+    end
+
+    class Base
+      def initialize(stored_project, stats_type)
+        @stats_type = stats_type
+        @stored_project = stored_project
+      end
+
+      def coerce_value(value)
+        return DateTime.parse(value)
+      rescue
+        value
+      end
+
+      def stored_data
+        @stored_project.send("#{@stats_type}_#{section}_data")
+      end
+    end
+
+    class Total < Base
+      def section
+        :total
+      end
+
+      def new_stats_object
+        Kernel.const_get("Ossert::Stats::#{@stats_type.capitalize}Total").new
+      end
+
+      def process
+        MultiJson.load(
+          stored_data
+        ).each_with_object(new_stats_object) do |(metric, value), stats_object|
+          stats_object.send "#{metric}=", coerce_value(value)
+        end
+      end
+    end
+
+    class Quarter < Base
+      def section
+        :quarters
+      end
+
+      def new_stats_object
+        Ossert::QuartersStore.new(
+          "Ossert::Stats::#{@stats_type.capitalize}Quarter"
+        )
+      end
+
+      def process
+        MultiJson.load(
+          stored_data
+        ).each_with_object(new_stats_object) do |(time, metrics), quarter_store|
+          metrics.each_with_object(quarter_store[time.to_i]) do |(metric, value), quarter|
+            quarter.send "#{metric}=", coerce_value(value)
+          end
+        end
+      end
+    end
+  end
+end
+::Project.unrestrict_primary_key

data/lib/ossert/saveable.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+module Ossert
+  module Saveable
+    UNUSED_REFERENCE = 'unused'
+    ATTRIBUTE_EXTRACT_VALUE_MAP = {
+      agility_total_data: ->(project) { project.agility.total.to_json },
+      agility_quarters_data: ->(project) { project.agility.quarters.to_json },
+      community_total_data: ->(project) { project.community.total.to_json },
+      community_quarters_data: ->(project) { project.community.quarters.to_json },
+      meta_data: ->(project) { project.meta_to_json }
+    }.freeze
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def dump_attribute(attriibute)
+      attriibute = attriibute.to_sym
+      value = ATTRIBUTE_EXTRACT_VALUE_MAP.fetch(attriibute).call(self)
+
+      raise 'Not saved yet, sorry!' unless (found_project = ::Project.find(name: name))
+      found_project.update(name, attriibute => value, updated_at: Time.now.utc)
+      nil
+    end
+
+    def dump
+      validate!
+      if (found_project = ::Project.find(name: name))
+        found_project.update(attributes.merge(updated_at: Time.now.utc))
+      else
+        ::Project.create(attributes)
+      end
+      nil
+    end
+
+    def valid?
+      [name, github_alias, rubygems_alias].all?(&:present?)
+    end
+
+    class RecordInvalid < StandardError
+      attr_reader :message
+      def initialize(*)
+        super
+        @message = "Couldn't save project. Validation failed!"
+      end
+    end
+
+    def validate!
+      raise RecordInvalid.new unless valid?
+    end
+
+    def attributes
+      meta_attributes.merge(data_attributes)
+    end
+
+    def meta_attributes
+      {
+        name: name,
+        github_name: github_alias,
+        rubygems_name: rubygems_alias,
+        reference: reference
+      }
+    end
+
+    def data_attributes
+      {
+        meta_data: meta_to_json,
+        agility_total_data: agility.total.to_json,
+        agility_quarters_data: agility.quarters.to_json,
+        community_total_data: community.total.to_json,
+        community_quarters_data: community.quarters.to_json
+      }
+    end
+
+    def without_github_data?
+      github_alias == NO_GITHUB_NAME
+    end
+
+    module ClassMethods
+      def exist?(name)
+        ::Project.filter(name: name).get(:name).present?
+      end
+
+      def random_top(count = 10)
+        ::Project.where(reference: %w(ClassA ClassB)).random(count)
+      end
+
+      def random(count = 10)
+        ::Project.dataset.random(count)
+      end
+
+      def find_by_name(name, reference = Ossert::Saveable::UNUSED_REFERENCE)
+        if (name_exception = ::NameException.find(name: name))
+          new(name, name_exception.github_name, name, reference)
+        else
+          new(name, nil, name, reference)
+        end
+      end
+
+      def load_by_name(name)
+        stored_prj = ::Project.find(name: name)
+        deserialize(stored_prj) if stored_prj
+      end
+
+      def load_referenced
+        ::Project.referenced.map do |stored_prj|
+          deserialize(stored_prj)
+        end
+      end
+
+      def load_later_than(id)
+        ::Project.later_than(id).map do |stored_prj|
+          deserialize(stored_prj)
+        end
+      end
+
+      def cleanup_referencies!
+        ::Project.dataset.update(reference: UNUSED_REFERENCE)
+      end
+
+      def load_all
+        ::Project.paged_each.map do |stored_prj|
+          deserialize(stored_prj)
+        end
+      end
+
+      def yield_all
+        ::Project.paged_each do |stored_prj|
+          yield deserialize(stored_prj)
+        end
+      end
+
+      def dump
+        projects.each(&:dump)
+      end
+
+      private
+
+      def deserialize(stored_project)
+        project = Ossert::Project.new(
+          stored_project.name,
+          stored_project.github_name,
+          stored_project.rubygems_name,
+          stored_project.reference
+        )
+        project.assign_data(
+          ::Project::Unpacker.process(stored_project)
+        )
+        project
+      end
+    end
+  end
+end

data/lib/ossert/stats/agility_quarter.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+module Ossert
+  module Stats
+    class AgilityQuarter < Base
+      self.section = 'agility'
+      self.section_type = 'quarter'
+      create_attributes_accessors
+
+      define_percent(
+        issues_active: :issues_all,
+        issues_closed: :issues_all,
+        pr_active: :pr_all,
+        pr_closed: :pr_all,
+        pr_merged: :pr_all,
+        default_value: 100.0
+      )
+
+      define_counts(
+        :issues_active, :pr_active, :issues_closed, :issues_actual,
+        :pr_closed, :issues_all, :pr_all, :pr_actual
+      )
+
+      def issues_processed_in_median
+        median(issues_processed_in_days, default_value: PER_QUARTER_TOO_LONG)
+      end
+
+      def issues_processed_in_avg
+        return PER_QUARTER_TOO_LONG if (count = Array(issues_processed_in_days).size).zero?
+        issues_processed_in_days.sum / count
+      end
+
+      def pr_processed_in_median
+        median(pr_processed_in_days, default_value: PER_QUARTER_TOO_LONG)
+      end
+
+      def pr_processed_in_avg
+        return PER_QUARTER_TOO_LONG if (count = Array(pr_processed_in_days).size).zero?
+        pr_processed_in_days.sum / count
+      end
+
+      def issues_active
+        (issues_open | issues_actual) - issues_closed
+      end
+
+      def issues_all
+        (issues_open | issues_closed | issues_actual)
+      end
+
+      def pr_active
+        (pr_open | pr_actual) - pr_closed
+      end
+
+      def pr_all
+        (pr_open | pr_closed | pr_actual)
+      end
+
+      def releases_count
+        [releases_total_rg.count, releases_total_gh.count].max
+      end
+    end
+  end
+end

data/lib/ossert/stats/agility_total.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+module Ossert
+  module Stats
+    class AgilityTotal < Base
+      self.section = 'agility'
+      self.section_type = 'total'
+      create_attributes_accessors
+
+      define_percent(
+        issues_active: :issues_all,
+        issues_closed: :issues_all,
+        issues_non_owner: :issues_all,
+        issues_with_contrib_comments: :issues_all,
+        pr_active: :pr_all,
+        pr_closed: :pr_all,
+        pr_non_owner: :pr_all,
+        pr_with_contrib_comments: :pr_all,
+        default_value: 100.0
+      )
+
+      define_ints(
+        :first_pr_date, :last_pr_date, :first_issue_date,
+        :last_issue_date, :last_release_date
+      )
+
+      define_counts(:issues_all, :pr_all, :stale_branches, :dependencies)
+
+      def commits_count_since_last_release_count
+        commits_count_since_last_release.is_a?(Array) ? 0 : commits_count_since_last_release
+      end
+
+      def issues_active
+        issues_open - issues_closed
+      end
+
+      def issues_all
+        issues_open + issues_closed
+      end
+
+      def pr_active
+        pr_open - pr_closed
+      end
+
+      def pr_all
+        pr_open + pr_closed
+      end
+
+      def last_changed
+        [last_pr_date.presence, last_issue_date.presence].compact.max || 10.years.ago
+      end
+
+      def life_period
+        last_change = [last_pr_date.presence, last_issue_date.presence].compact.max
+        return 0 unless last_change
+
+        first_change = [first_pr_date, first_issue_date].compact.min
+        return 0 unless first_change
+
+        (last_change - first_change).to_i
+      end
+
+      def life_period_months
+        life_period / 1.month
+      end
+
+      def releases_count
+        [releases_total_rg.count, releases_total_gh.count].max
+      end
+    end
+  end
+end