ossert 0.1.0

data/lib/ossert/stats/base.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+module Ossert
+  module Stats
+    class Base
+      class << self
+        attr_accessor :section, :section_type
+
+        def config
+          @config ||= ::Settings['stats'][section][section_type]
+        end
+
+        def attributes
+          @attributes ||= config['attributes']
+        end
+
+        def uniq_attributes
+          @uniq_attributes ||= config['uniq_attributes'].to_a
+        end
+
+        def absolute_attributes
+          @absolute_attributes ||= config['absolute_attributes'].to_a
+        end
+
+        def attributes_names
+          @attributes_names ||= attributes.keys
+        end
+
+        def metrics
+          @metrics ||= config['metrics']
+        end
+
+        def create_attributes_accessors
+          attr_accessor(*attributes_names)
+        end
+
+        def define_ints(*attributes)
+          Array.wrap(attributes).each do |metric|
+            define_method("#{metric}_int") { public_send(metric).to_i }
+          end
+        end
+
+        def define_counts(*attributes)
+          Array.wrap(attributes).each do |metric|
+            define_method("#{metric}_count") { public_send(metric).count }
+          end
+        end
+
+        def define_percent(attributes, default_value: 0)
+          attributes.to_h.each do |metric, total|
+            define_method("#{metric}_percent") do
+              total_count = public_send(total).count
+              return default_value if total_count.zero?
+
+              ((public_send(metric).count.to_d / total_count.to_d) * 100).round(2)
+            end
+          end
+        end
+      end
+
+      def <<(other_stats)
+        self.class.attributes_names.each do |attr|
+          next unless other_value = other_stats.instance_variable_get("@#{attr}")
+          new_value = other_value
+          new_value += instance_variable_get("@#{attr}") unless self.class.absolute_attributes.include?(attr)
+          new_value.uniq! if self.class.uniq_attributes.include?(attr)
+
+          instance_variable_set("@#{attr}", new_value)
+        end
+        self
+      end
+
+      def initialize
+        self.class.attributes.each do |var, type|
+          instance_variable_set("@#{var}", Kernel.const_get(type).new) if type
+        end
+      end
+
+      def median(values, default_value: 0)
+        values = Array(values).sort
+        return default_value if (count = values.count).zero?
+
+        middle_idx = values.count / 2
+        return values[middle_idx] if count.odd?
+
+        (values[middle_idx - 1] + values[middle_idx]) / 2
+      end
+
+      def metric_values
+        self.class.metrics.map { |metric| public_send(metric).to_f }
+      end
+
+      def metrics_to_hash
+        self.class.metrics.each_with_object({}) do |var, result|
+          value = send(var)
+          value.uniq! if self.class.uniq_attributes.include?(var)
+          result[var] = value
+        end
+      end
+
+      def to_hash
+        self.class.attributes_names.each_with_object({}) do |var, result|
+          value = send(var)
+          value.uniq! if self.class.uniq_attributes.include?(var)
+          result[var] = value
+        end
+      end
+
+      def to_json
+        MultiJson.dump(self)
+      end
+    end
+  end
+end

data/lib/ossert/stats/community_quarter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+module Ossert
+  module Stats
+    class CommunityQuarter < Base
+      self.section = 'community'
+      self.section_type = 'quarter'
+      create_attributes_accessors
+
+      define_counts(
+        :users_creating_issues, :users_commenting_issues, :users_creating_pr,
+        :users_commenting_pr, :contributors, :stargazers, :forks,
+        :users_involved
+      )
+
+      def users_involved_no_stars_count
+        (users_involved - stargazers).count
+      end
+
+      def total_downloads_count
+        delta_downloads
+      end
+
+      def download_divergence
+        (delta_downloads.to_f / (1 + total_downloads.to_f)) * 100.0
+      end
+    end
+  end
+end

data/lib/ossert/stats/community_total.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+module Ossert
+  module Stats
+    class CommunityTotal < Base
+      self.section = 'community'
+      self.section_type = 'total'
+      create_attributes_accessors
+
+      define_counts(
+        :users_creating_issues, :users_commenting_issues, :users_creating_pr,
+        :users_commenting_pr, :contributors, :watchers, :stargazers, :forks,
+        :users_involved, :dependants
+      )
+
+      def users_involved_no_stars_count
+        (users_involved - stargazers).count
+      end
+
+      def total_downloads_count
+        total_downloads
+      end
+    end
+  end
+end

data/lib/ossert/stats.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+require 'ossert/stats/base'
+require 'ossert/stats/agility_total'
+require 'ossert/stats/agility_quarter'
+require 'ossert/stats/community_total'
+require 'ossert/stats/community_quarter'
+
+module Ossert
+  module Stats
+    PER_QUARTER_TOO_LONG = (5.years / 1.day).to_i
+    PER_YEAR_TOO_LONG = PER_QUARTER_TOO_LONG / 4
+
+    def guess_section_by_metric(metric)
+      found_section = :not_found
+      section_by_metric.each do |section, metrics|
+        next unless metrics.include? metric
+        found_section = section
+        break
+      end
+      found_section
+    end
+    module_function :guess_section_by_metric
+
+    def section_by_metric
+      @section_by_metric ||= {
+        agility: AgilityTotal.metrics + AgilityQuarter.metrics,
+        community: CommunityTotal.metrics + CommunityQuarter.metrics
+      }
+    end
+    module_function :section_by_metric
+  end
+end

data/lib/ossert/tasks/database.rake

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+namespace :db do
+  require 'sequel'
+  Sequel.extension :migration
+
+  namespace :test do
+    task :prepare do
+      test_database_url = ENV.fetch('TEST_DATABASE_URL')
+      database_name = test_database_url.split('/').last
+
+      DB = Sequel.connect(test_database_url)
+
+      sh "dropdb #{database_name}" do
+        # Ignore errors
+      end
+
+      sh "createdb #{database_name}" do
+        # Ignore errors
+      end
+
+      Sequel::Migrator.run(DB, File.expand_path('../../../../db/migrate', __FILE__))
+      Rake::Task['db:version'].execute
+    end
+  end
+
+  task :migrate => :load_config do
+    Sequel::Migrator.run(DB, File.expand_path('../../../../db/migrate', __FILE__))
+    Rake::Task['db:version'].execute
+  end
+
+  desc 'Prints current schema version'
+  task :version do
+    version = if DB.tables.include?(:schema_info)
+                DB[:schema_info].first[:version]
+              end || 0
+
+    puts "Schema Version: #{version}"
+  end
+
+  task :load_config do
+    DB = Sequel.connect(ENV.fetch('DATABASE_URL'))
+  end
+
+  desc 'Create the database, load the schema, and initialize with the seed data (db:reset to also drop the db first)'
+  task :setup do
+    Rake::Task['db:create'].invoke
+    Rake::Task['db:load_config'].invoke
+
+    Sequel::Migrator.run(DB, File.expand_path('../../../../db/migrate', __FILE__))
+    Rake::Task['db:version'].execute
+  end
+
+  task :drop do
+    sh "dropdb #{ENV.fetch('DATABASE_URL').split('/').last}" do
+      # Ignore errors
+    end
+  end
+
+  task :create do
+    sh "createdb #{ENV.fetch('DATABASE_URL').split('/').last}" do
+      # Ignore errors
+    end
+  end
+
+  desc 'Dumps the database to backups'
+  task :dump, [:fmt] do |_, args|
+    dump_fmt = args.fmt || 'c' # or 'p', 't', 'd'
+    dump_sfx = suffix_for_format dump_fmt
+    backup_dir = backup_directory true
+    cmd = nil
+    with_config do |app, db_url|
+      file_name = Time.now.strftime('%Y%m%d%H%M%S') + '_' + app + '_db.' + dump_sfx
+      cmd = "pg_dump #{db_url} --no-owner --no-acl -F #{dump_fmt} -v -f #{backup_dir}/#{file_name}"
+    end
+    puts cmd
+    sh cmd do
+      # Ignore errors
+    end
+  end
+
+  desc 'Show the existing database backups'
+  task :list_backups do
+    backup_dir = backup_directory
+    puts backup_dir.to_s
+    exec "/bin/ls -lht #{backup_dir}"
+  end
+
+  desc 'Restores the database from a backup using PATTERN'
+  task :restore, [:pat] do |_, args|
+    puts 'Please pass a pattern to the task' unless args.pat.present?
+    cmd = nil
+    with_config do |_, db_url|
+      cmd = command_for_files args.pat, db_url
+    end
+    unless cmd.nil?
+      Rake::Task['db:drop'].invoke
+      Rake::Task['db:create'].invoke
+      puts cmd
+      exec "#{cmd} || exit 0"
+    end
+  end
+
+  def command_for_files(pattern, db_url)
+    files = Dir.glob("#{backup_directory}/*#{pattern}*")
+    case files.size
+    when 0
+      puts "No backups found for the pattern '#{pattern}'"
+    when 1
+      command_for_file files.first, db_url
+    else
+      puts "Too many files match the pattern '#{pattern}': #{files.join("\n ")} "
+      puts 'Try a more specific pattern'
+    end
+  end
+
+  def command_for_file(file, db_url)
+    return puts("No recognized dump file suffix: #{file}") unless (fmt = format_for_file(file)).present?
+    "pg_restore -d '#{db_url}' -F #{fmt} -v -c #{file}"
+  end
+
+  namespace :restore do
+    desc 'Restores the database from latest backup'
+    task :last do
+      cmd = nil
+      with_config do |_, db_url|
+        file = Dir.glob("#{backup_directory}/*").max_by { |f| File.mtime(f) }
+        if file
+          fmt = format_for_file file
+          if fmt.nil?
+            puts "No recognized dump file suffix: #{file}"
+          else
+            cmd = "pg_restore -d '#{db_url}' -F #{fmt} -v #{file}"
+          end
+        else
+          puts 'No backups found'
+        end
+      end
+      unless cmd.nil?
+        Rake::Task['db:drop'].invoke
+        Rake::Task['db:create'].invoke
+        puts cmd
+        exec "#{cmd} || exit 0"
+      end
+    end
+  end
+
+  private
+
+  def suffix_for_format(suffix)
+    case suffix
+    when 'c' then 'dump'
+    when 'p' then 'sql'
+    when 't' then 'tar'
+    when 'd' then 'dir'
+    end
+  end
+
+  def format_for_file(file)
+    case file
+    when /\.dump$/ then 'c'
+    when /\.sql$/  then 'p'
+    when /\.dir$/  then 'd'
+    when /\.tar$/  then 't'
+    end
+  end
+
+  def backup_directory(create = false)
+    backup_dir = 'db/backups'
+    if create && !Dir.exist?(backup_dir)
+      puts "Creating #{backup_dir} .."
+      FileUtils.mkdir_p(backup_dir)
+    end
+    backup_dir
+  end
+
+  def with_config
+    yield 'ossert', ENV.fetch('DATABASE_URL')
+  end
+end

data/lib/ossert/tasks/ossert.rake

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+namespace :ossert do
+  namespace :cache do
+    desc 'Reset data cache'
+    task :reset do
+      ::Ossert.init
+      ::Classifier.dataset.delete
+      Ossert::Classifiers::Growing.new.train
+      true
+    end
+  end
+
+  desc 'Collect reference projects'
+  task :collect_referencies do
+    puts 'Run collecting process'
+    time = Benchmark.realtime do
+      ::Ossert.init
+      ::Project.db.transaction do
+        Ossert::Project.cleanup_referencies!
+        reference_projects = Ossert::Reference.prepare_projects!
+        Ossert::Reference.process_references(reference_projects)
+      end
+    end
+
+    puts "Collecting process finished in #{time.round(3)} sec."
+  end
+
+  desc 'Invoke data updates for stale projects'
+  task :refresh_data do
+    require './config/sidekiq.rb'
+    Ossert::Workers::RefreshFetch.perform_async
+  end
+
+  desc 'Add or replace project name exception'
+  task :exception, [:name, :github_name] do |_, args|
+    raise 'Arguments name and GitHub name expected' unless args.name.present? && args.github_name.present?
+    exceptions_repo = ExceptionsRepo.new(Ossert.rom)
+    if exceptions_repo[args.name]
+      exceptions_repo.update(
+        args.name,
+        name: args.name,
+        github_name: args.github_name
+      )
+    else
+      exceptions_repo.create(
+        name: args.name,
+        github_name: args.github_name
+      )
+    end
+    puts "Exception '#{args.name}' => '#{args.github_name}' saved!"
+  end
+end

data/lib/ossert/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Ossert
+  VERSION = '0.1.0'
+end

data/lib/ossert/workers/fetch.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+module Ossert
+  module Workers
+    class Fetch
+      include Sidekiq::Worker
+      include Process
+      sidekiq_options unique: :until_executed,
+                      unique_expiration: 1.hour,
+                      retry: 3
+
+      def perform(name, reference = Ossert::Saveable::UNUSED_REFERENCE)
+        puts "Fetching data for: '#{name}' (ref: #{reference})"
+        pid = fork do
+          Ossert.init
+          Ossert::Project.fetch_all(name, reference)
+        end
+        waitpid(pid)
+      end
+    end
+  end
+end

data/lib/ossert/workers/fetch_bestgems_page.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+module Ossert
+  module Workers
+    class FetchBestgemsPage
+      include Sidekiq::Worker
+      include Process
+
+      sidekiq_options retry: 3,
+                      unique: :until_executed
+
+      def perform(pages, type = :total)
+        pages = Array(pages)
+        bestgems_page_processor = Kernel.const_get(
+          "Ossert::Fetch::Bestgems#{type.to_s.capitalize}Stat"
+        )
+        pages.each do |page|
+          puts "Processing Bestgems page: '#{page}'"
+          bestgems_page_processor.process_page(page) do |_, _, gem_name|
+            puts "Processing project: '#{gem_name}'"
+            pid = fork do
+              Ossert.init
+              next(puts("Skipping project: '#{gem_name}'")) if Ossert::Project.exist?(gem_name)
+              Ossert::Project.fetch_all(gem_name)
+              sleep(10)
+            end
+            waitpid(pid)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ossert/workers/refresh_fetch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+module Ossert
+  module Workers
+    class RefreshFetch
+      include Sidekiq::Worker
+      include Process
+      sidekiq_options unique: :until_executed,
+                      unique_expiration: 1.hour,
+                      retry: 3
+
+      def perform
+        pid = fork do
+          Ossert.init
+          ::Project.select(:name, :reference).where('updated_at < ?', 1.week.ago).paged_each do |project|
+            Ossert::Workers::Fetch.perform_async(project.name, project.reference)
+          end
+        end
+        waitpid(pid)
+      end
+    end
+  end
+end

data/lib/ossert/workers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+require 'sidekiq'
+require 'sidekiq-unique-jobs'
+require 'ossert/workers/fetch'
+require 'ossert/workers/fetch_bestgems_page'
+require 'ossert/workers/refresh_fetch'
+
+module Ossert
+  module Workers
+  end
+end

data/lib/ossert.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require 'ossert/version'
+require 'sequel'
+require 'active_support/all' # remove later, we use only quarters and index_by here
+require 'json'
+require 'oj'
+
+require 'ossert/config'
+Ossert::Config.load(:stats, :classifiers, :translations, :descriptions)
+
+require 'ossert/stats'
+require 'ossert/quarters_store'
+require 'ossert/saveable'
+require 'ossert/presenters'
+require 'ossert/project'
+require 'ossert/fetch'
+require 'ossert/reference'
+require 'ossert/classifiers'
+require 'ossert/workers'
+
+module Ossert
+  def init(database_url = nil)
+    Sequel.connect(database_url || ENV.fetch('DATABASE_URL'))
+    require 'ossert/repositories'
+  end
+  module_function :init
+
+  def description(key)
+    descriptions.fetch(key.to_s, "Description for '#{key}' - not found")
+  end
+  alias descr description
+  module_function :descr
+
+  def descriptions
+    @descriptions ||= ::Settings['descriptions']
+  end
+  module_function :descriptions
+
+  def translate(key)
+    translations.fetch(key.to_s, "Translation for '#{key}' - not found")
+  end
+  alias t translate
+  module_function :t
+
+  def translations
+    @translations ||= ::Settings['translations']
+  end
+  module_function :translations
+
+  NO_GITHUB_NAME = '__unknown__'
+
+  class TooLong
+    def self.new
+      Ossert::Stats::PER_YEAR_TOO_LONG
+    end
+  end
+
+  class Zero
+    def self.new
+      0
+    end
+  end
+end

data/ossert.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+# frozen_string_literal: true
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ossert/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'ossert'
+  spec.version       = Ossert::VERSION
+  spec.authors       = ['Sergey Dolganov']
+  spec.email         = ['sclinede@gmail.com']
+
+  spec.summary       = 'Write a short summary, because Rubygems requires one.'
+  spec.description   = 'Write a longer description or delete this line.'
+  spec.homepage      = 'https://github.com/sclinede/ossert'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'octokit', '~> 4.0'
+
+  spec.add_runtime_dependency 'redis'
+  spec.add_runtime_dependency 'redis-namespace'
+  spec.add_runtime_dependency 'sidekiq'
+  spec.add_runtime_dependency 'sidekiq-unique-jobs'
+
+  spec.add_runtime_dependency 'activesupport' # TODO: remove dependency later
+  spec.add_runtime_dependency 'oj'
+  spec.add_runtime_dependency 'multi_json'
+  spec.add_runtime_dependency 'sequel'
+  spec.add_runtime_dependency 'pg'
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'graphr'
+  spec.add_runtime_dependency 'rake', '~> 10.0'
+
+  spec.add_development_dependency 'memory_profiler'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'webmock'
+  spec.add_development_dependency 'vcr'
+end