repocrawler 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6dcd4f22099b5b0f090b5062695b9db581cf4b7b
+  data.tar.gz: 4ccaaaa819b2b7eedab27c676be626e957360377
+SHA512:
+  metadata.gz: 68b6670db01659e7ba8cd60f175141cf29e03928fc8dceae120516af0004e63dc673e45d90a0c43db1effcb0760a9a1c1d5cc85b19d612d91f71e610e91b4f2f
+  data.tar.gz: 2f8a36b9e827ec7ceea8c4fd3c48153723e034dd489a3a76be7fafd575ddb087a81551dc3b3c3f1eedb1e33f61e2fd1ad7836105aad30937b160d1c972483d68

data/.gitignore

@@ -0,0 +1,3 @@
+data/*
+config/local.rb
+*.gem

data/Gemfile

@@ -0,0 +1,9 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+gem "gems"
+gem "github_api"
+gem "httparty"
+gem 'mongo', '~> 2.1'
+gem 'configuration'
+gem 'nokogiri'

data/Gemfile.lock

@@ -0,0 +1,52 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    bson (3.2.6)
+    configuration (1.3.4)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    gems (0.8.3)
+    github_api (0.12.4)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.4)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.6)
+      oauth2
+    hashie (3.4.3)
+    httparty (0.13.7)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    json (1.8.3)
+    jwt (1.5.2)
+    mini_portile (0.6.2)
+    mongo (2.1.2)
+      bson (~> 3.0)
+    multi_json (1.11.2)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.6.4)
+    thread_safe (0.3.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  configuration
+  gems
+  github_api
+  httparty
+  mongo (~> 2.1)
+  nokogiri

data/README.md

@@ -0,0 +1,9 @@
+# Description
+A simple program to get the needed data from a single gem, in this case is oga.
+
+# Run
+  1. ```bundle install``` for no reason.
+  2. copy the ```config/local.example.rb``` to ```config/local.rb```.
+  3. replace the ```STACKOVERFLOW_TOKEN```, ```TOKEN```, ```GITHUB_ACCOUNT```, and ```USER_AGENT``` in the ```config/local.rb``` with yours.
+  4. run your local Mongodb with command ```sudo mongod```.
+  5. run ```ruby repos_exec.rb``` to start the collecting process.

data/bin/repocrawler

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+require 'repocrawler'
+
+`stty -echo`
+print 'Github Password (not stored): '
+github_password = gets.chomp
+puts "\n"
+print 'Name of the gem: '
+gem_name = gets.chomp
+puts "\n"
+print 'Name of the repo: '
+repo_name = gets.chomp
+puts "\n"
+print 'Username of the repo: '
+repo_username = gets.chomp
+puts "\n"
+`stty echo`
+puts ""
+
+client = Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'gems_info')
+
+github = Repos::GithubData.new(repo_username, repo_name, github_password)
+rubygems = Repos::RubyGemsData.new(gem_name)
+ruby_toolbox = Repos::RubyToolBoxData.new(gem_name)
+stackoverflow = Repos::StackOverflow.new(gem_name)
+
+last_year_commit_activity = github.get_last_year_commit_activity
+contributors = github.get_contributors
+total_commits = github.get_total_commits
+forks = github.get_forks
+stars = github.get_stars
+issues = github.get_issues
+issues_info = github.get_issues_info
+last_commits_days = github.get_last_commits_days
+readme_word_count = github.get_readme_word_count
+
+version_downloads = rubygems.get_version_downloads
+version_downloads_trend = rubygems.get_version_downloads_trend
+
+dependencies = rubygems.get_dependencies
+total_downloads = rubygems.get_total_downloads
+
+ranking = ruby_toolbox.get_ranking
+
+#stackoverflow info
+questions, questions_word_count = stackoverflow.get_questions
+
+# aggregate the data
+gem_info = {
+  'name'  => gem_name,
+  'repo_name' => repo_name,
+  'repo_username' => repo_username,
+  'total_downloads' => total_downloads,
+  'version_downloads' => version_downloads,
+  'version_downloads_days' => version_downloads_trend,
+  'dependencies' => dependencies,
+  'last_commit' => last_commits_days,
+  'forks' => forks,
+  'stars' => stars,
+  'issues' => issues,
+  'ranking' => ranking,
+  'commits' => total_commits,
+  'commit_activity_last_year' => last_year_commit_activity,
+  'contributors' => contributors,
+  'issues_info' => issues_info,
+  'readme_word_count' => readme_word_count,
+  'questions' => questions,
+  'questions_word_count' => questions_word_count,
+  'created_at' => DateTime.now
+}
+
+puts gem_info
+
+result = client[:gems].insert_one(gem_info)

data/config/local.example.rb

@@ -0,0 +1,12 @@
+require 'configuration'
+
+Configuration.for('rubygems') do
+  github_token TOKEN
+  github_account GITHUB_ACCOUNT
+  github_password GITHUB_PASSWORD
+  user_agent USER_AGENT
+end
+
+Configuration.for('stackoverflow') do
+	stackoverflow_token STACKOVERFLOW_TOKEN
+end

data/lib/repocrawler/crawler.rb

@@ -0,0 +1,283 @@
+require 'rubygems'
+require 'gems'
+require 'mongo'
+require 'github_api'
+require 'httparty'
+require 'configuration'
+require 'nokogiri'
+require 'open-uri'
+
+module Repos
+  Kernel.load 'config/local.rb'
+
+  class GithubData
+    def initialize(repo_user, repo_name, github_password='')
+      @GITHUB_README_URL = "https://raw.githubusercontent.com/#{repo_user}/#{repo_name}/master"
+      @GITHUB_API_BASE_URL = "https://api.github.com/repos/#{repo_user}/#{repo_name}"
+      @rubygems = Configuration.for 'rubygems'
+      @access_token = @rubygems.github_token
+      @github_password = github_password === '' ? @rubygems.github_password : github_password
+      @user_agent = @rubygems.user_agent
+      @repo_user = repo_user
+      @repo_name = repo_name
+    end
+    
+    # get the commit activity in last year
+    def get_last_year_commit_activity
+      last_year_commit_activity = HTTParty.get(@GITHUB_API_BASE_URL + "/stats/commit_activity?access_token=#{@access_token}", headers: {
+        "User-Agent" => @user_agent
+      })
+    end
+
+    # Get the contributors
+    def get_contributors
+      contributors = HTTParty.get(@GITHUB_API_BASE_URL + "/contributors?access_token=#{@access_token}", headers: {
+        "User-Agent" => @user_agent
+      }).map do |contributor|
+        {
+          'name' => contributor['login'],
+          'contributions' => contributor['contributions']
+        }
+      end
+
+      contributors
+    end
+
+    # get the total commits
+    def get_total_commits
+      contributors = get_contributors
+      commits = contributors.reduce(0) do |sum, num|
+        sum + num['contributions']
+      end
+
+      commits
+    end
+
+    # get numbers of forks, stars and issues
+    def get_forks
+      repos_meta = HTTParty.get(@GITHUB_API_BASE_URL + "?access_token=#{@access_token}", headers: {
+        "User-Agent" => @user_agent
+      })
+      forks = repos_meta['forks_count']
+
+      forks
+    end
+
+    def get_stars
+      repos_meta = HTTParty.get(@GITHUB_API_BASE_URL + "?access_token=#{@access_token}", headers: {
+        "User-Agent" => @user_agent
+      })
+      stars = repos_meta['stargazers_count']
+
+      stars
+    end
+
+    def get_issues
+      repos_meta = HTTParty.get(@GITHUB_API_BASE_URL + "?access_token=#{@access_token}", headers: {
+        "User-Agent" => @user_agent
+      })
+      issues = repos_meta['open_issues_count']
+
+      issues
+    end 
+
+    # get information of the closed issues
+    def get_issues_info
+      closed_issues = []
+      stop = false
+      page = 1
+
+      until stop
+        issue_fetch = HTTParty.get(@GITHUB_API_BASE_URL + "/issues?state=closed&page=#{page}&access_token=#{@access_token}", headers: {
+          "User-Agent" => @user_agent
+        })
+        if issue_fetch.count === 0
+          stop = true
+        end
+
+        issue_fetch.each do |issue|
+          closed_issues << {
+            'number'    => issue['number'],
+            'created_at'  => issue['created_at'],
+            'closed_at'   => issue['closed_at'],
+            'duration'    => (Date.parse(issue['closed_at']) - Date.parse(issue['created_at'])).to_i
+          }
+        end
+
+        page += 1
+      end
+
+      closed_issues.reverse!
+    end
+
+    # get the date of the last commit
+    def get_last_commits_days
+      github = Github.new basic_auth: "#{@rubygems.github_account}:#{@github_password}"
+
+      commit = github.repos.commits.list(@repo_user, @repo_name).to_ary[0].to_hash['commit']['author']['date']
+      last_commit = (Date.today - Date.parse(commit)).to_i
+
+      last_commit
+    end
+
+    # get the readme file
+    def get_readme_word_count
+      github_contents = HTTParty.get(@GITHUB_API_BASE_URL + "/contents?access_token=#{@access_token}", headers: {
+        "User-Agent" => @user_agent
+      })
+      readme_file = ''
+      github_contents.each do |content|
+        readme_file = content['name'] if content['name'] =~ /^README/
+      end
+
+      stop_words = []
+      File.open(File.expand_path("../../public/stop_words.txt",  File.dirname(__FILE__)), "r") do |f|
+        f.each_line do |line|
+          stop_words << line.gsub(/\n/,"")
+        end
+      end
+
+      readme = HTTParty.get(@GITHUB_README_URL + "/#{readme_file}")
+      words = readme.split(' ')
+      freqs = Hash.new(0)
+      words.each do |word|
+        if word =~ /^\w+$/ && !stop_words.include?(word.downcase)
+          freqs[word] += 1 
+        end
+      end
+      freqs = freqs.sort_by { |word, freq| freq }.reverse!
+
+      freqs
+    end
+  end
+  
+  class RubyGemsData
+    def initialize(gem_name)
+      @gem_name = gem_name
+    end
+
+    # get the downloads of each versions
+    def get_version_downloads
+      versions = Gems.versions @gem_name
+
+      version_downloads = versions.map do |version|
+        if version['platform'] === 'ruby'
+          {
+            'number' => version['number'],
+            'downloads' =>version['downloads_count']
+          }
+        end
+      end.reverse!
+
+      version_downloads.compact! if version_downloads.include? nil
+      version_downloads
+    end 
+
+    def get_version_downloads_trend(start_date='', end_date='')
+      versions = Gems.versions @gem_name
+
+      end_date = Date.today if end_date.to_s == ''
+      version_downloads_trend = versions.map do |version|
+        start = version['created_at'] if start_date.to_s == ''
+
+        if version['platform'] === 'ruby'
+          version_downloads_days = Gems.downloads @gem_name, version['number'], start, end_date
+          {
+            'number' => version['number'],
+            'downloads_date' => version_downloads_days
+          }
+        end
+      end.reverse!
+
+      version_downloads_trend.compact! if version_downloads_trend.include? nil
+      version_downloads_trend
+    end
+
+    # get the dependencies
+    def get_dependencies
+      oga_info = Gems.info @gem_name
+      dependencies = oga_info['dependencies']
+
+      dependencies
+    end
+
+    # total number of downloads
+    def get_total_downloads
+      oga_info = Gems.info @gem_name
+      total_downloads = oga_info['downloads']
+
+      total_downloads
+    end
+  end
+
+  class RubyToolBoxData
+    def initialize(gem_name)
+      rubygems = Configuration.for 'rubygems'
+      @user_agent = rubygems.user_agent
+      @RUBY_TOOLBOX_BASE_URL = "https://www.ruby-toolbox.com/projects/"
+      @RANKING_PATH = "//div[@class='teaser-bar']//li[last()-1]//a"
+      @gem_name = gem_name
+    end
+
+    # get the ranking on Ruby ToolBox
+    def get_ranking
+      begin
+        document = open(@RUBY_TOOLBOX_BASE_URL + @gem_name,
+            'User-Agent' => @user_agent
+          )
+        noko_document = Nokogiri::HTML(document)
+        ranking = noko_document.xpath(@RANKING_PATH).text
+      rescue
+        ranking = 0
+      end
+      ranking
+    end 
+  end
+
+  class StackOverflow
+    def initialize(gem_name)
+      stackoverflow = Configuration.for 'stackoverflow'
+      @STACKOVERFLOW_API = "https://api.stackexchange.com/2.2/search/advanced?order=desc&sort=creation&q=#{gem_name}&site=stackoverflow&key=#{stackoverflow.stackoverflow_token}"
+    end
+
+    #get questions from stackexchange
+    def get_questions
+
+      stop_words = []
+      File.open(File.expand_path("../../public/stop_words.txt",  File.dirname(__FILE__)), "r") do |f|
+        f.each_line do |line|
+          stop_words << line.gsub(/\n/,"")
+        end
+      end
+
+      questions = []
+      fetch_questions = HTTParty.get(@STACKOVERFLOW_API)
+      fetch_questions['items'].each do |q|
+        #don't store stop words
+        good_words = []
+        q['title'].split(' ').map do |word|
+          if !stop_words.include?(word.downcase)
+            good_words << word
+          end
+        end
+
+        questions << {
+          'creation_date' => q['creation_date'],
+          'title' => good_words,
+          'views' => q['view_count']
+        }
+      end
+
+      questions_word_count = Hash.new(0)
+      questions.each do |question|
+        question['title'].each do |word|
+          questions_word_count[word] += 1
+        end
+      end
+
+      questions_word_count = questions_word_count.sort_by { |word, freq| freq }.reverse!
+      [questions, questions_word_count]
+    end
+  end
+
+end 

data/lib/repocrawler/version.rb

@@ -0,0 +1,4 @@
+# Versioning
+module Repos
+  VERSION = '0.1.3'
+end

data/lib/repocrawler.rb

@@ -0,0 +1 @@
+require 'repocrawler/crawler.rb'