ghtorrent 0.2 → 0.3

data/lib/ghtorrent/migrations/001_init_schema.rb

@@ -12,7 +12,7 @@ Sequel.migration do
       String :email, :null => true, :unique => true
       TrueClass :hireable, :null => true
       String :bio, :null => true
-      Time :created_at, :null => false
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
     end
 
     puts("Creating table projects")
@@ -23,7 +23,7 @@ Sequel.migration do
       String :name, :null => false
       String :description
       String :language
-      Time :created_at, :null => false
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
     end
 
     puts("Creating table commits")
@@ -32,7 +32,8 @@ Sequel.migration do
       String :sha, :size => 40, :unique => true
       foreign_key :author_id, :users
       foreign_key :committer_id, :users
-      Time :created_at, :null => false
+      foreign_key :project_id, :projects
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
     end
 
     puts("Creating table commit_parents")
@@ -46,6 +47,7 @@ Sequel.migration do
     create_table :followers do
       foreign_key :user_id, :users, :null => false
       foreign_key :follower_id, :users, :null => false
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
       primary_key [:user_id, :follower_id]
     end
   end

data/lib/ghtorrent/migrations/003_add_orgs.rb

@@ -0,0 +1,37 @@
+require 'sequel'
+
+Sequel.migration do
+  up do
+
+    puts("Adding organization descriminator field to table users")
+
+    alter_table :users do
+      add_column :type, "enum('USR', 'ORG')", :null => false
+    end
+
+    puts("Updating users with default values")
+    DB.transaction(:rollback => :reraise, :isolation => :committed) do
+      DB[:users].update(:type => "USR")
+    end
+
+    puts("Creating table organization-members")
+
+    create_table :organization_members do
+      foreign_key :org_id, :users, :null => false
+      foreign_key :user_id, :users, :null => false
+      primary_key [:org_id, :user_id]
+      DateTime :created_at, :null => false,
+               :default => Sequel::CURRENT_TIMESTAMP
+    end
+  end
+
+  down do
+    puts("Droping table organization-members")
+    drop_table :organization_members
+
+    puts("Droping organization descriminator field to table users")
+    alter_table :users do
+      drop_column :type
+    end
+  end
+end

data/lib/ghtorrent/migrations/004_add_commit_comments.rb

@@ -0,0 +1,27 @@
+require 'sequel'
+
+Sequel.migration do
+  up do
+
+    puts("Adding table commit comments")
+
+    create_table :commit_comments do
+      primary_key :id
+      foreign_key :commit_id, :commits, :null => false
+      foreign_key :user_id, :users, :null => false
+      String :body
+      Integer :line, :null => true
+      Integer :position, :null => true
+      Integer :comment_id, :null => false, :unique => true
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+      DateTime :created_at, :null => false,
+               :default => Sequel::CURRENT_TIMESTAMP
+    end
+  end
+
+  down do
+
+    drop_table :commit_comments
+
+  end
+end

data/lib/ghtorrent/retriever.rb

@@ -29,8 +29,8 @@
 module GHTorrent
   module Retriever
 
+    include GHTorrent::Utils
     include GHTorrent::APIClient
-    include GHTorrent::Settings
 
     def initialize(settings)
       super(settings)
@@ -44,16 +44,18 @@ module GHTorrent
         url = ghurl "users/#{user}"
         u = api_request(url)
 
-        if u.nil?
+        if u.empty?
           throw GHTorrentException.new("Cannot find user #{user}")
         end
 
         unq = @persister.store(:users, u)
         u[@uniq] = unq
-        info "Retriever: New user #{user}"
+        what = user_type(u['type'])
+        info "Retriever: New #{what} #{user}"
         u
       else
-        debug "Retriever: Already got user #{user}"
+        what = user_type(stored_user.first['type'])
+        debug "Retriever: Already got #{what} #{user}"
         stored_user.first
       end
     end
@@ -63,10 +65,13 @@ module GHTorrent
     # http://develop.github.com/p/users.html
     def retrieve_user_byemail(email, name)
       url = ghurl_v2("user/email/#{email}")
-      api_request(url)
+      r = api_request(url)
+
+      return nil if r.empty?
+      r
     end
 
-    def retrieve_new_user_followers(user)
+    def retrieve_user_followers(user)
       stored_followers = @persister.find(:followers, {'follows' => user})
 
       followers = paged_api_request(ghurl "users/#{user}/followers")
@@ -88,6 +93,7 @@ module GHTorrent
       @persister.find(:followers, {'follows' => user})
     end
 
+    # Retrieve a single commit from a repo
     def retrieve_commit(repo, sha, user)
       commit = @persister.find(:commits, {'sha' => "#{sha}"})
 
@@ -95,7 +101,7 @@ module GHTorrent
         url = ghurl "repos/#{user}/#{repo}/commits/#{sha}"
         c = api_request(url)
 
-        if c.nil?
+        if c.empty?
           throw GHTorrentException.new("Cannot find commit #{user}/#{repo}/#{sha}")
         end
 
@@ -109,6 +115,31 @@ module GHTorrent
       end
     end
 
+    # Retrieve all project commits or 500 (whatever comes first),
+    # starting from the provided +sha+
+    def retrieve_commits(repo, sha, user)
+      last_sha = if sha.nil?
+                  "master"
+                 else
+                  sha
+                 end
+
+      url = ghurl "repos/#{user}/#{repo}/commits?last_sha=#{last_sha}"
+      commits = paged_api_request(url, config(:mirror_commit_pages_new_repo))
+
+      commits.reduce(Array.new) do |acc, c|
+        commit = @persister.find(:commits, {'sha' => "#{c['sha']}"})
+
+        if commit.empty?
+          acc << retrieve_commit(repo, c['sha'], user)
+        else
+          debug "Retriever: Already got commit #{repo} -> #{c['sha']}"
+        end
+        acc
+      end
+    end
+
+
     def retrieve_repo(user, repo)
       stored_repo = @persister.find(:repos, {'owner.login' => user,
                                              'name' => repo })
@@ -116,7 +147,7 @@ module GHTorrent
         url = ghurl "repos/#{user}/#{repo}"
         r = api_request(url)
 
-        if r.nil?
+        if r.empty?
           throw GHTorrentException.new("Cannot find repo #{user}/#{repo}")
         end
 
@@ -130,6 +161,94 @@ module GHTorrent
       end
     end
 
+    # Retrieve organizations the provided user participates into
+    def retrieve_orgs(user)
+      url = ghurl "users/#{user}/orgs"
+      orgs = paged_api_request(url)
+      orgs.map{|o| retrieve_org(o['login'])}
+    end
+
+    # Retrieve a single organization
+    def retrieve_org(org)
+      retrieve_user_byusername(org)
+    end
+
+    # Retrieve organization members
+    def retrieve_org_members(org)
+      url = ghurl "orgs/#{org}/members"
+      stored_org_members = @persister.find(:org_members, {'org' => org})
+
+      org_members = paged_api_request(ghurl "orgs/#{org}/members")
+      org_members.each do |x|
+        x['org'] = org
+
+        exists = !stored_org_members.find { |f|
+          f['org'] == user && f['login'] == x['login']
+        }.nil?
+
+        if not exists
+          @persister.store(:org_members, x)
+          info "Retriever: Added member #{org} -> #{x['login']}"
+        else
+          debug "Retriever: Member #{org} -> #{x['login']} exists"
+        end
+      end
+
+      @persister.find(:org_members, {'org' => org}).map{|o| retrieve_org(o['login'])}
+    end
+
+    # Retrieve all commit comments for a specific repository
+    def retrieve_repo_comments(repo, user)
+      commit_comments = paged_api_request(ghurl "repos/#{user}/#{repo}/comments")
+      stored_comments = @persister.find(:commit_comments,
+                                        {'repo' => repo,
+                                         'user' => user})
+      store_commit_comments(repo, user, commit_comments, stored_comments)
+    end
+
+    # Retrieve all comments for a single commit
+    def retrieve_commit_comments(user, repo, sha, reentrer = false)
+      # Optimization: if no commits comments are registered for the repo
+      # get them en masse
+      #items = @persister.count(:commit_comments, {'repo' => repo, 'user' => user})
+      #if items == 0 && !reentrer
+      #  retrieve_repo_comments(repo, user)
+      #  return retrieve_commit_comments(user, repo, sha, true)
+      #end
+
+      stored_comments = @persister.find(:commit_comments, {'commit_id' => sha})
+      retrieved_comments = paged_api_request(ghurl "repos/#{user}/#{repo}/commits/#{sha}/comments")
+      store_commit_comments(repo, user, stored_comments, retrieved_comments)
+      @persister.find(:commit_comments, {'commit_id' => sha})
+    end
+
+    # Retrieve a single comment
+    def retrieve_commit_comment(user, repo, id, reentrer = false)
+      # Optimization: if no commits comments are registered for the repo
+      # get them en masse
+      #items = @persister.count(:commit_comments, {'repo' => repo, 'user' => user})
+      #if items == 0 && !reentrer
+      #  retrieve_repo_comments(repo, user)
+      #  return retrieve_commit_comment(user, repo, id)
+      #end
+
+      comment = @persister.find(:commit_comments, {'repo' => repo,
+                                                   'user' => user, 'id' => id})
+      if comment.empty?
+        r = api_request(ghurl "repos/#{user}/#{repo}/comments/#{id}")
+        r['repo'] = repo
+        r['user'] = user
+        @persister.store(:commit_comments, r)
+        info "Retriever: Added commit comment #{r['commit_id']} -> #{r['id']}"
+        r[@uniq] = r['_id']
+        r
+      else
+        debug "Retriever: Commit comment #{comment['commit_id']} -> #{comment['id']} exists"
+        comment[@uniq] = comment['_id']
+        comment
+      end
+    end
+
     # Get current Github events
     def get_events
       api_request "https://api.github.com/events"
@@ -144,5 +263,24 @@ module GHTorrent
     def ghurl_v2(path)
       config(:mirror_urlbase_v2) + path
     end
+
+    def store_commit_comments(repo, user, stored_comments, retrieved_comments)
+      retrieved_comments.each do |x|
+
+        exists = !stored_comments.find { |f|
+          f['commit_id'] == x['commit_id'] && f['id'] == x['id']
+        }.nil?
+
+        unless exists
+          x['repo'] = repo
+          x['user'] = user
+
+          @persister.store(:commit_comments, x)
+          info "Retriever: Added commit comment #{x['commit_id']} -> #{x['id']}"
+        else
+          debug "Retriever: Commit comment #{x['commit_id']} -> #{x['id']} exists"
+        end
+      end
+    end
   end
 end

data/lib/ghtorrent/settings.rb

@@ -47,6 +47,7 @@ module GHTorrent
         :mirror_reqrate => "mirror.reqrate",
         :mirror_pollevery => "mirror.pollevery",
         :mirror_persister => "mirror.persister",
+        :mirror_commit_pages_new_repo => "mirror.commit_pages_new_repo",
 
         :uniq_id => "uniq_id"
     }

data/lib/ghtorrent/utils.rb

@@ -28,6 +28,11 @@
 
 module GHTorrent
   module Utils
+
+    def self.included(other)
+      other.extend self
+    end
+
     # Read a value whose format is "foo.bar.baz" from a hierarchical map
     # (the result of a JSON parse or a Mongo query), where a dot represents
     # one level deep in the result hierarchy.
@@ -54,5 +59,13 @@ module GHTorrent
         end
       end
     end
+
+    def user_type(type)
+      if type == "User"
+        "USR"
+      else
+        "ORG"
+      end
+    end
   end
 end

data/test/callstack_test.rb

@@ -1,5 +1,5 @@
 require "test/unit"
-require 'ghtorrent-old'
+require 'ghtorrent'
 
 class CallStackTest < Test::Unit::TestCase
 

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: ghtorrent
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 13
   prerelease: 
   segments: 
   - 0
-  - 2
-  version: "0.2"
+  - 3
+  version: "0.3"
 platform: ruby
 authors: 
 - Georgios Gousios
@@ -107,6 +107,38 @@ dependencies:
         version: "3.35"
   type: :runtime
   version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: sqlite3-ruby
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 1
+        - 3
+        - 2
+        version: 1.3.2
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: daemons
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 1
+        - 1
+        - 8
+        version: 1.1.8
+  type: :runtime
+  version_requirements: *id008
 description: |-
   A library and a collection of associated programs
                       to mirror and process Github data
@@ -128,8 +160,9 @@ files:
 - lib/ghtorrent/ghtorrent.rb
 - lib/ghtorrent/logging.rb
 - lib/ghtorrent/migrations/001_init_schema.rb
-- lib/ghtorrent/migrations/002_add_followers_created_at.rb
-- lib/ghtorrent/migrations/003_add_external_ref_ids.rb
+- lib/ghtorrent/migrations/002_add_external_ref_ids.rb
+- lib/ghtorrent/migrations/003_add_orgs.rb
+- lib/ghtorrent/migrations/004_add_commit_comments.rb
 - lib/ghtorrent/persister.rb
 - lib/ghtorrent/retriever.rb
 - lib/ghtorrent/settings.rb

data/lib/ghtorrent/migrations/002_add_followers_created_at.rb

@@ -1,15 +0,0 @@
-require 'sequel'
-
-Sequel.migration do
-  up do
-    alter_table :followers do
-      add_column :created_at, :Time, :null => false, :default => Time.now
-    end
-  end
-
-  down do
-    alter_table :followers do
-      drop_column :created_at
-    end
-  end
-end