ghtorrent 0.3.1 → 0.4

data/lib/ghtorrent/logging.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'logger'
 
 module GHTorrent

data/lib/ghtorrent/migrations/005_add_repo_collaborators.rb

@@ -0,0 +1,23 @@
+require 'sequel'
+
+Sequel.migration do
+  up do
+
+    puts("Adding table project members")
+
+    create_table :project_members do
+      foreign_key :repo_id, :projects, :null => false
+      foreign_key :user_id, :users, :null => false
+      DateTime :created_at, :null => false,
+               :default => Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+      primary_key [:repo_id, :user_id]
+    end
+  end
+
+  down do
+
+    drop_table :project_members
+
+  end
+end

data/lib/ghtorrent/migrations/006_add_watchers.rb

@@ -0,0 +1,23 @@
+require 'sequel'
+
+Sequel.migration do
+  up do
+
+    puts("Adding table watchers")
+
+    create_table :watchers do
+      foreign_key :repo_id, :projects, :null => false
+      foreign_key :user_id, :users, :null => false
+      DateTime :created_at, :null => false,
+               :default => Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+      primary_key [:repo_id, :user_id]
+    end
+  end
+
+  down do
+
+    drop_table :watchers
+
+  end
+end

data/lib/ghtorrent/persister.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 module GHTorrent
 
   #

data/lib/ghtorrent/retriever.rb

@@ -1,30 +1,4 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
+require 'uri'
 
 module GHTorrent
   module Retriever
@@ -60,11 +34,11 @@ module GHTorrent
       end
     end
 
-    # Try Github API v2 user search by email. This is optional info, so
+    # Try Github user search by email. This is optional info, so
     # it may not return any data.
-    # http://develop.github.com/p/users.html
+    # http://developer.github.com/v3/search/#email-search
     def retrieve_user_byemail(email, name)
-      url = ghurl_v2("user/email/#{email}")
+      url = ghurl("legacy/user/email/#{URI.escape(email)}")
       r = api_request(url)
 
       return nil if r.empty?
@@ -127,15 +101,8 @@ module GHTorrent
       url = ghurl "repos/#{user}/#{repo}/commits?last_sha=#{last_sha}"
       commits = paged_api_request(url, config(:mirror_commit_pages_new_repo))
 
-      commits.reduce(Array.new) do |acc, c|
-        commit = @persister.find(:commits, {'sha' => "#{c['sha']}"})
-
-        if commit.empty?
-          acc << retrieve_commit(repo, c['sha'], user)
-        else
-          debug "Retriever: Already got commit #{repo} -> #{c['sha']}"
-        end
-        acc
+      commits.map do |c|
+        retrieve_commit(repo, c['sha'], user)
       end
     end
 
@@ -197,45 +164,39 @@ module GHTorrent
       @persister.find(:org_members, {'org' => org}).map{|o| retrieve_org(o['login'])}
     end
 
-    # Retrieve all commit comments for a specific repository
-    def retrieve_repo_comments(repo, user)
-      commit_comments = paged_api_request(ghurl "repos/#{user}/#{repo}/comments")
-      stored_comments = @persister.find(:commit_comments,
-                                        {'repo' => repo,
-                                         'user' => user})
-      store_commit_comments(repo, user, commit_comments, stored_comments)
-    end
-
     # Retrieve all comments for a single commit
-    def retrieve_commit_comments(user, repo, sha, reentrer = false)
-      # Optimization: if no commits comments are registered for the repo
-      # get them en masse
-      #items = @persister.count(:commit_comments, {'repo' => repo, 'user' => user})
-      #if items == 0 && !reentrer
-      #  retrieve_repo_comments(repo, user)
-      #  return retrieve_commit_comments(user, repo, sha, true)
-      #end
-
+    def retrieve_commit_comments(user, repo, sha)
       stored_comments = @persister.find(:commit_comments, {'commit_id' => sha})
       retrieved_comments = paged_api_request(ghurl "repos/#{user}/#{repo}/commits/#{sha}/comments")
-      store_commit_comments(repo, user, stored_comments, retrieved_comments)
-      @persister.find(:commit_comments, {'commit_id' => sha})
-    end
 
-    # Retrieve a single comment
-    def retrieve_commit_comment(user, repo, id, reentrer = false)
-      # Optimization: if no commits comments are registered for the repo
-      # get them en masse
-      #items = @persister.count(:commit_comments, {'repo' => repo, 'user' => user})
-      #if items == 0 && !reentrer
-      #  retrieve_repo_comments(repo, user)
-      #  return retrieve_commit_comment(user, repo, id)
-      #end
+      retrieved_comments.each{ |x|
+        x['repo'] = repo
+        x['user'] = user
+        x['commit_id'] = sha
+
+        if @persister.find(:commit_comments, {'repo' => repo,
+                                              'user' => user,
+                                              'id' => x['id']}).empty?
+          @persister.store(:commit_comments, x)
+        end
+      }
+      @persister.find(:commit_comments, {'commit_id' => sha})#.map{|x| x[@uniq] = x['_id']; x}
+  end
+
+  # Retrieve a single comment
+    def retrieve_commit_comment(user, repo, id)
 
       comment = @persister.find(:commit_comments, {'repo' => repo,
-                                                   'user' => user, 'id' => id})
-      if comment.empty?
+                                                   'user' => user,
+                                                   'id' => id}).first
+      if comment.nil?
         r = api_request(ghurl "repos/#{user}/#{repo}/comments/#{id}")
+
+        if r.empty?
+          debug "Retriever: Commit comment #{id} deleted"
+          return
+        end
+
         r['repo'] = repo
         r['user'] = user
         @persister.store(:commit_comments, r)
@@ -244,43 +205,101 @@ module GHTorrent
         r
       else
         debug "Retriever: Commit comment #{comment['commit_id']} -> #{comment['id']} exists"
-        comment[@uniq] = comment['_id']
         comment
       end
     end
 
-    # Get current Github events
-    def get_events
-      api_request "https://api.github.com/events"
-    end
+    # Retrieve all collaborators for a repository
+    def retrieve_repo_collaborators(user, repo)
 
-    private
+      url = ghurl "repos/#{user}/#{repo}/collaborators"
+      stored_collaborators = @persister.find(:repo_collaborators,
+                                             {'repo' => repo, 'owner' => user})
 
-    def ghurl(path)
-      config(:mirror_urlbase) + path
+      collaborators = paged_api_request url
+      collaborators.each do |x|
+        x['repo'] = repo
+        x['owner'] = user
+
+        exists = !stored_collaborators.find { |f|
+          f['login'] == x['login']
+        }.nil?
+
+        if not exists
+          @persister.store(:repo_collaborators, x)
+          info "Retriever: Added collaborator #{repo} -> #{x['login']}"
+        else
+          debug "Retriever: Collaborator #{repo} -> #{x['login']} exists"
+        end
+      end
+
+      @persister.find(:repo_collaborators, {'repo' => repo, 'owner' => user})
     end
 
-    def ghurl_v2(path)
-      config(:mirror_urlbase_v2) + path
+    # Retrieve a single repository collaborator
+    def retrieve_repo_collaborator(user, repo, new_member)
+
+      collaborator = @persister.find(:repo_collaborators,
+                                     {'repo' => repo,
+                                      'owner' => user,
+                                      'login' => new_member})
+
+      if collaborator.empty?
+        retrieve_repo_collaborators(user, repo).find{|x| x['login'] == new_member}
+      else
+        collaborator.first
+      end
     end
 
-    def store_commit_comments(repo, user, stored_comments, retrieved_comments)
-      retrieved_comments.each do |x|
+    # Retrieve all watchers for a repository
+    def retrieve_watchers(user, repo)
+      stored_watchers = @persister.find(:watchers,
+                                        {'repo' => repo, 'owner' => user})
 
-        exists = !stored_comments.find { |f|
-          f['commit_id'] == x['commit_id'] && f['id'] == x['id']
-        }.nil?
+      watchers = paged_api_request(ghurl "repos/#{user}/#{repo}/watchers")
+      watchers.each do |x|
+        x['repo'] = repo
+        x['owner'] = user
 
-        unless exists
-          x['repo'] = repo
-          x['user'] = user
+        exists = !stored_watchers.find { |f|
+          f['login'] == x['login']
+        }.nil?
 
-          @persister.store(:commit_comments, x)
-          info "Retriever: Added commit comment #{x['commit_id']} -> #{x['id']}"
+        if not exists
+          @persister.store(:watchers, x)
+          info "Retriever: Added watcher #{repo} -> #{x['login']}"
         else
-          debug "Retriever: Commit comment #{x['commit_id']} -> #{x['id']} exists"
+          debug "Retriever: Watcher #{repo} -> #{x['login']} exists"
         end
       end
+
+      @persister.find(:watchers, {'repo' => repo, 'owner' => user})
+    end
+
+    # Retrieve a single watcher for a repositry
+    def retrieve_watcher(user, repo, watcher)
+      stored_watcher = @persister.find(:watchers,
+                                {'repo' => repo,
+                                 'owner' => user,
+                                 'login' => watcher})
+
+      if stored_watcher.empty?
+        retrieve_watchers(user, repo).find{|x| x['login'] == watcher}
+      else
+        stored_watcher.first
+      end
+    end
+
+    # Get current Github events
+    def get_events
+      api_request "https://api.github.com/events"
     end
+
+    private
+
+    def ghurl(path)
+      config(:mirror_urlbase) + path
+    end
+
   end
 end

data/lib/ghtorrent/settings.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'yaml'
 
 module GHTorrent
@@ -49,7 +21,7 @@ module GHTorrent
         :mirror_persister => "mirror.persister",
         :mirror_commit_pages_new_repo => "mirror.commit_pages_new_repo",
 
-        :uniq_id => "uniq_id"
+        :uniq_id => "mirror.uniq_id"
     }
 
     def config(key)

data/lib/ghtorrent/utils.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 module GHTorrent
   module Utils
 

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: ghtorrent
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 3
   prerelease: 
   segments: 
   - 0
-  - 3
-  - 1
-  version: 0.3.1
+  - 4
+  version: "0.4"
 platform: ruby
 authors: 
 - Georgios Gousios
@@ -16,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-17 00:00:00 Z
+date: 2012-07-11 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: amqp
@@ -164,6 +163,8 @@ files:
 - lib/ghtorrent/migrations/002_add_external_ref_ids.rb
 - lib/ghtorrent/migrations/003_add_orgs.rb
 - lib/ghtorrent/migrations/004_add_commit_comments.rb
+- lib/ghtorrent/migrations/005_add_repo_collaborators.rb
+- lib/ghtorrent/migrations/006_add_watchers.rb
 - lib/ghtorrent/persister.rb
 - lib/ghtorrent/retriever.rb
 - lib/ghtorrent/settings.rb
@@ -175,6 +176,7 @@ files:
 - bin/ght-periodic-dump
 - bin/ght-rm-dupl
 - bin/ght-torrent-index
+- LICENSE
 - Rakefile
 - README.md
 - test/callstack_test.rb