ghtorrent 0.4 → 0.5

data/lib/ghtorrent/hash.rb

@@ -0,0 +1,11 @@
+class Hash
+  def merge_recursive(o, overwrite = true)
+    merge(o) do |_,x,y|
+      if x.respond_to?(:merge_recursive) && y.is_a?(Hash)
+        x.merge_recursive(y)
+      else
+        if overwrite then y else [*x,*y] end
+      end
+    end
+  end
+end

data/lib/ghtorrent/logging.rb

@@ -21,21 +21,25 @@ module GHTorrent
 
     # Log a message at the given level.
     def log(level, msg)
-      return unless @logger
+
       case level
         when :fatal then
-          @logger.fatal msg
+          logger.fatal msg
         when :error then
-          @logger.error msg
+          logger.error msg
         when :warn then
-          @logger.warn msg
+          logger.warn msg
         when :info then
-          @logger.info msg
+          logger.info msg
         when :debug then
-          @logger.debug msg
+          logger.debug msg
         else
-          @logger.debug msg
+          logger.debug msg
       end
     end
+
+    def logger
+      raise Exception.new("Unimplemented")
+    end
   end
 end

data/lib/ghtorrent/migrations/001_init_schema.rb

@@ -1,5 +1,7 @@
 require 'sequel'
 
+require 'ghtorrent/migrations/mysql_defaults'
+
 Sequel.migration do
   up do
     puts("Creating table users")
@@ -9,9 +11,7 @@ Sequel.migration do
       String :name
       String :company, :null => true
       String :location, :null => true
-      String :email, :null => true, :unique => true
-      TrueClass :hireable, :null => true
-      String :bio, :null => true
+      String :email, :null => true
       DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
     end
 

data/lib/ghtorrent/migrations/002_add_external_ref_ids.rb

@@ -1,5 +1,7 @@
 require 'sequel'
 
+require 'ghtorrent/migrations/mysql_defaults'
+
 Sequel.migration do
   up do
     alter_table :users do

data/lib/ghtorrent/migrations/003_add_orgs.rb

@@ -1,12 +1,15 @@
 require 'sequel'
 
+require 'ghtorrent/migrations/mysql_defaults'
+
 Sequel.migration do
   up do
 
     puts("Adding organization descriminator field to table users")
 
     alter_table :users do
-      add_column :type, "enum('USR', 'ORG')", :null => false
+      add_column :type, String, :null => false, :default => 'USR'
+      add_constraint(:type_allowed_values, :type => %w[USR ORG])
     end
 
     puts("Updating users with default values")

data/lib/ghtorrent/migrations/004_add_commit_comments.rb

@@ -1,5 +1,7 @@
 require 'sequel'
 
+require 'ghtorrent/migrations/mysql_defaults'
+
 Sequel.migration do
   up do
 
@@ -9,7 +11,7 @@ Sequel.migration do
       primary_key :id
       foreign_key :commit_id, :commits, :null => false
       foreign_key :user_id, :users, :null => false
-      String :body
+      String :body, :size => 256
       Integer :line, :null => true
       Integer :position, :null => true
       Integer :comment_id, :null => false, :unique => true
@@ -24,4 +26,4 @@ Sequel.migration do
     drop_table :commit_comments
 
   end
-end
+end

data/lib/ghtorrent/migrations/005_add_repo_collaborators.rb

@@ -1,5 +1,7 @@
 require 'sequel'
 
+require 'ghtorrent/migrations/mysql_defaults'
+
 Sequel.migration do
   up do
 

data/lib/ghtorrent/migrations/006_add_watchers.rb

@@ -1,5 +1,7 @@
 require 'sequel'
 
+require 'ghtorrent/migrations/mysql_defaults'
+
 Sequel.migration do
   up do
 

data/lib/ghtorrent/migrations/007_add_pull_requests.rb

@@ -0,0 +1,64 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+Sequel.migration do
+  up do
+
+    puts("Adding table pull requests")
+
+    create_table :pull_requests do
+      primary_key :id
+      foreign_key :head_repo_id, :projects
+      foreign_key :base_repo_id, :projects, :null => false
+      foreign_key :head_commit_id, :commits
+      foreign_key :base_commit_id, :commits, :null => false
+      foreign_key :user_id, :users, :null => false
+      Integer :pullreq_id, :null => false
+      TrueClass :intra_branch, :null => false
+      unique([:pullreq_id, :base_repo_id])
+    end
+
+    puts("Adding table pull request history")
+
+    create_table :pull_request_history do
+      primary_key :id
+      foreign_key :pull_request_id, :pull_requests, :null => false
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+      String :action, :null => false
+      check(:action=>%w[opened closed merged synchronize reopened])
+    end
+
+    puts("Adding table pull request commits")
+
+    create_table :pull_request_commits do
+      foreign_key :pull_request_id, :pull_requests, :null => false
+      foreign_key :commit_id, :commits, :null => false
+      primary_key [:pull_request_id, :commit_id]
+    end
+
+    puts("Adding table pull request comments")
+
+    create_table :pull_request_comments do
+      foreign_key :pull_request_id, :pull_requests, :null => false
+      foreign_key :user_id, :users, :null => false
+      Long :comment_id, :null =>  false
+      Integer :position, :null => true
+      String :body, :size => 256
+      foreign_key :commit_id, :commits, :null => false
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+    end
+
+  end
+
+  down do
+
+    drop_table :pull_requests
+    drop_table :pull_request_history
+    drop_table :pull_request_commits
+    drop_table :pull_request_comments
+
+  end
+end

data/lib/ghtorrent/migrations/008_add_project_unq.rb

@@ -0,0 +1,23 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+Sequel.migration do
+  up do
+
+    puts("Adding unique(name, owner) constraint to table projects")
+
+    alter_table :projects do
+      add_unique_constraint([:name, :owner_id])
+    end
+  end
+
+  down do
+
+    puts("Removing unique(name, owner) constraint from table projects")
+
+    alter_table :projects do
+      drop_unique_constraint([:name, :owner_id])
+    end
+  end
+end

data/lib/ghtorrent/migrations/009_add_project_commit.rb

@@ -0,0 +1,27 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+Sequel.migration do
+  up do
+
+    puts("Create table project_commits")
+    create_table :project_commits do
+      foreign_key :project_id, :projects
+      foreign_key :commit_id, :commits
+      primary_key [:project_id, :commit_id]
+    end
+
+    puts("Migrating data from commits to project_commits")
+    transaction(:rollback => :reraise, :isolation => :committed) do
+      self[:project_commits].insert([:project_id, :commit_id],
+                                    self[:commits].select(:project_id, :id))
+    end
+
+  end
+
+  down do
+
+    drop_table :project_commits
+  end
+end

data/lib/ghtorrent/migrations/010_add_forks.rb

@@ -0,0 +1,28 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+Sequel.migration do
+  up do
+
+    puts("Adding table forks")
+
+    create_table :forks do
+      foreign_key :forked_project_id, :projects, :null => false
+      foreign_key :forked_from_id, :projects, :null => false
+      Integer :fork_id, :null => false, :unique => true
+      DateTime :created_at, :null => false,
+               :default => Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+      primary_key([:forked_project_id, :forked_from_id])
+    end
+
+  end
+
+  down do
+
+    puts("Droping table forks")
+    drop_table :forks
+
+  end
+end

data/lib/ghtorrent/migrations/mysql_defaults.rb

@@ -0,0 +1,6 @@
+require 'sequel'
+
+Sequel::MySQL.default_engine = 'InnoDB' if defined?(Sequel::MySQL)
+Sequel::MySQL.default_charset = 'utf8'  if defined?(Sequel::MySQL)
+Sequel::MySQL.default_collate = 'utf8_general_ci' if defined?(Sequel::MySQL)
+

data/lib/ghtorrent/persister.rb

@@ -1,3 +1,6 @@
+require 'ghtorrent/adapters/mongo_persister'
+require 'ghtorrent/adapters/noop_persister'
+
 module GHTorrent
 
   #

data/lib/ghtorrent/retriever.rb

@@ -1,29 +1,37 @@
 require 'uri'
 
+require 'ghtorrent/api_client'
+require 'ghtorrent/settings'
+require 'ghtorrent/utils'
+require 'ghtorrent/gh_torrent_exception'
+
 module GHTorrent
   module Retriever
 
+    include GHTorrent::Settings
     include GHTorrent::Utils
     include GHTorrent::APIClient
 
-    def initialize(settings)
-      super(settings)
-      @settings = settings
-      @uniq = config(:uniq_id)
+    def ext_uniq
+      raise Exception("Unimplemented")
+    end
+
+    def persister
+      raise Exception("Unimplemented")
     end
 
     def retrieve_user_byusername(user)
-      stored_user = @persister.find(:users, {'login' => user})
+      stored_user = persister.find(:users, {'login' => user})
       if stored_user.empty?
         url = ghurl "users/#{user}"
         u = api_request(url)
 
         if u.empty?
-          throw GHTorrentException.new("Cannot find user #{user}")
+          return
         end
 
-        unq = @persister.store(:users, u)
-        u[@uniq] = unq
+        unq = persister.store(:users, u)
+        u[ext_uniq] = unq
         what = user_type(u['type'])
         info "Retriever: New #{what} #{user}"
         u
@@ -41,62 +49,79 @@ module GHTorrent
       url = ghurl("legacy/user/email/#{URI.escape(email)}")
       r = api_request(url)
 
-      return nil if r.empty?
-      r
+      unless r.empty? or r['user']['login'].nil?
+        info "Retriever: User #{r['user']['login']} retrieved by email #{email}"
+        retrieve_user_byusername(r['user']['login'])
+      else
+        if r.empty?
+          nil
+        else
+          u = r['user']
+          unq = persister.store(:users, u)
+          u[ext_uniq] = unq
+          what = user_type(u['type'])
+          info "Retriever: New #{what} #{user}"
+          u
+        end
+      end
     end
 
-    def retrieve_user_followers(user)
-      stored_followers = @persister.find(:followers, {'follows' => user})
+    def retrieve_user_follower(followed, follower)
+      stored_item = persister.find(:followers, {'follows' => followed,
+                                                'login' => follower})
+
+      if stored_item.empty?
+        retrieve_user_followers(followed).find{|x| x['login'] == follower}
+      else
+        stored_item.first
+      end
+    end
 
+    def retrieve_user_followers(user)
       followers = paged_api_request(ghurl "users/#{user}/followers")
       followers.each do |x|
         x['follows'] = user
 
-        exists = !stored_followers.find { |f|
-          f['follows'] == user && f['login'] == x['login']
-        }.nil?
+        exists = !persister.find(:followers, {'follows' => user,
+                                     'login' => x['login']}).empty?
 
         if not exists
-          @persister.store(:followers, x)
+          persister.store(:followers, x)
           info "Retriever: Added follower #{user} -> #{x['login']}"
         else
           debug "Retriever: Follower #{user} -> #{x['login']} exists"
         end
       end
 
-      @persister.find(:followers, {'follows' => user})
+      persister.find(:followers, {'follows' => user})
     end
 
     # Retrieve a single commit from a repo
     def retrieve_commit(repo, sha, user)
-      commit = @persister.find(:commits, {'sha' => "#{sha}"})
+      commit = persister.find(:commits, {'sha' => "#{sha}"})
 
       if commit.empty?
         url = ghurl "repos/#{user}/#{repo}/commits/#{sha}"
         c = api_request(url)
 
         if c.empty?
-          throw GHTorrentException.new("Cannot find commit #{user}/#{repo}/#{sha}")
+          return
         end
 
-        unq = @persister.store(:commits, c)
-        info "Retriever: New commit #{repo} -> #{sha}"
-        c[@uniq] = unq
+        unq = persister.store(:commits, c)
+        info "Retriever: New commit #{user}/#{repo} -> #{sha}"
+        c[ext_uniq] = unq
         c
       else
-        debug "Retriever: Already got commit #{repo} -> #{sha}"
+        debug "Retriever: Already got commit #{user}/#{repo} -> #{sha}"
         commit.first
       end
     end
 
-    # Retrieve all project commits or 500 (whatever comes first),
+    # Retrieve up to 30 * +:mirror_commit_pages_new_repo+ commits
     # starting from the provided +sha+
     def retrieve_commits(repo, sha, user)
-      last_sha = if sha.nil?
-                  "master"
-                 else
-                  sha
-                 end
+      last_sha = if sha.nil? then "master" else sha end
 
       url = ghurl "repos/#{user}/#{repo}/commits?last_sha=#{last_sha}"
       commits = paged_api_request(url, config(:mirror_commit_pages_new_repo))
@@ -108,19 +133,19 @@ module GHTorrent
 
 
     def retrieve_repo(user, repo)
-      stored_repo = @persister.find(:repos, {'owner.login' => user,
+      stored_repo = persister.find(:repos, {'owner.login' => user,
                                              'name' => repo })
       if stored_repo.empty?
         url = ghurl "repos/#{user}/#{repo}"
         r = api_request(url)
 
         if r.empty?
-          throw GHTorrentException.new("Cannot find repo #{user}/#{repo}")
+          return
         end
 
-        unq = @persister.store(:repos, r)
+        unq = persister.store(:repos, r)
         info "Retriever: New repo #{user} -> #{repo}"
-        r[@uniq] = unq
+        r[ext_uniq] = unq
         r
       else
         debug "Retriever: Already got repo #{user} -> #{repo}"
@@ -142,51 +167,49 @@ module GHTorrent
 
     # Retrieve organization members
     def retrieve_org_members(org)
-      url = ghurl "orgs/#{org}/members"
-      stored_org_members = @persister.find(:org_members, {'org' => org})
+      stored_org_members = persister.find(:org_members, {'org' => org})
 
       org_members = paged_api_request(ghurl "orgs/#{org}/members")
       org_members.each do |x|
         x['org'] = org
 
         exists = !stored_org_members.find { |f|
-          f['org'] == user && f['login'] == x['login']
+          f['org'] == org && f['login'] == x['login']
         }.nil?
 
         if not exists
-          @persister.store(:org_members, x)
-          info "Retriever: Added member #{org} -> #{x['login']}"
+          persister.store(:org_members, x)
+          info "Retriever: Added org member #{org} -> #{x['login']}"
         else
-          debug "Retriever: Member #{org} -> #{x['login']} exists"
+          debug "Retriever: Org Member #{org} -> #{x['login']} exists"
         end
       end
 
-      @persister.find(:org_members, {'org' => org}).map{|o| retrieve_org(o['login'])}
+      persister.find(:org_members, {'org' => org}).map{|o| retrieve_org(o['login'])}
     end
 
     # Retrieve all comments for a single commit
     def retrieve_commit_comments(user, repo, sha)
-      stored_comments = @persister.find(:commit_comments, {'commit_id' => sha})
       retrieved_comments = paged_api_request(ghurl "repos/#{user}/#{repo}/commits/#{sha}/comments")
 
-      retrieved_comments.each{ |x|
+      retrieved_comments.each { |x|
         x['repo'] = repo
         x['user'] = user
         x['commit_id'] = sha
 
-        if @persister.find(:commit_comments, {'repo' => repo,
+        if persister.find(:commit_comments, {'repo' => repo,
                                               'user' => user,
                                               'id' => x['id']}).empty?
-          @persister.store(:commit_comments, x)
+          persister.store(:commit_comments, x)
         end
       }
-      @persister.find(:commit_comments, {'commit_id' => sha})#.map{|x| x[@uniq] = x['_id']; x}
-  end
+      persister.find(:commit_comments, {'commit_id' => sha})
+    end
 
-  # Retrieve a single comment
+    # Retrieve a single comment
     def retrieve_commit_comment(user, repo, id)
 
-      comment = @persister.find(:commit_comments, {'repo' => repo,
+      comment = persister.find(:commit_comments, {'repo' => repo,
                                                    'user' => user,
                                                    'id' => id}).first
       if comment.nil?
@@ -199,10 +222,10 @@ module GHTorrent
 
         r['repo'] = repo
         r['user'] = user
-        @persister.store(:commit_comments, r)
+        persister.store(:commit_comments, r)
         info "Retriever: Added commit comment #{r['commit_id']} -> #{r['id']}"
-        r[@uniq] = r['_id']
-        r
+        persister.find(:commit_comments, {'repo' => repo, 'user' => user,
+                                          'id' => id}).first
       else
         debug "Retriever: Commit comment #{comment['commit_id']} -> #{comment['id']} exists"
         comment
@@ -211,91 +234,264 @@ module GHTorrent
 
     # Retrieve all collaborators for a repository
     def retrieve_repo_collaborators(user, repo)
+      repo_bound_items(user, repo, :repo_collaborators,
+                       "repos/#{user}/#{repo}/collaborators",
+                       {'repo' => repo, 'owner' => user},
+                       'login')
+    end
+
+    # Retrieve a single repository collaborator
+    def retrieve_repo_collaborator(user, repo, new_member)
+      repo_bound_item(user, repo, new_member, :repo_collaborators,
+                      "repos/#{user}/#{repo}/collaborators",
+                      {'repo' => repo, 'owner' => user},
+                      'login')
+    end
+
+    # Retrieve all watchers for a repository
+    def retrieve_watchers(user, repo)
+      repo_bound_items(user, repo, :watchers,
+                       "repos/#{user}/#{repo}/watchers",
+                       {'repo' => repo, 'owner' => user},
+                       'login')
+    end
+
+    # Retrieve a single watcher for a repositry
+    def retrieve_watcher(user, repo, watcher)
+      repo_bound_item(user, repo, watcher, :watchers,
+                      "repos/#{user}/#{repo}/watchers",
+                      {'repo' => repo, 'owner' => user},
+                      'login')
+    end
 
-      url = ghurl "repos/#{user}/#{repo}/collaborators"
-      stored_collaborators = @persister.find(:repo_collaborators,
-                                             {'repo' => repo, 'owner' => user})
+    def retrieve_pull_requests(user, repo)
+      open = "repos/#{user}/#{repo}/pulls"
+      closed = "repos/#{user}/#{repo}/pulls?state=closed"
+      repo_bound_items(user, repo, :pull_requests,
+                       [open, closed],
+                       {'repo' => repo, 'owner' => user},
+                       'number')
+    end
+
+    def retrieve_pull_request(user, repo, pullreq_id)
+      open = "repos/#{user}/#{repo}/pulls"
+      closed = "repos/#{user}/#{repo}/pulls?state=closed"
+      repo_bound_item(user, repo, pullreq_id, :pull_requests,
+                      [open, closed],
+                      {'repo' => repo, 'owner' => user,
+                       'number' => pullreq_id},
+                      'number')
+    end
+
+    def retrieve_forks(user, repo)
+      repo_bound_items(user, repo, :forks,
+                       "repos/#{user}/#{repo}/forks",
+                       {'repo' => repo, 'owner' => user},
+                       'id')
+    end
+
+    def retrieve_fork(user, repo, fork_id)
+      repo_bound_item(user, repo, fork_id, :forks,
+                       "repos/#{user}/#{repo}/forks",
+                       {'repo' => repo, 'owner' => user},
+                       'id')
+    end
 
-      collaborators = paged_api_request url
-      collaborators.each do |x|
+    def retrieve_pull_req_commits(user, repo, pullreq_id)
+      is_intra_branch = Proc.new do |req|
+        req['head']['repo'].nil?
+      end
+
+      pull_req = retrieve_pull_request(user, repo, pullreq_id)
+
+      unless is_intra_branch.call(pull_req)
+        head_user = pull_req['head']['repo']['owner']['login']
+        head_repo = pull_req['head']['repo']['name']
+
+        commits = paged_api_request(ghurl "repos/#{user}/#{repo}/pulls/#{pullreq_id}/commits")
+        commits.map { |x|
+          retrieve_commit(head_repo, x['sha'], head_user)
+        }
+      else
+        commits = paged_api_request(ghurl "repos/#{user}/#{repo}/pulls/#{pullreq_id}/commits")
+        commits.map { |x|
+          retrieve_commit(repo, x['sha'], user)
+        }
+      end
+    end
+
+    def retrieve_pull_req_comments(owner, repo, pullreq_id)
+      review_comments_url = ghurl "repos/#{owner}/#{repo}/pulls/#{pullreq_id}/comments"
+
+      url = review_comments_url
+      retrieved_comments = paged_api_request url
+
+      retrieved_comments.each { |x|
+        x['owner'] = owner
         x['repo'] = repo
-        x['owner'] = user
+        x['issue_id'] = pullreq_id
 
-        exists = !stored_collaborators.find { |f|
-          f['login'] == x['login']
-        }.nil?
+        if persister.find(:pull_request_comments, {'owner' => owner,
+                                                   'repo' => repo,
+                                                   'pullreq_id' => pullreq_id,
+                                                   'id' => x['id']}).empty?
+          persister.store(:pull_request_comments, x)
+        end
+      }
 
-        if not exists
-          @persister.store(:repo_collaborators, x)
-          info "Retriever: Added collaborator #{repo} -> #{x['login']}"
-        else
-          debug "Retriever: Collaborator #{repo} -> #{x['login']} exists"
+      persister.find(:pull_request_comments, {'owner' => owner, 'repo' => repo,
+                                              'pullreq_id' => pullreq_id})
+    end
+
+    def retrieve_pull_req_comment(owner, repo, pullreq_id, comment_id)
+      comment = persister.find(:pull_request_comments, {'repo' => repo,
+                                                 'owner' => owner,
+                                                 'pullreq_id' => pullreq_id,
+                                                 'id' => comment_id}).first
+      if comment.nil?
+        r = api_request(ghurl "repos/#{owner}/#{repo}/pulls/comments/#{comment_id}")
+
+        if r.empty?
+          debug "Retriever: Pullreq comment #{owner}/#{repo} #{pullreq_id}->#{comment_id} deleted"
+          return
         end
+
+        r['repo'] = repo
+        r['owner'] = owner
+        r['pullreq_id'] = pullreq_id
+        persister.store(:pull_request_comments, r)
+        info "Retriever: Added pullreq comment #{owner}/#{repo} #{pullreq_id}->#{comment_id}"
+        persister.find(:pull_request_comments, {'repo' => repo, 'owner' => owner,
+                                         'pullreq_id' => pullreq_id,
+                                         'id' => comment_id}).first
+      else
+        debug "Retriever: Pullreq comment #{owner}/#{repo} #{pullreq_id}->#{comment_id} exists"
+        comment
       end
+    end
 
-      @persister.find(:repo_collaborators, {'repo' => repo, 'owner' => user})
+    def retrieve_issues(user, repo)
+      repo_bound_items(user, repo, :issues,
+                       "repos/#{user}/#{repo}/issues",
+                       {'repo' => repo, 'owner' => user},
+                       'id')
     end
 
-    # Retrieve a single repository collaborator
-    def retrieve_repo_collaborator(user, repo, new_member)
+    def retrieve_issue(user, repo, issue_id)
+      repo_bound_item(user, repo, issue_id, :issues,
+                      "repos/#{user}/#{repo}/issues/#{issue_id}",
+                      {'repo' => repo, 'owner' => user},
+                      'id')
+    end
+
+    def retrieve_issue_comments(owner, repo, issue_id)
+      url = ghurl "repos/#{owner}/#{repo}/issues/#{issue_id}/comments"
+      retrieved_comments = paged_api_request url
 
-      collaborator = @persister.find(:repo_collaborators,
-                                     {'repo' => repo,
-                                      'owner' => user,
-                                      'login' => new_member})
+      retrieved_comments.each { |x|
+        x['owner'] = owner
+        x['repo'] = repo
+        x['issue_id'] = issue_id
+
+        if persister.find(:issue_comments, {'owner' => owner,
+                                            'repo' => repo,
+                                            'issue_id' => issue_id,
+                                            'id' => x['id']}).empty?
+          persister.store(:issue_comments, x)
+        end
+      }
+      persister.find(:issue_comments, {'owner' => owner, 'repo' => repo,
+                                       'issue_id' => issue_id})
+    end
+
+    def retrieve_issue_comment(owner, repo, issue_id, comment_id)
+      comment = persister.find(:issue_comments, {'repo' => repo,
+                                                 'owner' => owner,
+                                                 'issue_id' => issue_id,
+                                                 'id' => comment_id}).first
+      if comment.nil?
+        r = api_request(ghurl "repos/#{owner}/#{repo}/issues/#{issue_id}/comments/#{comment_id}")
+
+        if r.empty?
+          debug "Retriever: Issue comment #{owner}/#{repo} #{issue_id}->#{comment_id} deleted"
+          return
+        end
 
-      if collaborator.empty?
-        retrieve_repo_collaborators(user, repo).find{|x| x['login'] == new_member}
+        r['repo'] = repo
+        r['owner'] = owner
+        x['issue_id'] = issue_id
+        persister.store(:issue_comments, r)
+        info "Retriever: Added issue comment #{owner}/#{repo} #{issue_id}->#{comment_id}"
+        persister.find(:issue_comments, {'repo' => repo, 'owner' => owner,
+                                         'issue_id' => issue_id,
+                                         'id' => comment_id}).first
+        r
       else
-        collaborator.first
+        debug "Retriever: Commit comment #{owner}/#{repo} #{issue_id}->#{comment_id} exists"
+        comment
       end
     end
 
-    # Retrieve all watchers for a repository
-    def retrieve_watchers(user, repo)
-      stored_watchers = @persister.find(:watchers,
-                                        {'repo' => repo, 'owner' => user})
+    # Get current Github events
+    def get_events
+      api_request "https://api.github.com/events"
+    end
+
+    private
+
+    def repo_bound_items(user, repo, entity, urls, selector, descriminator,
+                         item_id = nil)
 
-      watchers = paged_api_request(ghurl "repos/#{user}/#{repo}/watchers")
-      watchers.each do |x|
+      items = if urls.class == Array
+                urls.map { |url| paged_api_request(ghurl url) }.flatten
+              else
+                paged_api_request(ghurl urls)
+              end
+
+      items.each do |x|
         x['repo'] = repo
         x['owner'] = user
 
-        exists = !stored_watchers.find { |f|
-          f['login'] == x['login']
-        }.nil?
+        exists = !repo_bound_instance(entity, selector,
+                                      descriminator, x[descriminator]).empty?
 
         if not exists
-          @persister.store(:watchers, x)
-          info "Retriever: Added watcher #{repo} -> #{x['login']}"
+          persister.store(entity, x)
+          info "Retriever: Added #{entity} #{user}/#{repo} -> #{x[descriminator]}"
         else
-          debug "Retriever: Watcher #{repo} -> #{x['login']} exists"
+          debug "Retriever: #{entity} #{user}/#{repo} -> #{x[descriminator]} exists"
         end
       end
 
-      @persister.find(:watchers, {'repo' => repo, 'owner' => user})
+      if item_id.nil?
+        persister.find(entity, selector)
+      else
+        repo_bound_instance(entity, selector, descriminator, item_id)
+      end
     end
 
-    # Retrieve a single watcher for a repositry
-    def retrieve_watcher(user, repo, watcher)
-      stored_watcher = @persister.find(:watchers,
-                                {'repo' => repo,
-                                 'owner' => user,
-                                 'login' => watcher})
+    def repo_bound_item(user, repo, item_id, entity, url, selector, descriminator)
+      stored_item = repo_bound_instance(entity, selector, descriminator, item_id)
 
-      if stored_watcher.empty?
-        retrieve_watchers(user, repo).find{|x| x['login'] == watcher}
+      if stored_item.empty?
+        repo_bound_items(user, repo, entity, url, selector, descriminator,
+                         item_id).first
       else
-        stored_watcher.first
+        stored_item.first
       end
     end
 
-    # Get current Github events
-    def get_events
-      api_request "https://api.github.com/events"
-    end
+    def repo_bound_instance(entity, selector, descriminator, item_id)
 
-    private
+      id = if item_id.to_i.to_s != item_id
+             item_id # item_id is string
+           else
+             item_id.to_i # convert to int
+           end
+
+      instance_selector = selector.merge({descriminator => id})
+      persister.find(entity, instance_selector)
+    end
 
     def ghurl(path)
       config(:mirror_urlbase) + path