ghtorrent 0.6 → 0.7.1

data/lib/ghtorrent/commands/ght_load.rb

@@ -122,11 +122,11 @@ Loads object ids from a collection to a queue for further processing.
         connection.close { EventMachine.stop }
       }
 
-      # Read next 1000 items and queue them
+      # Read next 100000 items and queue them
       read_and_publish = Proc.new {
 
         to_read = if options.number == -1
-                    1000
+                    100000
                   else
                     if options.number - num_read - 1 <= 0
                       -1

data/lib/ghtorrent/commands/ght_retrieve_repo.rb

@@ -46,31 +46,44 @@ An efficient way to get all data for a single repo
   end
 
   def go
+    self.settings = override_config(settings, :mirror_history_pages_back, -1)
     user_entry = ght.transaction{ght.ensure_user(ARGV[0], false, false)}
 
     if user_entry.nil?
-      Trollop::die "Cannot find user #{owner}"
+      Trollop::die "Cannot find user #{ARGV[0]}"
     end
 
     user = user_entry[:login]
 
-    repo_entry = ght.transaction{ght.ensure_repo(ARGV[0], ARGV[1], false, false, false)}
+    repo_entry = ght.transaction{ght.ensure_repo(ARGV[0], ARGV[1], false, false,
+                                                 false, false)}
 
     if repo_entry.nil?
-      Trollop::die "Cannot find repository #{owner}/#{ARGV[1]}"
+      Trollop::die "Cannot find repository #{ARGV[0]}/#{ARGV[1]}"
     end
 
     repo = repo_entry[:name]
 
-    %w(ensure_commits ensure_forks ensure_pull_requests
-       ensure_issues ensure_project_members ensure_watchers).each {|x|
+    def send_message(function, user, repo)
       begin
-        ght.send(x, user, repo)
+        ght.send(function, user, repo, refresh = true)
       rescue Exception => e
         puts STDERR, e.message
         puts STDERR, e.backtrace
       end
-    }
+    end
+
+    functions = %w(ensure_commits ensure_forks ensure_pull_requests
+       ensure_issues ensure_project_members ensure_watchers)
+
+    if ARGV[2].nil?
+      functions.each do |x|
+        send_message(x, user, repo)
+      end
+    else
+      Trollop::die("Not a valid function: #{ARGV[2]}") unless functions.include? ARGV[2]
+      send_message(ARGV[2], user, repo)
+    end
   end
 end
 
@@ -79,40 +92,57 @@ end
 class TransactedGHTorrent < GHTorrent::Mirror
 
   def ensure_commit(repo, sha, user, comments = true)
-    transaction do
+    check_transaction do
       super(repo, sha, user, comments)
     end
   end
 
-  def ensure_fork(owner, repo, fork_id, date_added = nil)
-    transaction do
-      super(owner, repo, fork_id, date_added)
+  def ensure_fork(owner, repo, fork_id)
+    check_transaction do
+      super(owner, repo, fork_id)
     end
   end
 
   def ensure_pull_request(owner, repo, pullreq_id,
       comments = true, commits = true,
       state = nil, created_at = nil)
-    transaction do
+    check_transaction do
       super(owner, repo, pullreq_id, comments, commits, state, created_at)
     end
   end
 
   def ensure_issue(owner, repo, issue_id, events = true, comments = true)
-    transaction do
+    check_transaction do
       super(owner, repo, issue_id, events, comments)
     end
   end
 
   def ensure_project_member(owner, repo, new_member, date_added)
-    transaction do
+    check_transaction do
       super(owner, repo, new_member, date_added)
     end
   end
 
   def ensure_watcher(owner, repo, watcher, date_added = nil)
-    transaction do
+    check_transaction do
       super(owner, repo, watcher, date_added)
     end
   end
+
+  def check_transaction(&block)
+    begin
+      if @db.in_transaction?
+        debug "Transaction already started"
+        yield block
+      else
+        transaction do
+          yield block
+        end
+      end
+    rescue Exception => e
+      puts STDERR, e.message
+      puts STDERR, e.backtrace
+    end
+  end
+
 end

data/lib/ghtorrent/commands/ght_retrieve_user.rb

@@ -0,0 +1,72 @@
+require 'rubygems'
+
+require 'ghtorrent/ghtorrent'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+require 'ghtorrent/retriever'
+require 'ghtorrent/commands/ght_retrieve_repo'
+
+class GHTRetrieveUser < GHTRetrieveRepo
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+An efficient way to get all data for a single user
+
+#{command_name} [options] user
+
+    BANNER
+  end
+
+  def validate
+    super
+    Trollop::die "One argument are required" unless args[0] && !args[0].empty?
+  end
+
+  def go
+    self.settings = override_config(settings, :mirror_history_pages_back, -1)
+    user_entry = ght.transaction{ght.ensure_user(ARGV[0], false, false)}
+
+    if user_entry.nil?
+      Trollop::die "Cannot find user #{ARGV[0]}"
+    end
+
+    user = user_entry[:login]
+
+    def send_message(function, user)
+      begin
+        ght.send(function, user)
+      rescue Exception => e
+        puts STDERR, e.message
+        puts STDERR, e.backtrace
+      end
+    end
+
+    functions = %w(ensure_user_followers ensure_orgs)
+
+    if ARGV[2].nil?
+      functions.each do |x|
+        send_message(x, user)
+      end
+    else
+      Trollop::die("Not a valid function: #{ARGV[2]}") unless functions.include? ARGV[2]
+      send_message(ARGV[2], user)
+    end
+
+  end
+end
+
+class TransactedGHTorrent < GHTorrent::Mirror
+
+  def ensure_user_followers(user)
+    check_transaction do
+      super(user)
+    end
+  end
+
+  def ensure_orgs(user)
+    check_transaction do
+      super(user)
+    end
+  end
+end

data/lib/ghtorrent/ghtorrent.rb

@@ -24,6 +24,8 @@ module GHTorrent
 
     # Get a connection to the database
     def get_db
+      return @db unless @db.nil?
+
       Sequel.single_threaded = true
       @db = Sequel.connect(config(:sql_url), :encoding => 'utf8')
       #@db.loggers << @logger
@@ -54,7 +56,6 @@ module GHTorrent
       end
 
       transaction do
-        ensure_user(user, true, true)
         ensure_commit(repo, sha, user)
       end
     end
@@ -78,10 +79,9 @@ module GHTorrent
     #  [user] The login of the repository owner
     #  [repo] The name of the repository
     #  [comment_id] The login of the member to add
-    #  [date_added] The timestamp that the add event took place
-    def get_commit_comment(user, repo, comment_id, date_added)
+    def get_commit_comment(user, repo, comment_id)
       transaction do
-        ensure_commit_comment(user, repo, comment_id, date_added)
+        ensure_commit_comment(user, repo, comment_id)
       end
     end
 
@@ -106,8 +106,6 @@ module GHTorrent
     #  [date_added] The timestamp that the add event took place
     def get_follower(follower, followed, date_added)
       transaction do
-        ensure_user(follower, true, true)
-        ensure_user(followed, true, true)
         ensure_user_follower(followed, follower, date_added)
       end
     end
@@ -130,10 +128,9 @@ module GHTorrent
     #  [owner] The login of the repository owner
     #  [repo] The name of the repository
     #  [fork_id] The fork item id
-    #  [date_added] The timestamp that the add event took place
-    def get_fork(owner, repo, fork_id, date_added)
+    def get_fork(owner, repo, fork_id)
       transaction do
-        ensure_fork(owner, repo, fork_id, date_added)
+        ensure_fork(owner, repo, fork_id)
       end
     end
 
@@ -144,9 +141,9 @@ module GHTorrent
     #  [repo] The name of the repository
     #  [fork_id] The fork item id
     #  [date_added] The timestamp that the add event took place
-    def get_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+    def get_pullreq_comment(owner, repo, pullreq_id, comment_id)
       transaction do
-        ensure_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+        ensure_pullreq_comment(owner, repo, pullreq_id, comment_id)
       end
     end
 
@@ -158,9 +155,9 @@ module GHTorrent
     #  [issue_id] The fork item id
     #  [action] The action that took place for the issue
     #  [date_added] The timestamp that the add event took place
-    def get_issue(owner, repo, issue_id, created_at)
+    def get_issue(owner, repo, issue_id)
       transaction do
-        ensure_issue(owner, repo, issue_id, created_at)
+        ensure_issue(owner, repo, issue_id)
       end
     end
 
@@ -181,7 +178,7 @@ module GHTorrent
     # Make sure a commit exists
     #
     def ensure_commit(repo, sha, user, comments = true)
-      ensure_repo(user, repo)
+      ensure_repo(user, repo, false, false, false, false)
       c = retrieve_commit(repo, sha, user)
 
       if c.nil?
@@ -208,8 +205,9 @@ module GHTorrent
     # [sha]   The first commit to start retrieving from. If nil, then the
     #         earliest stored commit will be used instead.
     # [num_pages] The number of commit pages to retrieve
-    def ensure_commits(user, repo, sha = nil,
-                       num_pages = config(:mirror_commit_pages_new_repo))
+    def ensure_commits(user, repo, refresh = false, sha = nil,
+                       num_pages = config(:mirror_commit_pages_new_repo)
+                       )
       userid = @db[:users].filter(:login => user).first[:id]
       repoid = @db[:projects].filter(:owner_id => userid,
                                      :name => repo).first[:id]
@@ -236,32 +234,37 @@ module GHTorrent
     # in the database.
     def ensure_parents(commit)
       commits = @db[:commits]
-      commit['parents'].each do |p|
-          parents =  @db[:commit_parents]
-          url = p['url'].split(/\//)
-          this = commits.first(:sha => commit['sha'])
-          parent = commits.first(:sha => url[7])
-
-          if parent.nil?
-            store_commit(retrieve_commit(url[5], url[7], url[4]), url[5], url[4])
-            parent = commits.first(:sha => url[7])
-          end
-
-          if parent.nil?
+      parents = @db[:commit_parents]
+      commit['parents'].map do |p|
+        url = p['url'].split(/\//)
+        this = commits.first(:sha => commit['sha'])
+        parent = commits.first(:sha => url[7])
+
+        if parent.nil?
+          c = retrieve_commit(url[5], url[7], url[4])
+          if c.nil?
             warn "GHTorrent: Could not retrieve #{url[4]}/#{url[5]} -> #{url[7]}, parent to commit #{this[:sha]}"
-            return
+            next
           end
+          parent = store_commit(c, url[5], url[4])
+        end
 
-          if parents.first(:commit_id => this[:id],
-                           :parent_id => parent[:id]).nil?
+        if parent.nil?
+          warn "GHTorrent: Could not retrieve #{url[4]}/#{url[5]} -> #{url[7]}, parent to commit #{this[:sha]}"
+          next
+        end
 
-            parents.insert(:commit_id => this[:id],
-                           :parent_id => parent[:id])
-            info "GHTorrent: Added parent #{parent[:sha]} to commit #{this[:sha]}"
-          else
-            debug "GHTorrent: Parent #{parent[:sha]} for commit #{this[:sha]} exists"
-          end
+        if parents.first(:commit_id => this[:id],
+                         :parent_id => parent[:id]).nil?
+
+          parents.insert(:commit_id => this[:id],
+                         :parent_id => parent[:id])
+          info "GHTorrent: Added parent #{parent[:sha]} to commit #{this[:sha]}"
+        else
+          debug "GHTorrent: Parent #{parent[:sha]} for commit #{this[:sha]} exists"
         end
+        parents.first(:commit_id => this[:id], :parent_id => parent[:id])
+      end
     end
 
     ##
@@ -271,23 +274,27 @@ module GHTorrent
     #  [repo] The repo receiving the commit
     #  [sha] The commit SHA
     def ensure_repo_commit(user, repo, sha)
-      userid = @db[:users].first(:login => user)[:id]
-      projectid = @db[:projects].first(:owner_id => userid,
-                                     :name => repo)[:id]
+      project = ensure_repo(user, repo, false, false, false, false)
+
+      if project.nil?
+        warn "GHTorrent: Repo #{user}/#{repo} does not exist"
+        return
+      end
+
       commitid = @db[:commits].first(:sha => sha)[:id]
 
-      exists = @db[:project_commits].first(:project_id => projectid,
+      exists = @db[:project_commits].first(:project_id => project[:id],
                                            :commit_id => commitid)
       if exists.nil?
         @db[:project_commits].insert(
-            :project_id => projectid,
+            :project_id => project[:id],
             :commit_id => commitid
         )
-        info "GHTorrent: Added commit #{user}/#{repo} -> #{sha}"
-        @db[:project_commits].first(:project_id => projectid,
+        info "GHTorrent: Associating commit  #{sha} with #{user}/#{repo}"
+        @db[:project_commits].first(:project_id => project[:id],
                                     :commit_id => commitid)
       else
-        debug "GHTorrent: Commit #{user}/#{repo} -> #{sha} exists"
+        debug "GHTorrent: Commit #{sha} already associated with #{user}/#{repo}"
         exists
       end
     end
@@ -333,6 +340,17 @@ module GHTorrent
             return users.first(:login => byemail[:login])
           end
 
+          # This means that the user's login has been associated with a
+          # Github user by the time the commit was done (and hence Github was
+          # able to associate the commit to an account), but afterwards the
+          # user has deleted his account (before GHTorrent processed it).
+          # On absense of something better to do, try to find the user by email
+          # and return a "fake" user entry.
+          if added.nil?
+            warn "GHTorrent: User account for user #{login} deleted from Github"
+            return ensure_user("#{name}<#{email}>", false, false)
+          end
+
           if byemail.nil?
             users.filter(:login => login).update(:name => name) if added[:name].nil?
             users.filter(:login => login).update(:email => email) if added[:email].nil?
@@ -373,10 +391,12 @@ module GHTorrent
     def ensure_user(user, followers, orgs)
       # Github only supports alpa-nums and dashes in its usernames.
       # All other sympbols are treated as emails.
-      if not user.match(/^[A-Za-z0-9\-]*$/)
+      if not user.match(/^[\w\-]*$/)
         begin
           name, email = user.split("<")
           email = email.split(">")[0]
+          name = name.strip unless name.nil?
+          email = email.strip unless email.nil?
         rescue Exception
           raise new GHTorrentException.new("Not a valid email address: #{user}")
         end
@@ -384,7 +404,7 @@ module GHTorrent
         unless is_valid_email(email)
           warn("GHTorrent: Extracted email(#{email}) not valid for user #{user}")
         end
-        u = ensure_user_byemail(email.strip, name.strip)
+        u = ensure_user_byemail(email, name)
       else
         u = ensure_user_byuname(user)
         ensure_user_followers(user) if followers
@@ -446,9 +466,8 @@ module GHTorrent
     #
     # ==Parameters:
     # [user]  The user login to find followers by
-    def ensure_user_followers(followed, date_added = nil)
+    def ensure_user_followers(followed)
       curuser = ensure_user(followed, false, false)
-      time = curuser[:created_at]
       followers = @db.from(:followers, :users).\
           where(:followers__follower_id => :users__id).
           where(:followers__user_id => curuser[:id]).select(:login).all
@@ -459,12 +478,12 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_user_follower(followed, x['login'], time) }
+      end.map { |x| ensure_user_follower(followed, x['login']) }
     end
 
     ##
     # Make sure that a user follows another one
-    def ensure_user_follower(followed, follower, date_added)
+    def ensure_user_follower(followed, follower, date_added = nil)
       follower_user = ensure_user(follower, false, false)
       followed_user = ensure_user(followed, false, false)
 
@@ -474,14 +493,17 @@ module GHTorrent
       end
 
       followers = @db[:followers]
-      followed_id = follower_user[:id]
-      follower_id = followed_user[:id]
+      follower_id = follower_user[:id]
+      followed_id = followed_user[:id]
 
       follower_exists = followers.first(:user_id => followed_id,
                                         :follower_id => follower_id)
-
       if follower_exists.nil?
-        added = if date_added.nil? then Time.now else date_added end
+        added = if date_added.nil?
+                  max(follower_user[:created_at], followed_user[:created_at])
+                else
+                  date_added
+                end
         retrieved = retrieve_user_follower(followed, follower)
 
         if retrieved.nil?
@@ -495,13 +517,17 @@ module GHTorrent
                          :ext_ref_id => retrieved[@ext_uniq])
         info "GHTorrent: User #{follower} follows #{followed}"
       else
-        unless date_added.nil?
-          followers.filter(:user_id => followed_id,
-                           :follower_id => follower_id)\
+        debug "GHTorrent: Follower #{follower} exists for user #{followed}"
+      end
+
+      unless date_added.nil?
+        followers.filter(:user_id => followed_id,
+                         :follower_id => follower_id)\
                     .update(:created_at => date(date_added))
-          debug "GHTorrent: Updating follower #{followed} -> #{follower}"
-        end
+        debug "GHTorrent: Updating follower #{followed} -> #{follower}, created_at -> #{date(date_added)}"
       end
+
+      followers.first(:user_id => followed_id, :follower_id => follower_id)
     end
 
     ##
@@ -529,19 +555,29 @@ module GHTorrent
                        :name => name,
                        :login => login,
                        :created_at => Time.now,
-                       :ext_ref_id => ""
-          )
+                       :ext_ref_id => "")
           info "GHTorrent: Added fake user #{login} -> #{email}"
           users.first(:login => login)
         else
-          users.insert(:login => u['login'],
-                       :name => u['name'],
-                       :company => u['company'],
-                       :email => u['email'],
-                       :location => u['location'],
-                       :created_at => date(u['created_at']),
-                       :ext_ref_id => u[@ext_uniq])
-          info "GHTorrent: Found #{email} through search API query"
+          in_db = users.first(:login => u['login'])
+          if in_db.nil?
+            users.insert(:login => u['login'],
+                         :name => u['name'],
+                         :company => u['company'],
+                         :email => u['email'],
+                         :location => u['location'],
+                         :created_at => date(u['created_at']),
+                         :ext_ref_id => u[@ext_uniq])
+            info "GHTorrent: Found #{email} through search API query"
+          else
+            in_db.update(:name => u['name'],
+                         :company => u['company'],
+                         :email => u['email'],
+                         :location => u['location'],
+                         :created_at => date(u['created_at']),
+                         :ext_ref_id => u[@ext_uniq])
+            info "GHTorrent: User with email #{email} exists with username #{u['login']}"
+          end
           users.first(:login => u['login'])
         end
       else
@@ -560,10 +596,17 @@ module GHTorrent
     # == Returns:
     #  If the repo can be retrieved, it is returned as a Hash. Otherwise,
     #  the result is nil
-    def ensure_repo(user, repo, commits = true, project_members = true, watchers = true)
+    def ensure_repo(user, repo, commits = true, project_members = true,
+                    watchers = true, forks = true)
 
       repos = @db[:projects]
       curuser = ensure_user(user, false, false)
+
+      if curuser.nil?
+        warn "Cannot find user #{user}"
+        return
+      end
+
       currepo = repos.first(:owner_id => curuser[:id], :name => repo)
 
       if currepo.nil?
@@ -575,17 +618,29 @@ module GHTorrent
         end
 
         repos.insert(:url => r['url'],
-                     :owner_id => @db[:users].filter(:login => user).first[:id],
+                     :owner_id => curuser[:id],
                      :name => r['name'],
                      :description => r['description'],
                      :language => r['language'],
                      :created_at => date(r['created_at']),
                      :ext_ref_id => r[@ext_uniq])
 
-        info "GHTorrent: New repo #{repo}"
+        unless r['parent'].nil?
+          parent_owner = r['parent']['owner']['login']
+          parent_repo = r['parent']['name']
+
+          parent = ensure_repo(parent_owner, parent_repo, false, false, false, false)
+
+          repos.filter(:owner_id => curuser[:id], :name => repo).update(:forked_from => parent[:id])
+
+          info "Repo #{user}/#{repo} is a fork from #{parent_owner}/#{parent_repo}"
+        end
+
+        info "GHTorrent: New repo #{user}/#{repo}"
         ensure_commits(user, repo) if commits
         ensure_project_members(user, repo) if project_members
         ensure_watchers(user, repo) if watchers
+        ensure_forks(user, repo) if forks
         repos.first(:owner_id => curuser[:id], :name => repo)
       else
         debug "GHTorrent: Repo #{user}/#{repo} exists"
@@ -595,8 +650,8 @@ module GHTorrent
 
     ##
     # Make sure that a project has all the registered members defined
-    def ensure_project_members(user, repo)
-      currepo = ensure_repo(user, repo, true, false, true)
+    def ensure_project_members(user, repo, refresh = false)
+      currepo = ensure_repo(user, repo, false, false, false, false)
       time = currepo[:created_at]
 
       project_members = @db.from(:project_members, :users).\
@@ -616,7 +671,7 @@ module GHTorrent
     # Make sure that a project member exists in a project
     def ensure_project_member(owner, repo, new_member, date_added)
       pr_members = @db[:project_members]
-      project = ensure_repo(owner, repo, true, false, true)
+      project = ensure_repo(owner, repo, false, false, false, false)
       new_user = ensure_user(new_member, false, false)
 
       if project.nil? or new_user.nil?
@@ -627,7 +682,11 @@ module GHTorrent
                                     :repo_id => project[:id])
 
       if memb_exist.nil?
-        added = if date_added.nil? then Time.now else date_added end
+        added = if date_added.nil?
+                  max(project[:created_at], new_user[:created_at])
+                else
+                  date_added
+                end
         retrieved = retrieve_repo_collaborator(owner, repo, new_member)
 
         if retrieved.nil?
@@ -644,12 +703,13 @@ module GHTorrent
         info "GHTorrent: Added project member #{repo} -> #{new_member}"
       else
         debug "GHTorrent: Project member #{repo} -> #{new_member} exists"
-        unless date_added.nil?
-          pr_members.filter(:user_id => new_user[:id],
-                            :repo_id => project[:id])\
+      end
+
+      unless date_added.nil?
+        pr_members.filter(:user_id => new_user[:id],
+                          :repo_id => project[:id])\
                     .update(:created_at => date(date_added))
-          info "GHTorrent: Updating project member #{repo} -> #{new_member}"
-        end
+        info "GHTorrent: Updating project member #{repo} -> #{new_member}, created_at -> #{date(date_added)}"
       end
     end
 
@@ -734,7 +794,7 @@ module GHTorrent
         end
       end
 
-      not_saved.map{|x| ensure_commit_comment(user, repo, x['id'], nil)}
+      not_saved.map{|x| ensure_commit_comment(user, repo, x['id'])}
     end
 
     ##
@@ -745,7 +805,7 @@ module GHTorrent
     # [repo]  The repository containing the commit whose comment will be retrieved
     # [id]  The comment id to retrieve
     # [created_at]  The timestamp that the comment was made.
-    def ensure_commit_comment(user, repo, id, created_at)
+    def ensure_commit_comment(user, repo, id)
       stored_comment = @db[:commit_comments].first(:comment_id => id)
 
       if stored_comment.nil?
@@ -770,21 +830,15 @@ module GHTorrent
         )
         info "GHTorrent: Added commit comment #{commit[:sha]} -> #{user[:login]}"
       else
-        unless created_at.nil?
-          @db[:commit_comments].filter(:comment_id => id)\
-                               .update(:created_at => date(created_at))
-          info "GHTorrent: Updating comment #{user}/#{repo} -> #{id}"
-        end
         info "GHTorrent: Commit comment #{id} exists"
       end
       @db[:commit_comments].first(:comment_id => id)
     end
 
     ##
-    # Make sure that
-    def ensure_watchers(owner, repo)
-      currepo = ensure_repo(owner, repo, true, true, false)
-      time = currepo[:created_at]
+    # Make sure that all watchers exist for a repository
+    def ensure_watchers(owner, repo, refresh = false)
+      currepo = ensure_repo(owner, repo, false, false, false, false)
 
       if currepo.nil?
         warn "Could not retrieve watchers for #{owner}/#{repo}"
@@ -803,13 +857,13 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_watcher(owner, repo, x['login'], time) }
+      end.map { |x| ensure_watcher(owner, repo, x['login'], nil) }
     end
 
     ##
-    # Make sure that a project member exists in a project
+    # Make sure that a watcher/stargazer exists for a repository
     def ensure_watcher(owner, repo, watcher, date_added = nil)
-      project = ensure_repo(owner, repo, false, false, false)
+      project = ensure_repo(owner, repo, false, false, false, false)
       new_watcher = ensure_user(watcher, false, false)
 
       if new_watcher.nil? or project.nil?
@@ -818,11 +872,15 @@ module GHTorrent
       end
 
       watchers = @db[:watchers]
-      memb_exist = watchers.first(:user_id => new_watcher[:id],
-                                  :repo_id => project[:id])
+      watcher_exist = watchers.first(:user_id => new_watcher[:id],
+                                     :repo_id => project[:id])
 
-      if memb_exist.nil?
-        added = if date_added.nil? then Time.now else date_added end
+      if watcher_exist.nil?
+        added = if date_added.nil?
+                  max(project[:created_at], new_watcher[:created_at])
+                else
+                  date_added
+                end
         retrieved = retrieve_watcher(owner, repo, watcher)
 
         if retrieved.nil?
@@ -839,33 +897,42 @@ module GHTorrent
         info "GHTorrent: Added watcher #{owner}/#{repo} -> #{watcher}"
       else
         debug "GHTorrent: Watcher #{owner}/#{repo} -> #{watcher} exists"
-        unless date_added.nil?
-          watchers.filter(:user_id => new_watcher[:id],
-                          :repo_id => project[:id])\
+      end
+
+      unless date_added.nil?
+        watchers.filter(:user_id => new_watcher[:id],
+                        :repo_id => project[:id])\
                   .update(:created_at => date(date_added))
-          info "GHTorrent: Updating watcher #{owner}/#{repo} -> #{watcher}"
-        end
+        info "GHTorrent: Updating watcher #{owner}/#{repo} -> #{watcher}, created_at -> #{date_added}"
       end
+
+      watchers.first(:user_id => new_watcher[:id],
+                     :repo_id => project[:id])
     end
 
     ##
     # Process all pull requests
-    def ensure_pull_requests(owner, repo)
-      currepo = ensure_repo(owner, repo, false, false, false)
+    def ensure_pull_requests(owner, repo, refresh = false)
+      currepo = ensure_repo(owner, repo, false, false, false, false)
       if currepo.nil?
         warn "Could not retrieve pull requests from #{owner}/#{repo}"
         return
       end
 
-      pull_reqs = @db[:pull_requests].filter(:base_repo_id => currepo[:id]).all
+      raw_pull_reqs = if refresh
+                        retrieve_pull_requests(owner, repo, refresh = true)
+                      else
+                        pull_reqs = @db[:pull_requests].filter(:base_repo_id => currepo[:id]).all
+                        retrieve_pull_requests(owner, repo).reduce([]) do |acc, x|
+                          if pull_reqs.find { |y| y[:pullreq_id] == x['number'] }.nil?
+                            acc << x
+                          else
+                            acc
+                          end
+                        end
+                      end
 
-      retrieve_pull_requests(owner, repo).reduce([]) do |acc, x|
-        if pull_reqs.find { |y| y[:pullreq_id] == x['number'] }.nil?
-          acc << x
-        else
-          acc
-        end
-      end.map { |x| ensure_pull_request(owner, repo, x['number']) }
+      raw_pull_reqs.map { |x| ensure_pull_request(owner, repo, x['number']) }
     end
 
     ##
@@ -875,7 +942,7 @@ module GHTorrent
                             state = nil, created_at = nil)
       pulls_reqs = @db[:pull_requests]
 
-      project = ensure_repo(owner, repo, false, false, false)
+      project = ensure_repo(owner, repo, false, false, false, false)
 
       if project.nil?
         return
@@ -885,15 +952,13 @@ module GHTorrent
       def add_history(id, ts, unq, act)
         pull_req_history = @db[:pull_request_history]
         entry = pull_req_history.first(:pull_request_id => id,
-                                       :ext_ref_id => unq, :action => act)
+                                       :created_at => ts, :action => act)
         if entry.nil?
           pull_req_history.insert(:pull_request_id => id, :created_at => ts,
                                   :ext_ref_id => unq, :action => act)
-          info "GHTorrent: New pull request (#{id}) history entry (#{act})"
+          info "GHTorrent: New pull request (#{id}) history entry (#{act}) timestamp #{ts}"
         else
-          pull_req_history.filter(:pull_request_id => id, :ext_ref_id => unq,
-                                :action => act).update(:created_at => ts)
-          info "GHTorrent: Updating pull request (#{id}) history entry (#{act}) timestamp #{ts}"
+          info "GHTorrent: Pull request (#{id}) history entry (#{act}) timestamp #{ts} exists"
         end
       end
 
@@ -939,7 +1004,7 @@ module GHTorrent
 
       base_repo = ensure_repo(retrieved['base']['repo']['owner']['login'],
                               retrieved['base']['repo']['name'],
-                              false, false, false)
+                              false, false, false, false)
 
       base_commit = ensure_commit(retrieved['base']['repo']['name'],
                                   retrieved['base']['sha'],
@@ -955,7 +1020,7 @@ module GHTorrent
         head_repo = if has_head_repo(retrieved)
                       ensure_repo(retrieved['head']['repo']['owner']['login'],
                                   retrieved['head']['repo']['name'],
-                                  false, false, false)
+                                  false, false, false, false)
                     end
 
         head_commit = if not head_repo.nil?
@@ -967,7 +1032,12 @@ module GHTorrent
 
       pull_req_user = ensure_user(retrieved['user']['login'], false, false)
 
-      merged = if retrieved['merged_at'].nil? then false else true end
+      merged = if retrieved['merged_at'].nil? then
+               # Check if the pr's commits are in the repository
+                false
+               else
+                 true
+               end
       closed = if retrieved['closed_at'].nil? then false else true end
 
       pull_req = pulls_reqs.first(:base_repo_id => project[:id],
@@ -980,10 +1050,11 @@ module GHTorrent
             :base_commit_id => base_commit[:id],
             :user_id => pull_req_user[:id],
             :pullreq_id => pullreq_id,
-            :intra_branch => is_intra_branch(retrieved)
+            :intra_branch => is_intra_branch(retrieved),
+            :merged => merged
         )
 
-        info log_msg(retrieved)
+        info log_msg(retrieved) + " was added"
       else
         debug log_msg(retrieved) + " exists"
       end
@@ -1001,22 +1072,21 @@ module GHTorrent
                        state) unless state.nil?
 
       ensure_pull_request_commits(owner, repo, pullreq_id) if commits
-      ensure_pullreq_comments(owner, repo, pullreq_id, created_at) if comments
+      ensure_pullreq_comments(owner, repo, pullreq_id) if comments
 
       pulls_reqs.first(:base_repo_id => project[:id],
                        :pullreq_id => pullreq_id)
     end
 
-    def ensure_pullreq_comments(owner, repo, pullreq_id, created_at)
-      currepo = ensure_repo(owner, repo, true, true, false)
-      time = if created_at.nil? then currepo[:created_at] else Time.now() end
+    def ensure_pullreq_comments(owner, repo, pullreq_id)
+      currepo = ensure_repo(owner, repo, false, false, false, false)
 
       if currepo.nil?
         warn "GHTorrent: Could not find repository #{owner}/#{repo}"
         return
       end
 
-      pull_req = ensure_pull_request(owner, repo, pullreq_id, false, true)
+      pull_req = ensure_pull_request(owner, repo, pullreq_id, false, false)
 
       if pull_req.nil?
         warn "Could not retrieve pull req #{owner}/#{repo} -> #{pullreq_id}"
@@ -1032,12 +1102,16 @@ module GHTorrent
           acc
         end
       end.map { |x|
-        ensure_pullreq_comment(owner, repo, pullreq_id, x['id'], time)
+        ensure_pullreq_comment(owner, repo, pullreq_id, x['id'])
       }
     end
 
-    def ensure_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
-      pull_req = ensure_pull_request(owner, repo, pullreq_id, false, true)
+    def ensure_pullreq_comment(owner, repo, pullreq_id, comment_id)
+      # Commit retrieval is set to false to ensure that no duplicate work
+      # is done on retrieving a pull request. This has the side effect that
+      # commits might not be retrieved if a pullreqcomment event gets processed
+      # before the pullreq event, until the pullreq event has been processed
+      pull_req = ensure_pull_request(owner, repo, pullreq_id, false, false)
 
       if pull_req.nil?
         warn "GHTorrent: Could not retrieve pull req #{owner}/#{repo} -> #{pullreq_id}"
@@ -1084,12 +1158,21 @@ module GHTorrent
     end
 
     def ensure_pull_request_commits(owner, repo, pullreq_id)
+      pullreq = ensure_pull_request(owner, repo, pullreq_id, false, false)
+
+      if pullreq.nil?
+        warn "GHTorrent: Pull request #{pullreq_id} does not exist for #{owner}/#{repo}"
+        return
+      end
+
       retrieve_pull_req_commits(owner, repo, pullreq_id).reduce([]){|acc, c|
-        x = ensure_commit(repo, c['sha'], owner, true)
+        next if c.nil?
+        head_repo_owner = c['url'].split(/\//)[4]
+        head_repo_name = c['url'].split(/\//)[5]
+        x = ensure_commit(head_repo_name, c['sha'], head_repo_owner, true)
         acc << x if not x.nil?
         acc
       }.map { |c|
-        pullreq = ensure_pull_request(owner, repo, pullreq_id, false, false)
         exists = @db[:pull_request_commits].first(:pull_request_id => pullreq[:id],
                                                   :commit_id => c[:id])
         if exists.nil?
@@ -1110,18 +1193,17 @@ module GHTorrent
     # ==Parameters:
     # [owner]  The user to which the project belongs
     # [repo]  The repository/project to find forks for
-    def ensure_forks(owner, repo)
-      currepo = ensure_repo(owner, repo, false, false, false)
+    def ensure_forks(owner, repo, refresh = false)
+      currepo = ensure_repo(owner, repo, false, false, false, false)
 
       if currepo.nil?
         warn "Could not retrieve forks for #{owner}/#{repo}"
         return
       end
 
-      existing_forks = @db.from(:forks, :projects, :users).\
-          where(:forks__forked_project_id => :projects__id). \
+      existing_forks = @db.from(:projects, :users).\
           where(:users__id => :projects__owner_id). \
-          where(:forks__forked_from_id => currepo[:id]).select(:projects__name, :login).all
+          where(:projects__forked_from => currepo[:id]).select(:projects__name, :login).all
 
       retrieve_forks(owner, repo).reduce([]) do |acc, x|
         if existing_forks.find {|y|
@@ -1138,65 +1220,49 @@ module GHTorrent
 
     ##
     # Make sure that a fork is retrieved for a project
-    def ensure_fork(owner, repo, fork_id, date_added = nil)
-
-      forks = @db[:forks]
-      forked = ensure_repo(owner, repo, false, false, false)
-      fork_exists = forks.first(:fork_id => fork_id)
-
-      if fork_exists.nil?
-        retrieved = retrieve_fork(owner, repo, fork_id)
-        added = if date_added.nil? then retrieved['created_at'] else date_added end
-
-        if retrieved.nil?
-          warn "GHTorrent: Fork #{fork_id} does not exist for #{owner}/#{repo}"
-          return
-        end
+    def ensure_fork(owner, repo, fork_id)
+      fork = retrieve_fork(owner, repo, fork_id)
 
-        forked_repo_owner = retrieved['full_name'].split(/\//)[0]
-        forked_repo_name = retrieved['full_name'].split(/\//)[1]
+      if fork.nil?
+        warn "GHTorrent: Fork #{fork_id} does not exist for #{owner}/#{repo}"
+        return
+      end
 
-        fork = ensure_repo(forked_repo_owner, forked_repo_name)
+      fork_owner = fork['full_name'].split(/\//)[0]
+      fork_name = fork['full_name'].split(/\//)[1]
 
-        if forked.nil? or fork.nil?
-          warn "Could not add fork #{fork_id}"
-          return
-        end
+      r = ensure_repo(fork_owner, fork_name, false, false, false, false)
 
-        forks.insert(:forked_project_id => fork[:id],
-                     :forked_from_id => forked[:id],
-                     :fork_id => fork_id,
-                     :created_at => added,
-                     :ext_ref_id => retrieved[@ext_uniq])
-        info "GHTorrent: Added #{forked_repo_owner}/#{forked_repo_name} as fork of  #{owner}/#{repo}"
+      if r.nil?
+        warn "GHTorrent: Failed to add #{fork_owner}/#{fork_name} as fork of  #{owner}/#{repo}"
       else
-        debug "GHTorrent: Fork #{fork_id} exists as fork of #{owner}/#{repo}"
-        unless date_added.nil?
-          forks.filter(:fork_id => fork_id)\
-               .update(:created_at => date(date_added))
-          debug "GHTorrent: Updating fork #{owner}/#{repo} (#{fork_id})"
-        end
+        info "GHTorrent: Added #{fork_owner}/#{fork_name} as fork of  #{owner}/#{repo}"
       end
     end
 
     ##
     # Make sure all issues exist for a project
-    def ensure_issues(owner, repo)
-      currepo = ensure_repo(owner, repo, false, false, false)
+    def ensure_issues(owner, repo, refresh = false)
+      currepo = ensure_repo(owner, repo, false, false, false, false)
       if currepo.nil?
         warn "GHTorrent: Could not retrieve issues for #{owner}/#{repo}"
         return
       end
 
-      issues = @db[:issues].filter(:repo_id => currepo[:id]).all
+      raw_issues = if refresh
+                     retrieve_issues(owner, repo, refresh = true)
+                   else
+                     issues = @db[:issues].filter(:repo_id => currepo[:id]).all
+                     retrieve_issues(owner, repo).reduce([]) do |acc, x|
+                       if issues.find { |y| y[:issue_id] == x['number'] }.nil?
+                         acc << x
+                       else
+                         acc
+                       end
+                     end
+                   end
 
-      retrieve_issues(owner, repo).reduce([]) do |acc, x|
-        if issues.find { |y| y[:issue_id] == x['number'] }.nil?
-          acc << x
-        else
-          acc
-        end
-      end.map { |x| ensure_issue(owner, repo, x['number']) }
+      raw_issues.map { |x| ensure_issue(owner, repo, x['number']) }
     end
 
     ##
@@ -1204,7 +1270,7 @@ module GHTorrent
     def ensure_issue(owner, repo, issue_id, events = true, comments = true)
 
       issues = @db[:issues]
-      repository = ensure_repo(owner, repo, false, false, false)
+      repository = ensure_repo(owner, repo, false, false, false, false)
 
       if repo.nil?
         warn "Cannot find repo #{owner}/#{repo}"
@@ -1214,24 +1280,27 @@ module GHTorrent
       cur_issue = issues.first(:issue_id => issue_id,
                                :repo_id => repository[:id])
 
-      if cur_issue.nil?
-        retrieved = retrieve_issue(owner, repo, issue_id)
+      retrieved = retrieve_issue(owner, repo, issue_id)
 
-        if retrieved.nil?
-          warn "GHTorrent: Issue #{issue_id} does not exist for #{owner}/#{repo}"
-          return
-        end
+      if retrieved.nil?
+        warn "GHTorrent: Issue #{issue_id} does not exist for #{owner}/#{repo}"
+        return
+      end
+
+      # Pull requests and issues share the same issue_id
+      pull_req = unless retrieved['pull_request'].nil? or
+          retrieved['pull_request']['patch_url'].nil?
+                   info "GHTorrent: Issue #{owner}/#{repo}->#{issue_id} is a pull request"
+                   ensure_pull_request(owner, repo, issue_id)
+                 end
+
+      if cur_issue.nil?
 
         reporter = ensure_user(retrieved['user']['login'], false, false)
         assignee = unless retrieved['assignee'].nil?
                      ensure_user(retrieved['assignee']['login'], false, false)
                    end
 
-        # Pull requests and issues share the same issue_id
-        pull_req = unless retrieved['pull_request'].nil? or retrieved['pull_request']['patch_url'].nil?
-                     ensure_pull_request(owner, repo, issue_id)
-                   end
-
         issues.insert(:repo_id => repository[:id],
                      :assignee_id => unless assignee.nil? then assignee[:id] end,
                      :reporter_id => reporter[:id],
@@ -1241,23 +1310,26 @@ module GHTorrent
                      :created_at => date(retrieved['created_at']),
                      :ext_ref_id => retrieved[@ext_uniq])
 
-        ensure_issue_events(owner, repo, issue_id) if events
-        ensure_issue_comments(owner, repo, issue_id) if comments and retrieved['comments'] > 0
-
         info "GHTorrent: Added issue #{owner}/#{repo} -> #{issue_id}"
-        issues.first(:issue_id => issue_id,
-                     :repo_id => repository[:id])
       else
         info "GHTorrent: Issue #{owner}/#{repo}->#{issue_id} exists"
-        cur_issue
+        if cur_issue[:pull_request] == false and not pull_req.nil?
+          info "GHTorrent: Updating issue #{owner}/#{repo}->#{issue_id} as pull request"
+          issues.filter(:issue_id => issue_id, :repo_id => repository[:id]).update(
+              :pull_request => true,
+              :pull_request_id => pull_req[:id])
+        end
       end
+      ensure_issue_events(owner, repo, issue_id) if events
+      ensure_issue_comments(owner, repo, issue_id) if comments
+      issues.first(:issue_id => issue_id,
+                   :repo_id => repository[:id])
     end
 
     ##
     # Retrieve and process all events for an issue
     def ensure_issue_events(owner, repo, issue_id)
-      currepo = ensure_repo(owner, repo, true, true, false)
-      #time = if created_at.nil? then currepo[:created_at] else Time.now() end
+      currepo = ensure_repo(owner, repo, false, false, false, false)
 
       if currepo.nil?
         warn "GHTorrent: Could not find repository #{owner}/#{repo}"
@@ -1361,7 +1433,7 @@ module GHTorrent
     ##
     # Retrieve and process all comments for an issue
     def ensure_issue_comments(owner, repo, issue_id)
-      currepo = ensure_repo(owner, repo, true, true, false)
+      currepo = ensure_repo(owner, repo, false, false, false, false)
 
       if currepo.nil?
         warn "GHTorrent: Could not find repository #{owner}/#{repo}"
@@ -1390,7 +1462,7 @@ module GHTorrent
     ##
     # Retrieve and process +comment_id+ for an +issue_id+
     def ensure_issue_comment(owner, repo, issue_id, comment_id)
-      issue = ensure_issue(owner, repo, issue_id)
+      issue = ensure_issue(owner, repo, issue_id, false, false)
 
       if issue.nil?
         warn "GHTorrent: Could not retrieve issue #{owner}/#{repo} -> #{issue_id}"
@@ -1467,14 +1539,13 @@ module GHTorrent
         repository = ensure_repo(user, repo, false, false, false)
 
         if repository.nil?
-          warn "Could not store commit #{user}/#{repo} #{c['sha']}"
-          return
+          warn "GHTorrent: repository #{user}/#{repo} deleted"
         end
 
         commits.insert(:sha => c['sha'],
                        :author_id => author[:id],
                        :committer_id => commiter[:id],
-                       :project_id => repository[:id],
+                       :project_id => if repository.nil? then nil else repository[:id] end ,
                        :created_at => date(c['commit']['author']['date']),
                        :ext_ref_id => c[@ext_uniq]
         )
@@ -1513,6 +1584,14 @@ module GHTorrent
     def is_valid_email(email)
       email =~ /^[a-zA-Z][\w\.-]*[a-zA-Z0-9]@[a-zA-Z0-9][\w\.-]*[a-zA-Z0-9]\.[a-zA-Z][a-zA-Z\.]*[a-zA-Z]$/
     end
+
+    def max(a, b)
+      if a >= b
+        a
+      else
+        b
+      end
+    end
   end
 end