ghtorrent 0.4 → 0.5

data/lib/ghtorrent/gh_torrent_exception.rb

@@ -0,0 +1,6 @@
+# Base exception for all GHTorrent exceptions
+module GHTorrent
+  class GHTorrentException < Exception
+
+  end
+end

data/lib/ghtorrent/ghtorrent.rb

@@ -1,5 +1,11 @@
 require 'sequel'
 
+require 'ghtorrent/time'
+require 'ghtorrent/logging'
+require 'ghtorrent/settings'
+require 'ghtorrent/retriever'
+require 'ghtorrent/persister'
+
 module GHTorrent
   class Mirror
 
@@ -8,32 +14,34 @@ module GHTorrent
     include GHTorrent::Retriever
     include GHTorrent::Persister
 
-    attr_reader :settings, :persister
-
-    def initialize(configuration)
+    attr_reader :settings, :persister, :ext_uniq, :logger
 
-      @settings = YAML::load_file configuration
-      super(@settings)
+    def initialize(settings)
+      @settings = settings
       @ext_uniq = config(:uniq_id)
       @logger = Logger.new(STDOUT)
-      @persister = connect(:mongo, @settings)
-      get_db
     end
 
     # db related functions
     def get_db
-
-      @db = Sequel.connect(config(:sql_url))
-
+      Sequel.single_threaded = true
+      @db = Sequel.connect(config(:sql_url), :encoding => 'utf8')
+      #@db.loggers << @logger
       if @db.tables.empty?
         dir = File.join(File.dirname(__FILE__), 'migrations')
         puts "Database empty, running migrations from #{dir}"
         Sequel.extension :migration
         Sequel::Migrator.apply(@db, dir)
       end
+
       @db
     end
 
+    def persister
+      @persister ||= connect(:mongo, @settings)
+      @persister
+    end
+
     ##
     # Ensure that a user exists, or fetch its latest state from Github
     # ==Parameters:
@@ -46,7 +54,7 @@ module GHTorrent
       end
 
       transaction do
-        ensure_repo(user, repo)
+        ensure_user(user, true, true)
         ensure_commit(repo, sha, user)
       end
     end
@@ -60,7 +68,6 @@ module GHTorrent
     #  [date_added] The timestamp that the add event took place
     def get_project_member(owner, repo, new_member, date_added)
       transaction do
-        ensure_repo(owner, repo)
         ensure_project_member(owner, repo, new_member, date_added)
       end
     end
@@ -74,7 +81,6 @@ module GHTorrent
     #  [date_added] The timestamp that the add event took place
     def get_commit_comment(user, repo, comment_id, date_added)
       transaction do
-        ensure_repo(user, repo)
         ensure_commit_comment(user, repo, comment_id, date_added)
       end
     end
@@ -88,7 +94,6 @@ module GHTorrent
     #  [date_added] The timestamp that the add event took place
     def get_watcher(owner, repo, watcher, date_added)
       transaction do
-        ensure_repo(owner, repo)
         ensure_watcher(owner, repo, watcher, date_added)
       end
     end
@@ -101,20 +106,84 @@ module GHTorrent
     #  [date_added] The timestamp that the add event took place
     def get_follower(follower, followed, date_added)
       transaction do
-        ensure_user(follower, false, false)
-        ensure_user(followed, false, false)
-        ensure_user_followers(followed, date_added)
+        ensure_user(follower, true, true)
+        ensure_user(followed, true, true)
+        ensure_user_follower(followed, follower, date_added)
+      end
+    end
+
+    ##
+    # Get a pull request and record the changes it affects
+    # ==Parameters:
+    #  [owner] The owner of the repository to which the pullreq will be applied
+    #  [repo] The repository to which the pullreq will be applied
+    #  [pullreq_id] The ID of the pull request relative to the repository
+    def get_pull_request(owner, repo, pullreq_id, state, created_at)
+      transaction do
+        ensure_pull_request(owner, repo, pullreq_id, true, true, state, created_at)
+      end
+    end
+
+    ##
+    # Retrieve details about a project fork (including the forked project)
+    # ==Parameters:
+    #  [owner] The login of the repository owner
+    #  [repo] The name of the repository
+    #  [fork_id] The fork item id
+    #  [date_added] The timestamp that the add event took place
+    def get_fork(owner, repo, fork_id, date_added)
+      transaction do
+        ensure_fork(owner, repo, fork_id, date_added)
       end
     end
 
+    ##
+    # Retrieve a pull request review comment
+    # ==Parameters:
+    #  [owner] The login of the repository owner
+    #  [repo] The name of the repository
+    #  [fork_id] The fork item id
+    #  [date_added] The timestamp that the add event took place
+    def get_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+      transaction do
+        ensure_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+      end
+    end
+
+    ##
+    # Retrieve a pull request review comment
+    # ==Parameters:
+    #  [owner] The login of the repository owner
+    #  [repo] The name of the repository
+    #  [fork_id] The fork item id
+    #  [date_added] The timestamp that the add event took place
+    def get_issue_comment(owner, repo, issue_id, comment_id, created_at)
+      transaction do
+        raise "Not implemented"
+        #ensure_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+      end
+    end
+
+
     ##
     # Make sure a commit exists
     #
     def ensure_commit(repo, sha, user, comments = true)
+      ensure_repo(user, repo)
       c = retrieve_commit(repo, sha, user)
+
+      if c.nil?
+        warn "GHTorrent: Commit #{user}/#{repo} -> #{sha} does not exist"
+        return
+      end
+
       stored = store_commit(c, repo, user)
       ensure_parents(c)
-      ensure_commit_comments(user, repo, sha) if comments
+      if not c['commit']['comment_count'].nil? \
+         and c['commit']['comment_count'] > 0
+        ensure_commit_comments(user, repo, sha) if comments
+      end
+      ensure_repo_commit(user, repo, sha)
       stored
     end
 
@@ -162,13 +231,41 @@ module GHTorrent
 
             parents.insert(:commit_id => this[:id],
                            :parent_id => parent[:id])
-            info "Added parent #{parent[:sha]} to commit #{this[:sha]}"
+            info "GHTorrent: Added parent #{parent[:sha]} to commit #{this[:sha]}"
           else
-            info "Parent #{parent[:sha]} for commit #{this[:sha]} exists"
+            debug "GHTorrent: Parent #{parent[:sha]} for commit #{this[:sha]} exists"
           end
         end
     end
 
+    ##
+    # Make sure that a commit has been associated with the provided repo
+    # ==Parameters:
+    #  [user] The user that owns the repo this commit has been submitted to
+    #  [repo] The repo receiving the commit
+    #  [sha] The commit SHA
+    def ensure_repo_commit(user, repo, sha)
+      userid = @db[:users].first(:login => user)[:id]
+      projectid = @db[:projects].first(:owner_id => userid,
+                                     :name => repo)[:id]
+      commitid = @db[:commits].first(:sha => sha)[:id]
+
+      exists = @db[:project_commits].first(:project_id => projectid,
+                                           :commit_id => commitid)
+      if exists.nil?
+        @db[:project_commits].insert(
+            :project_id => projectid,
+            :commit_id => commitid
+        )
+        info "GHTorrent: Added commit #{user}/#{repo} -> #{sha}"
+        @db[:project_commits].first(:project_id => projectid,
+                                    :commit_id => commitid)
+      else
+        debug "GHTorrent: Commit #{user}/#{repo} -> #{sha} exists"
+        exists
+      end
+    end
+
     ##
     # Add (or update) an entry for a commit author. This method uses information
     # in the JSON object returned by Github to add (or update) a user in the
@@ -193,15 +290,14 @@ module GHTorrent
       login = githubuser['login'] unless githubuser.nil?
 
       if login.nil?
-        ensure_user("#{name}<#{email}>", true, false)
+        ensure_user("#{name}<#{email}>", false, false)
       else
         dbuser = users.first(:login => login)
         byemail = users.first(:email => email)
         if dbuser.nil?
           # We do not have the user in the database yet. Add him
-          added = ensure_user(login, true, false)
+          added = ensure_user(login, false, false)
           if byemail.nil?
-            #
             users.filter(:login => login).update(:name => name) if added[:name].nil?
             users.filter(:login => login).update(:email => email) if added[:email].nil?
           else
@@ -215,8 +311,6 @@ module GHTorrent
                 :login => login,
                 :company => added['company'],
                 :location => added['location'],
-                :hireable => added['hireable'],
-                :bio => added['bio'],
                 :created_at => added['created_at']
             )
           end
@@ -259,7 +353,6 @@ module GHTorrent
       return u
     end
 
-
     ##
     # Ensure that a user exists, or fetch its latest state from Github
     # ==Parameters:
@@ -275,6 +368,12 @@ module GHTorrent
 
       if usr.nil?
         u = retrieve_user_byusername(user)
+
+        if u.nil?
+          warn "GHTorrent: User #{user} does not exist"
+          return
+        end
+
         email = unless u['email'].nil?
                   if u['email'].strip == "" then
                     nil
@@ -283,49 +382,16 @@ module GHTorrent
                   end
                 end
 
-        if not email.nil?
-          # Check whether a user has been added by email before
-          byemail = users.first(:email => email)
-          unless byemail.nil?
-            users.filter(:email => email).update(:login => u['login'],
-                           :name => u['name'],
-                           :company => u['company'],
-                           :hireable => boolean(u['hirable']),
-                           :bio => u['bio'],
-                           :location => u['location'],
-                           :type => user_type(u['type']),
-                           :created_at => date(u['created_at']),
-                           :ext_ref_id => u[@ext_uniq]
-            )
-            info "GHTorrent: Updating user #{user} (email #{email})"
-          else
-            users.insert(:login => u['login'],
-                         :name => u['name'],
-                         :company => u['company'],
-                         :email => email,
-                         :hireable => boolean(u['hirable']),
-                         :bio => u['bio'],
-                         :location => u['location'],
-                         :type => user_type(u['type']),
-                         :created_at => date(u['created_at']),
-                         :ext_ref_id => u[@ext_uniq])
-
-            info "GHTorrent: New user #{user}"
-          end
-        else
-          users.insert(:login => u['login'],
-                       :name => u['name'],
-                       :company => u['company'],
-                       :email => email,
-                       :hireable => boolean(u['hirable']),
-                       :bio => u['bio'],
-                       :location => u['location'],
-                       :type => user_type(u['type']),
-                       :created_at => date(u['created_at']),
-                       :ext_ref_id => u[@ext_uniq])
+        users.insert(:login => u['login'],
+                     :name => u['name'],
+                     :company => u['company'],
+                     :email => email,
+                     :location => u['location'],
+                     :type => user_type(u['type']),
+                     :created_at => date(u['created_at']),
+                     :ext_ref_id => u[@ext_uniq])
 
-          info "GHTorrent: New user #{user}"
-        end
+        info "GHTorrent: New user #{user}"
         users.first(:login => user)
       else
         debug "GHTorrent: User #{user} exists"
@@ -340,37 +406,62 @@ module GHTorrent
     #
     # ==Parameters:
     # [user]  The user login to find followers by
-    def ensure_user_followers(user, date_added = nil)
+    def ensure_user_followers(followed, date_added = nil)
+      curuser = ensure_user(followed, false, false)
+      time = curuser[:created_at]
+      followers = @db.from(:followers, :users).\
+          where(:followers__follower_id => :users__id).
+          where(:followers__user_id => curuser[:id]).select(:login).all
+
+      retrieve_user_followers(followed).reduce([]) do |acc, x|
+        if followers.find {|y| y[:login] == x['login']}.nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x| ensure_user_follower(followed, x['login'], time) }
+    end
+
+    ##
+    # Make sure that a user follows another one
+    def ensure_user_follower(followed, follower, date_added)
+      follower_user = ensure_user(follower, false, false)
+      followed_user = ensure_user(followed, false, false)
+
+      if followed_user.nil? or follower_user.nil?
+        warn "Could not add follower #{follower} to #{followed}"
+        return
+      end
+
       followers = @db[:followers]
-      userid = @db[:users].first(:login => user)[:id]
+      followed_id = follower_user[:id]
+      follower_id = followed_user[:id]
 
-      retrieved = retrieve_user_followers(user)
-      retrieved.each { |f|
-        follower = f['login']
-        ensure_user(user, false, false)
-        ensure_user(follower, false, false)
+      follower_exists = followers.first(:user_id => followed_id,
+                                        :follower_id => follower_id)
 
-        followerid = @db[:users].first(:login => follower)[:id]
+      if follower_exists.nil?
+        added = if date_added.nil? then Time.now else date_added end
+        retrieved = retrieve_user_follower(followed, follower)
 
+        if retrieved.nil?
+          warn "Follower #{follower} does not exist for user #{followed}"
+          return
+        end
 
-        if followers.first(:user_id => userid, :follower_id => followerid).nil?
-          added = if date_added.nil? then Time.now else date_added end
-          followers.insert(:user_id => userid,
-                           :follower_id => followerid,
-                           :created_at => added,
-                           :ext_ref_id => f[@ext_uniq]
-          )
-          info "GHTorrent: User #{follower} follows #{user}"
-        else
-          unless date_added.nil?
-            followers.filter(:user_id => userid,
-                             :follower_id => followerid).\
-                      update(:created_at => date(date_added))
-            info "GHTorrent: Updated follower #{follower} -> #{user}"
-          end
-          debug "GHTorrent: User #{follower} already follows #{user}"
+        followers.insert(:user_id => followed_id,
+                         :follower_id => follower_id,
+                         :created_at => added,
+                         :ext_ref_id => retrieved[@ext_uniq])
+        info "GHTorrent: User #{follower} follows #{followed}"
+      else
+        unless date_added.nil?
+          followers.filter(:user_id => followed_id,
+                           :follower_id => follower_id)\
+                    .update(:created_at => date(date_added))
+          debug "GHTorrent: Updating follower #{followed} -> #{follower}"
         end
-      }
+      end
     end
 
     ##
@@ -379,8 +470,7 @@ module GHTorrent
     #
     # ==Parameters:
     # [email]  The email to lookup the user by
-    # [email]  The user's name
-    # [followers]  If true, the user's followers will be retrieved
+    # [name]  The user's name
     # == Returns:
     # If the user can be retrieved, it is returned as a Hash. Otherwise,
     # the result is nil
@@ -392,27 +482,27 @@ module GHTorrent
 
         u = retrieve_user_byemail(email, name)
 
-        if u.nil? or u['user'].nil? or u['user']['login'].nil?
+        if u.nil? or u['login'].nil?
           debug "GHTorrent: Cannot find #{email} through search API query"
+          login = (0...8).map { 65.+(rand(25)).chr }.join
           users.insert(:email => email,
                        :name => name,
-                       :login => (0...8).map { 65.+(rand(25)).chr }.join,
+                       :login => login,
                        :created_at => Time.now,
                        :ext_ref_id => ""
           )
-          users.first(:email => email)
+          info "GHTorrent: Added fake user #{login} -> #{email}"
+          users.first(:login => login)
         else
-          users.insert(:login => u['user']['login'],
-                       :name => u['user']['name'],
-                       :company => u['user']['company'],
-                       :email => u['user']['email'],
-                       :hireable => nil,
-                       :bio => nil,
-                       :location => u['user']['location'],
-                       :created_at => date(u['user']['created_at']),
+          users.insert(:login => u['login'],
+                       :name => u['name'],
+                       :company => u['company'],
+                       :email => u['email'],
+                       :location => u['location'],
+                       :created_at => date(u['created_at']),
                        :ext_ref_id => u[@ext_uniq])
-          debug "GHTorrent: Found #{email} through search API query"
-          users.first(:email => email)
+          info "GHTorrent: Found #{email} through search API query"
+          users.first(:login => u['login'])
         end
       else
         debug "GHTorrent: User with email #{email} exists"
@@ -430,15 +520,21 @@ module GHTorrent
     # == Returns:
     #  If the repo can be retrieved, it is returned as a Hash. Otherwise,
     #  the result is nil
-    def ensure_repo(user, repo)
+    def ensure_repo(user, repo, commits = true, project_members = true, watchers = true)
 
-      ensure_user(user, true, true)
+      ensure_user(user, false, false)
       repos = @db[:projects]
       curuser = @db[:users].first(:login => user)
       currepo = repos.first(:owner_id => curuser[:id], :name => repo)
 
       if currepo.nil?
         r = retrieve_repo(user, repo)
+
+        if r.nil?
+          warn "Repo #{user}/#{repo} does not exist"
+          return
+        end
+
         repos.insert(:url => r['url'],
                      :owner_id => @db[:users].filter(:login => user).first[:id],
                      :name => r['name'],
@@ -448,9 +544,9 @@ module GHTorrent
                      :ext_ref_id => r[@ext_uniq])
 
         info "GHTorrent: New repo #{repo}"
-        ensure_commits(user, repo)
-        ensure_project_members(user, repo)
-        ensure_watchers(user, repo)
+        ensure_commits(user, repo) if commits
+        ensure_project_members(user, repo) if project_members
+        ensure_watchers(user, repo) if watchers
         repos.first(:owner_id => curuser[:id], :name => repo)
       else
         debug "GHTorrent: Repo #{repo} exists"
@@ -461,27 +557,32 @@ module GHTorrent
     ##
     # Make sure that a project has all the registered members defined
     def ensure_project_members(user, repo)
-      curuser = @db[:users].first(:login => user)
-      currepo = @db[:projects].first(:owner_id => curuser[:id], :name => repo)
-      project_members = @db[:project_members].filter(:user_id => curuser[:id],
-                                                   :repo_id => currepo[:id])
+      currepo = ensure_repo(user, repo, true, false, true)
+      time = currepo[:created_at]
+
+      project_members = @db.from(:project_members, :users).\
+          where(:project_members__user_id => :users__id).\
+          where(:project_members__repo_id => currepo[:id]).select(:login).all
 
       retrieve_repo_collaborators(user, repo).reduce([]) do |acc, x|
-        if project_members.find { |y| y[:login] == x['login'] }.nil?
+        if project_members.find {|y| y[:login] == x['login']}.nil?
           acc << x
         else
           acc
         end
-      end.map { |x| ensure_project_member(user, repo, x['login'], nil) }
+      end.map { |x| ensure_project_member(user, repo, x['login'], time) }
     end
 
     ##
     # Make sure that a project member exists in a project
     def ensure_project_member(owner, repo, new_member, date_added)
       pr_members = @db[:project_members]
+      project = ensure_repo(owner, repo, true, false, true)
       new_user = ensure_user(new_member, false, false)
-      owner_id = @db[:users].first(:login => owner)[:id]
-      project = @db[:projects].first(:owner_id => owner_id, :name => repo)
+
+      if project.nil? or new_user.nil?
+        return
+      end
 
       memb_exist = pr_members.first(:user_id => new_user[:id],
                                     :repo_id => project[:id])
@@ -489,6 +590,12 @@ module GHTorrent
       if memb_exist.nil?
         added = if date_added.nil? then Time.now else date_added end
         retrieved = retrieve_repo_collaborator(owner, repo, new_member)
+
+        if retrieved.nil?
+          warn "Project member #{new_member} does not exist in #{owner}/#{repo}"
+          return
+        end
+
         pr_members.insert(
             :user_id => new_user[:id],
             :repo_id => project[:id],
@@ -513,7 +620,6 @@ module GHTorrent
     # [user]  The login name of the user to check the organizations for
     #
     def ensure_orgs(user)
-      usr = @db[:users].first(:login => user)
       retrieve_orgs(user).map{|o| ensure_participation(user, o['login'])}
     end
 
@@ -525,8 +631,8 @@ module GHTorrent
     # [org]  The login name of the organization to check whether the user
     #        belongs in
     #
-    def ensure_participation(user, organization)
-      org = ensure_org(organization)
+    def ensure_participation(user, organization, members = true)
+      org = ensure_org(organization, members)
       usr = ensure_user(user, false, false)
 
       org_members = @db[:organization_members]
@@ -550,14 +656,21 @@ module GHTorrent
     # ==Parameters:
     # [organization]  The login name of the organization
     #
-    def ensure_org(organization)
-      org = @db[:users].find(:login => organization, :type => 'org')
+    def ensure_org(organization, members)
+      org = @db[:users].first(:login => organization, :type => 'org')
 
       if org.nil?
-        ensure_user(org, false, false)
+        org = ensure_user(organization, false, false)
+        if members
+        retrieve_org_members(organization).map { |x|
+          ensure_participation(ensure_user(x['login'], false, false)[:login],
+                               organization, false)
+        }
+        end
+        org
       else
         debug "GHTorrent: Organization #{organization} exists"
-        org.first
+        org
       end
     end
 
@@ -572,7 +685,6 @@ module GHTorrent
       commit_id = @db[:commits].first(:sha => sha)[:id]
       stored_comments = @db[:commit_comments].filter(:commit_id => commit_id)
       commit_comments = retrieve_commit_comments(user, repo, sha)
-      #user_id = @db[:users].first(:login => user)[:id]
 
       not_saved = commit_comments.reduce([]) do |acc, x|
         if stored_comments.find{|y| y[:comment_id] == x['id']}.nil?
@@ -600,16 +712,16 @@ module GHTorrent
         retrieved = retrieve_commit_comment(user, repo, id)
 
         if retrieved.nil?
-          debug "GHTorrent: Commit comment #{id} deleted"
+          warn "GHTorrent: Commit comment #{id} deleted"
           return
         end
 
-        commit = ensure_commit(repo, retrieved['commit_id'], user, comments = false)
+        commit = ensure_commit(repo, retrieved['commit_id'], user, false)
         user = ensure_user(user, false, false)
         @db[:commit_comments].insert(
             :commit_id => commit[:id],
             :user_id => user[:id],
-            :body => retrieved['body'],
+            :body => retrieved['body'][0..255],
             :line => retrieved['line'],
             :position => retrieved['position'],
             :comment_id => retrieved['id'],
@@ -617,45 +729,67 @@ module GHTorrent
             :created_at => date(retrieved['created_at'])
         )
         info "GHTorrent: Added commit comment #{commit[:sha]} -> #{user[:login]}"
-        @db[:commit_comments].first(:comment_id => id)
       else
+        unless created_at.nil?
+          @db[:commit_comments].filter(:comment_id => id)\
+                               .update(:created_at => date(created_at))
+          info "GHTorrent: Updating comment #{user}/#{repo} -> #{id}"
+        end
         info "GHTorrent: Commit comment #{id} exists"
-        stored_comment
       end
+      @db[:commit_comments].first(:comment_id => id)
     end
 
     ##
     # Make sure that
     def ensure_watchers(owner, repo)
-      curuser = @db[:users].first(:login => owner)
-      currepo = @db[:projects].first(:owner_id => curuser[:id],
-                                     :name => repo)
-      watchers = @db[:watchers].filter(:user_id => curuser[:id],
-                                       :repo_id => currepo[:id])
+      currepo = ensure_repo(owner, repo, true, true, false)
+      time = currepo[:created_at]
+
+      if currepo.nil?
+        warn "Could not retrieve watchers for #{owner}/#{repo}"
+        return
+      end
+
+      watchers = @db.from(:watchers, :users).\
+          where(:watchers__user_id => :users__id).\
+          where(:watchers__repo_id => currepo[:id]).select(:login).all
 
       retrieve_watchers(owner, repo).reduce([]) do |acc, x|
-        if watchers.find { |y| y[:login] == x['login'] }.nil?
+        if watchers.find { |y|
+          y[:login] == x['login']
+        }.nil?
           acc << x
         else
           acc
         end
-      end.map { |x| ensure_watcher(owner, repo, x['login']) }
+      end.map { |x| ensure_watcher(owner, repo, x['login'], time) }
     end
 
     ##
     # Make sure that a project member exists in a project
     def ensure_watcher(owner, repo, watcher, date_added = nil)
-      watchers = @db[:watchers]
+      project = ensure_repo(owner, repo, false, false, false)
       new_watcher = ensure_user(watcher, false, false)
-      owner_id = @db[:users].first(:login => owner)[:id]
-      project = @db[:projects].first(:owner_id => owner_id, :name => repo)
 
+      if new_watcher.nil? or project.nil?
+        warn "GHTorrent: Watcher #{watcher} does not exist"
+        return
+      end
+
+      watchers = @db[:watchers]
       memb_exist = watchers.first(:user_id => new_watcher[:id],
-                                    :repo_id => project[:id])
+                                  :repo_id => project[:id])
 
       if memb_exist.nil?
         added = if date_added.nil? then Time.now else date_added end
         retrieved = retrieve_watcher(owner, repo, watcher)
+
+        if retrieved.nil?
+          warn "Watcher #{watcher} no longer watches #{owner}/#{repo}"
+          return
+        end
+
         watchers.insert(
             :user_id => new_watcher[:id],
             :repo_id => project[:id],
@@ -673,6 +807,313 @@ module GHTorrent
       end
     end
 
+    ##
+    # Process all pull requests
+    def ensure_pull_requests(owner, repo)
+      currepo = ensure_repo(owner, repo, false, false, false)
+      if currepo.nil?
+        warn "Could not retrieve pull requests from #{owner}/#{repo}"
+        return
+      end
+
+      pull_reqs = @db[:pull_requests].filter(:base_repo_id => currepo[:id])
+
+      retrieve_pull_requests(owner, repo).reduce([]) do |acc, x|
+        if pull_reqs.find { |y| y[:pullreq_id] == x['number'] }.nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x| ensure_pull_request(owner, repo, x['number']) }
+    end
+
+    ##
+    # Process a pull request
+    def ensure_pull_request(owner, repo, pullreq_id,
+                            comments = true, commits = true,
+                            state = nil, created_at = nil)
+      pulls_reqs = @db[:pull_requests]
+      pull_req_history = @db[:pull_request_history]
+
+      project = ensure_repo(owner, repo, false, false, false)
+
+      if project.nil?
+        return
+      end
+
+      # Adds a pull request history event
+      add_history = Proc.new do |id, ts, unq, act|
+
+        entry = pull_req_history.first(:pull_request_id => id,
+                                       :ext_ref_id => unq, :action => act)
+        if entry.nil?
+          pull_req_history.insert(:pull_request_id => id, :created_at => ts,
+                                  :ext_ref_id => unq, :action => act)
+          info "GHTorrent: New pull request (#{id}) history entry (#{act})"
+        else
+          pull_req_history.filter(:pull_request_id => id, :ext_ref_id => unq,
+                                :action => act).update(:created_at => ts)
+          info "GHTorrent: Updating pull request (#{id}) history entry (#{act}) timestamp #{ts}"
+        end
+      end
+
+      # Checks whether a pull request concerns two branches of the same
+      # repository
+      is_intra_branch = Proc.new do |req|
+        req['head']['repo'].nil?
+      end
+
+      # Produces a log message
+      log_msg = Proc.new do |req|
+        head = if is_intra_branch.call(req)
+                 req['base']['repo']['full_name']
+               else
+                 req['head']['repo']['full_name']
+               end
+
+        <<-eos.gsub(/\s+/, " ").strip
+            GHTorrent: Pull request #{pullreq_id}
+            #{head} -> #{req['base']['repo']['full_name']}
+        eos
+      end
+
+      retrieved = retrieve_pull_request(owner, repo, pullreq_id)
+
+      if retrieved.nil?
+        warn "GHTorrent: Cannot retrieve pull request (#{owner}/#{repo} #{pullreq_id})"
+        return
+      end
+
+      base_repo = ensure_repo(retrieved['base']['repo']['owner']['login'],
+                              retrieved['base']['repo']['name'],
+                              false, false, false)
+
+      base_commit = ensure_commit(retrieved['base']['repo']['name'],
+                                  retrieved['base']['sha'],
+                                  retrieved['base']['repo']['owner']['login']
+      )
+
+      if is_intra_branch.call(retrieved)
+        head_repo = base_repo
+        head_commit =
+        warn "GHTorrent: Pull request is intra branch"
+      else
+
+        head_repo = ensure_repo(retrieved['head']['repo']['owner']['login'],
+                                retrieved['head']['repo']['name'],
+                                false, false, false)
+
+        head_commit = ensure_commit(retrieved['head']['repo']['name'],
+                                    retrieved['head']['sha'],
+                                    retrieved['head']['repo']['owner']['login'])
+      end
+
+      pull_req_user = ensure_user(retrieved['user']['login'], false, false)
+
+      merged = if retrieved['merged_at'].nil? then false else true end
+      closed = if retrieved['closed_at'].nil? then false else true end
+
+      pull_req = pulls_reqs.first(:base_repo_id => project[:id],
+                                  :pullreq_id => pullreq_id)
+      if pull_req.nil?
+        pulls_reqs.insert(
+            :head_repo_id => if not head_repo.nil? then head_repo[:id] end,
+            :base_repo_id => base_repo[:id],
+            :head_commit_id => if not head_commit.nil? then head_commit[:id] end,
+            :base_commit_id => base_commit[:id],
+            :user_id => pull_req_user[:id],
+            :pullreq_id => pullreq_id,
+            :intra_branch => is_intra_branch.call(retrieved)
+        )
+
+        info log_msg.call(retrieved)
+      else
+        debug log_msg.call(retrieved) + " exists"
+      end
+
+      pull_req = pulls_reqs.first(:base_repo_id => project[:id],
+                                  :pullreq_id => pullreq_id)
+
+      add_history.call(pull_req[:id], date(retrieved['created_at']),
+                       retrieved[@ext_uniq], 'opened')
+      add_history.call(pull_req[:id], date(retrieved['merged_at']),
+                       retrieved[@ext_uniq], 'merged') if merged
+      add_history.call(pull_req[:id], date(retrieved['closed_at']),
+                       retrieved[@ext_uniq], 'closed') if closed
+      add_history.call(pull_req[:id], date(created_at), retrieved[@ext_uniq],
+                       state) unless state.nil?
+
+      ensure_pull_request_commits(owner, repo, pullreq_id) if commits
+      ensure_pullreq_comments(owner, repo, pullreq_id, created_at) if comments
+
+      pulls_reqs.first(:base_repo_id => project[:id],
+                       :pullreq_id => pullreq_id)
+    end
+
+    def ensure_pullreq_comments(owner, repo, pullreq_id, created_at)
+      currepo = ensure_repo(owner, repo, true, true, false)
+      time = if created_at.nil? then currepo[:created_at] else Time.now() end
+
+      if currepo.nil?
+        warn "Could not repository #{owner}/#{repo}"
+        return
+      end
+
+      pull_req = ensure_pull_request(owner, repo, pullreq_id, false, true)
+
+      if pull_req.nil?
+        warn "Could not retrieve pull req #{owner}/#{repo} -> #{pullreq_id}"
+        return
+      end
+
+      retrieve_pull_req_comments(owner, repo, pullreq_id).reduce([]) do |acc, x|
+
+        if @db[:pull_request_comments].first(:pullreq_id => pull_req[:id],
+                                             :comment_id => x['id']).nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x|
+        ensure_pullreq_comment(owner, repo, pullreq_id, x['id'], time)
+      }
+    end
+
+    def ensure_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+      pull_req = ensure_pull_request(owner, repo, pullreq_id, false, true)
+
+      if pull_req.nil?
+        warn "Could not retrieve pull req #{owner}/#{repo} -> #{pullreq_id}"
+        return
+      end
+
+      exists = @db[:pull_request_comments].first(:pull_request_id => pull_req[:id],
+                                                 :comment_id => comment_id)
+
+      if exists.nil?
+        retrieved = retrieve_pull_req_comment(owner, repo, pullreq_id, comment_id)
+
+        if retrieved.nil?
+          warn "Could not retrieve comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id}"
+          return
+        end
+
+        commenter = ensure_user(retrieved['user']['login'], false, false)
+
+        if commenter.nil?
+          warn "Could not retrieve commenter #{retrieved['user']['login']}" +
+               "for pullreq comment #{owner}/#{repo} -> #{pullreq_id}(#{comment_id}) "
+        end
+
+        commit = ensure_commit(repo, retrieved['original_commit_id'],owner)
+
+        @db[:pull_request_comments].insert(
+            :pull_request_id => pull_req[:id],
+            :user_id => commenter[:id],
+            :comment_id => comment_id,
+            :position => retrieved['original_position'],
+            :body => retrieved['body'][0..254],
+            :commit_id => (commit[:id] unless commit.nil?),
+            :created_at => retrieved['created_at'],
+            :ext_ref_id => retrieved[@ext_uniq]
+        )
+        debug "GHTorrent: Adding comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id}"
+      else
+        debug "GHTorrent: Updating comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id}"
+      end
+    end
+
+    def ensure_pull_request_commits(owner, repo, pullreq_id)
+      retrieve_pull_req_commits(owner, repo, pullreq_id).map {|c|
+        ensure_commit(repo, c['sha'], owner, true)
+      }.map { |c|
+        pullreq = ensure_pull_request(owner, repo, pullreq_id, false, false)
+        exists = @db[:pull_request_commits].first(:pull_request_id => pullreq[:id],
+                                                  :commit_id => c[:id])
+        if exists.nil?
+          @db[:pull_request_commits].insert(:pull_request_id => pullreq[:id],
+                                            :commit_id => c[:id])
+
+          info "GHTorrent: Added commit #{c[:sha]} to pullreq #{owner}/#{repo} -> #{pullreq_id}"
+        else
+          debug "GHTorrent: Commit #{c[:sha]} exists in pullreq #{owner}/#{repo} -> #{pullreq_id}"
+          exists
+        end
+      }
+    end
+
+    ##
+    # Get all forks for a project.
+    #
+    # ==Parameters:
+    # [owner]  The user to which the project belongs
+    # [repo]  The repository/project to find forks for
+    def ensure_forks(owner, repo)
+      currepo = ensure_repo(owner, repo, false, false, false)
+      time = currepo[:created_at]
+
+      if currepo.nil?
+        warn "Could not retrieve forks for #{owner}/#{repo}"
+        return
+      end
+
+      existing_forks = @db.from(:forks, :projects).\
+          where(:forks__forked_project_id => :projects__id). \
+          where(:forks__forked_from_id => currepo[:id]).select(:name, :login).all
+
+      retrieve_forks(owner, repo).reduce([]) do |acc, x|
+        if existing_forks.find {|y|
+          y[:login] == x['owner']['login'] && y[:name] == x['name']
+        }.nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x| ensure_fork(owner, repo, x['id'], time) }
+    end
+
+    ##
+    # Make sure that a fork is retrieved for a project
+    def ensure_fork(owner, repo, fork_id, date_added = nil)
+
+      forks = @db[:forks]
+      forked = ensure_repo(owner, repo, false, false, false)
+      fork_exists = forks.first(:fork_id => fork_id)
+
+      if fork_exists.nil?
+        added = if date_added.nil? then Time.now else date_added end
+        retrieved = retrieve_fork(owner, repo, fork_id)
+
+        if retrieved.nil?
+          warn "GHTorrent: Fork #{fork_id} does not exist for #{owner}/#{repo}"
+          return
+        end
+
+        forked_repo_owner = retrieved['full_name'].split(/\//)[0]
+        forked_repo_name = retrieved['full_name'].split(/\//)[1]
+
+        fork = ensure_repo(forked_repo_owner, forked_repo_name)
+
+        if forked.nil? or fork.nil?
+          warn "Could not add fork #{fork_id}"
+          return
+        end
+
+        forks.insert(:forked_project_id => fork[:id],
+                     :forked_from_id => forked[:id],
+                     :fork_id => fork_id,
+                     :created_at => added,
+                     :ext_ref_id => retrieved[@ext_uniq])
+        info "GHTorrent: Added #{forked_repo_owner}/#{forked_repo_name} as fork of  #{owner}/#{repo}"
+      else
+        unless date_added.nil?
+          forks.filter(:fork_id => fork_id)\
+               .update(:created_at => date(date_added))
+          debug "GHTorrent: Updating fork #{owner}/#{repo} (#{fork_id})"
+        end
+      end
+    end
+
     private
 
     # Store a commit contained in a hash. First check whether the commit exists.
@@ -684,21 +1125,24 @@ module GHTorrent
         author = commit_user(c['author'], c['commit']['author'])
         commiter = commit_user(c['committer'], c['commit']['committer'])
 
-        userid = @db[:users].filter(:login => user).first[:id]
-        repoid = @db[:projects].filter(:owner_id => userid,
-                                       :name => repo).first[:id]
+        repository = ensure_repo(user, repo, false, false, false)
+
+        if repository.nil?
+          warn "Could not store commit #{user}/#{repo} #{c['sha']}"
+          return
+        end
 
         commits.insert(:sha => c['sha'],
                        :author_id => author[:id],
                        :committer_id => commiter[:id],
-                       :project_id => repoid,
+                       :project_id => repository[:id],
                        :created_at => date(c['commit']['author']['date']),
                        :ext_ref_id => c[@ext_uniq]
         )
+        debug "GHTorrent: New commit #{user}/#{repo} -> #{c['sha']} "
         commits.first(:sha => c['sha'])
-        debug "GHTorrent: New commit #{repo} -> #{c['sha']} "
       else
-        debug "GHTorrent: Commit #{repo} -> #{c['sha']} exists"
+        debug "GHTorrent: Commit #{user}/#{repo} -> #{c['sha']} exists"
         commit
       end
     end
@@ -706,12 +1150,28 @@ module GHTorrent
     # Run a block in a DB transaction. Exceptions trigger transaction rollback
     # and are rethrown.
     def transaction(&block)
+      @db ||= get_db
+      @persister ||= persister
+
       start_time = Time.now
-      @db.transaction(:rollback => :reraise, :isolation => :committed) do
-        yield block
+      begin
+        @db.transaction(:rollback => :reraise, :isolation => :committed) do
+          yield block
+        end
+        total = Time.now.to_ms - start_time.to_ms
+        debug "GHTorrent: Transaction committed (#{total} ms)"
+      rescue Exception => e
+        total = Time.now.to_ms - start_time.to_ms
+        warn "GHTorrent: Transaction failed (#{total} ms)"
+        raise e
+      ensure
+        @db.disconnect
+        @persister.close
+
+        @db = nil
+        @persister = nil
+        GC.start
       end
-      total = Time.now.to_ms - start_time.to_ms
-      debug "GHTorrent: Transaction committed (#{total} ms)"
     end
 
     ##
@@ -742,15 +1202,6 @@ module GHTorrent
       email =~ /^[a-zA-Z][\w\.-]*[a-zA-Z0-9]@[a-zA-Z0-9][\w\.-]*[a-zA-Z0-9]\.[a-zA-Z][a-zA-Z\.]*[a-zA-Z]$/
     end
   end
-  # Base exception for all GHTorrent exceptions
-  class GHTorrentException < Exception
-  end
-end
-
-class Time
-  def to_ms
-    (self.to_f * 1000.0).to_i
-  end
 end
 
 # vim: set sta sts=2 shiftwidth=2 sw=2 et ai :