ghtorrent 0.5 → 0.6

data/lib/ghtorrent/commands/ght_retrieve_repo.rb

@@ -0,0 +1,118 @@
+require 'rubygems'
+
+require 'ghtorrent/ghtorrent'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+require 'ghtorrent/retriever'
+
+class GHTRetrieveRepo < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Retriever
+  include GHTorrent::Persister
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+An efficient way to get all data for a single repo
+
+#{command_name} [options] owner repo
+
+    BANNER
+  end
+
+  def validate
+    super
+    Trollop::die "Two arguments are required" unless args[0] && !args[0].empty?
+  end
+
+  def logger
+    ght.logger
+  end
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
+
+  def ext_uniq
+    @ext_uniq ||= config(:uniq_id)
+    @ext_uniq
+  end
+
+  def ght
+    @ght ||= TransactedGHTorrent.new(settings)
+    @ght
+  end
+
+  def go
+    user_entry = ght.transaction{ght.ensure_user(ARGV[0], false, false)}
+
+    if user_entry.nil?
+      Trollop::die "Cannot find user #{owner}"
+    end
+
+    user = user_entry[:login]
+
+    repo_entry = ght.transaction{ght.ensure_repo(ARGV[0], ARGV[1], false, false, false)}
+
+    if repo_entry.nil?
+      Trollop::die "Cannot find repository #{owner}/#{ARGV[1]}"
+    end
+
+    repo = repo_entry[:name]
+
+    %w(ensure_commits ensure_forks ensure_pull_requests
+       ensure_issues ensure_project_members ensure_watchers).each {|x|
+      begin
+        ght.send(x, user, repo)
+      rescue Exception => e
+        puts STDERR, e.message
+        puts STDERR, e.backtrace
+      end
+    }
+  end
+end
+
+# A version of the GHTorrent class that creates a transaction per processed
+# item
+class TransactedGHTorrent < GHTorrent::Mirror
+
+  def ensure_commit(repo, sha, user, comments = true)
+    transaction do
+      super(repo, sha, user, comments)
+    end
+  end
+
+  def ensure_fork(owner, repo, fork_id, date_added = nil)
+    transaction do
+      super(owner, repo, fork_id, date_added)
+    end
+  end
+
+  def ensure_pull_request(owner, repo, pullreq_id,
+      comments = true, commits = true,
+      state = nil, created_at = nil)
+    transaction do
+      super(owner, repo, pullreq_id, comments, commits, state, created_at)
+    end
+  end
+
+  def ensure_issue(owner, repo, issue_id, events = true, comments = true)
+    transaction do
+      super(owner, repo, issue_id, events, comments)
+    end
+  end
+
+  def ensure_project_member(owner, repo, new_member, date_added)
+    transaction do
+      super(owner, repo, new_member, date_added)
+    end
+  end
+
+  def ensure_watcher(owner, repo, watcher, date_added = nil)
+    transaction do
+      super(owner, repo, watcher, date_added)
+    end
+  end
+end

data/lib/ghtorrent/commands/ght_rm_dupl.rb

@@ -0,0 +1,132 @@
+require 'rubygems'
+require 'mongo'
+
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+require 'ghtorrent/persister'
+
+class GHRMDupl < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Persister
+
+  def col_info()
+    {
+        :commits => {
+            :unq => "sha",
+            :col => persister.get_underlying_connection.collection(:commits.to_s),
+        },
+        :events => {
+            :unq => "id",
+            :col => persister.get_underlying_connection.collection(:events.to_s),
+        }
+    }
+  end
+
+  def persister
+    @persister ||= connect(:mongo, @settings)
+    @persister
+  end
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Removes duplicate entries from collections (currently, commits and events)
+
+#{command_name} [options] collection
+
+#{command_name} options:
+    BANNER
+
+    options.opt :earliest, 'Seconds since epoch of earliest item to load',
+                :short => 'e', :default => 0, :type => :int
+    options.opt :snapshot, 'Perform clean up every x records',
+                :short => 's', :default => -1, :type => :int
+  end
+
+  def validate
+    super
+    Trollop::die "no collection specified" unless args[0] && !args[0].empty?
+  end
+
+  # Print MongoDB remove statements that
+  # remove all but one entries for each commit.
+  def remove_duplicates(data, col)
+    removed = 0
+    data.select { |k, v| v.size > 1 }.each do |k, v|
+      v.slice(0..(v.size - 2)).map do |x|
+        removed += 1 if delete_by_id col, x
+      end
+    end
+    removed
+  end
+
+  def delete_by_id(col, id)
+    begin
+      col.remove({'_id' => id})
+      true
+    rescue Mongo::OperationFailure
+      puts "Cannot remove record with id #{id} from #{col.name}"
+      false
+    end
+  end
+
+  def go
+    collection = case ARGV[0]
+                   when "commits" then
+                     :commits
+                   when "events" then
+                     :events
+                   else
+                     puts "Not a known collection name: #{ARGV[0]}\n"
+                 end
+
+    from = {'_id' => {'$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest]))}}
+
+    snapshot = options[:snapshot]
+
+    puts "Deleting duplicates from collection #{collection}"
+    puts "Deleting duplicates after #{Time.at(options[:earliest])}"
+    puts "Perform clean up every #{snapshot} records"
+
+    # Various counters to report stats
+    processed = total_processed = removed = 0
+
+    data = Hash.new
+
+    # The following code needs to save intermediate results to cope
+    # with large datasets
+    col_info[collection][:col].find(from, :fields => col_info[collection][:unq]).each do |r|
+      _id = r["_id"]
+      commit = read_value(r, col_info[collection][:unq])
+
+      # If entries cannot be parsed, remove them
+      if commit.empty?
+        puts "Deleting unknown entry #{_id}"
+        removed += 1 if delete_by_id col_info[collection][:col], _id
+      else
+        data[commit] = [] if data[commit].nil?
+        data[commit] << _id
+      end
+
+      processed += 1
+      total_processed += 1
+
+      print "\rProcessed #{processed} records"
+
+      # Calculate duplicates, save intermediate result
+      if snapshot > 0 and processed > snapshot
+        puts "\nLoaded #{data.size} values, cleaning"
+        removed += remove_duplicates data, col_info[collection][:col]
+        data = Hash.new
+        processed = 0
+      end
+    end
+
+    removed += remove_duplicates data, col_info[collection][:col]
+
+    puts "\nProcessed #{total_processed}, deleted #{removed} duplicates"
+  end
+end
+
+# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/lib/ghtorrent/ghtorrent.rb

@@ -22,7 +22,7 @@ module GHTorrent
       @logger = Logger.new(STDOUT)
     end
 
-    # db related functions
+    # Get a connection to the database
     def get_db
       Sequel.single_threaded = true
       @db = Sequel.connect(config(:sql_url), :encoding => 'utf8')
@@ -151,19 +151,31 @@ module GHTorrent
     end
 
     ##
-    # Retrieve a pull request review comment
+    # Retrieve an issue
     # ==Parameters:
     #  [owner] The login of the repository owner
     #  [repo] The name of the repository
-    #  [fork_id] The fork item id
+    #  [issue_id] The fork item id
+    #  [action] The action that took place for the issue
     #  [date_added] The timestamp that the add event took place
-    def get_issue_comment(owner, repo, issue_id, comment_id, created_at)
+    def get_issue(owner, repo, issue_id, created_at)
       transaction do
-        raise "Not implemented"
-        #ensure_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+        ensure_issue(owner, repo, issue_id, created_at)
       end
     end
 
+    ##
+    # Retrieve a issue comment
+    # ==Parameters:
+    #  [owner] The login of the repository owner
+    #  [repo] The name of the repository
+    #  [issue_id] The fork item id
+    #  [comment_id] The issue comment unique identifier
+    def get_issue_comment(owner, repo, issue_id, comment_id)
+      transaction do
+        ensure_issue_comment(owner, repo, issue_id, comment_id)
+      end
+    end
 
     ##
     # Make sure a commit exists
@@ -188,21 +200,30 @@ module GHTorrent
     end
 
     ##
-    # Get as many commits for a repository as allowed by Github
-    #
+    # Retrieve commits for a repository, starting from +sha+
+    # and going back to 30 * +num_pages+ commit log entries.
     # ==Parameters:
     # [user]  The user to whom the repo belongs.
     # [repo]  The repo to look for commits into.
-    def ensure_commits(user, repo)
+    # [sha]   The first commit to start retrieving from. If nil, then the
+    #         earliest stored commit will be used instead.
+    # [num_pages] The number of commit pages to retrieve
+    def ensure_commits(user, repo, sha = nil,
+                       num_pages = config(:mirror_commit_pages_new_repo))
       userid = @db[:users].filter(:login => user).first[:id]
       repoid = @db[:projects].filter(:owner_id => userid,
                                      :name => repo).first[:id]
 
-      latest = @db[:commits].filter(:project_id => repoid).order(:created_at).last
+      latest = if sha.nil?
+                 @db[:commits].filter(:project_id => repoid).order(:created_at).last
+               else
+                 sha
+               end
+
       commits = if latest.nil?
-                  retrieve_commits(repo, nil, user)
+                  retrieve_commits(repo, "head", user, num_pages)
                 else
-                  retrieve_commits(repo, latest[:sha], user)
+                  retrieve_commits(repo, latest[:sha], user, num_pages)
                 end
 
       commits.map do |c|
@@ -226,6 +247,11 @@ module GHTorrent
             parent = commits.first(:sha => url[7])
           end
 
+          if parent.nil?
+            warn "GHTorrent: Could not retrieve #{url[4]}/#{url[5]} -> #{url[7]}, parent to commit #{this[:sha]}"
+            return
+          end
+
           if parents.first(:commit_id => this[:id],
                            :parent_id => parent[:id]).nil?
 
@@ -297,6 +323,16 @@ module GHTorrent
         if dbuser.nil?
           # We do not have the user in the database yet. Add him
           added = ensure_user(login, false, false)
+
+          # A commit user can be found by email but not
+          # by the user name he used to commit. This probably means that the
+          # user has probably changed his user name. Treat the user's by-email
+          # description as valid.
+          if added.nil? and not byemail.nil?
+            warn "GHTorrent: Found user #{byemail[:login]} with same email #{email} as non existing user #{login}. Assigning user #{login} to #{byemail[:login]}"
+            return users.first(:login => byemail[:login])
+          end
+
           if byemail.nil?
             users.filter(:login => login).update(:name => name) if added[:name].nil?
             users.filter(:login => login).update(:email => email) if added[:email].nil?
@@ -342,7 +378,11 @@ module GHTorrent
           name, email = user.split("<")
           email = email.split(">")[0]
         rescue Exception
-          raise new GHTorrentException("Not a valid email address: #{user}")
+          raise new GHTorrentException.new("Not a valid email address: #{user}")
+        end
+
+        unless is_valid_email(email)
+          warn("GHTorrent: Extracted email(#{email}) not valid for user #{user}")
         end
         u = ensure_user_byemail(email.strip, name.strip)
       else
@@ -522,9 +562,8 @@ module GHTorrent
     #  the result is nil
     def ensure_repo(user, repo, commits = true, project_members = true, watchers = true)
 
-      ensure_user(user, false, false)
       repos = @db[:projects]
-      curuser = @db[:users].first(:login => user)
+      curuser = ensure_user(user, false, false)
       currepo = repos.first(:owner_id => curuser[:id], :name => repo)
 
       if currepo.nil?
@@ -549,7 +588,7 @@ module GHTorrent
         ensure_watchers(user, repo) if watchers
         repos.first(:owner_id => curuser[:id], :name => repo)
       else
-        debug "GHTorrent: Repo #{repo} exists"
+        debug "GHTorrent: Repo #{user}/#{repo} exists"
         currepo
       end
     end
@@ -604,11 +643,12 @@ module GHTorrent
         )
         info "GHTorrent: Added project member #{repo} -> #{new_member}"
       else
+        debug "GHTorrent: Project member #{repo} -> #{new_member} exists"
         unless date_added.nil?
           pr_members.filter(:user_id => new_user[:id],
                             :repo_id => project[:id])\
                     .update(:created_at => date(date_added))
-          info "GHTorrent: Updating  #{repo} -> #{new_member}"
+          info "GHTorrent: Updating project member #{repo} -> #{new_member}"
         end
       end
     end
@@ -796,13 +836,14 @@ module GHTorrent
             :created_at => date(added),
             :ext_ref_id => retrieved[@ext_uniq]
         )
-        info "GHTorrent: Added watcher #{repo} -> #{watcher}"
+        info "GHTorrent: Added watcher #{owner}/#{repo} -> #{watcher}"
       else
+        debug "GHTorrent: Watcher #{owner}/#{repo} -> #{watcher} exists"
         unless date_added.nil?
           watchers.filter(:user_id => new_watcher[:id],
                           :repo_id => project[:id])\
                   .update(:created_at => date(date_added))
-          info "GHTorrent: Updating  #{repo} -> #{watcher}"
+          info "GHTorrent: Updating watcher #{owner}/#{repo} -> #{watcher}"
         end
       end
     end
@@ -816,7 +857,7 @@ module GHTorrent
         return
       end
 
-      pull_reqs = @db[:pull_requests].filter(:base_repo_id => currepo[:id])
+      pull_reqs = @db[:pull_requests].filter(:base_repo_id => currepo[:id]).all
 
       retrieve_pull_requests(owner, repo).reduce([]) do |acc, x|
         if pull_reqs.find { |y| y[:pullreq_id] == x['number'] }.nil?
@@ -833,7 +874,6 @@ module GHTorrent
                             comments = true, commits = true,
                             state = nil, created_at = nil)
       pulls_reqs = @db[:pull_requests]
-      pull_req_history = @db[:pull_request_history]
 
       project = ensure_repo(owner, repo, false, false, false)
 
@@ -842,8 +882,8 @@ module GHTorrent
       end
 
       # Adds a pull request history event
-      add_history = Proc.new do |id, ts, unq, act|
-
+      def add_history(id, ts, unq, act)
+        pull_req_history = @db[:pull_request_history]
         entry = pull_req_history.first(:pull_request_id => id,
                                        :ext_ref_id => unq, :action => act)
         if entry.nil?
@@ -859,20 +899,33 @@ module GHTorrent
 
       # Checks whether a pull request concerns two branches of the same
       # repository
-      is_intra_branch = Proc.new do |req|
-        req['head']['repo'].nil?
+      def is_intra_branch(req)
+        return false unless has_head_repo(req)
+
+        if req['head']['repo']['owner']['login'] ==
+            req['base']['repo']['owner']['login'] and
+            req['head']['repo']['full_name'] == req['base']['repo']['full_name']
+          true
+        else
+          false
+        end
+      end
+
+      # Checks if the pull request has a head repo specified
+      def has_head_repo(req)
+        not req['head']['repo'].nil?
       end
 
       # Produces a log message
-      log_msg = Proc.new do |req|
-        head = if is_intra_branch.call(req)
-                 req['base']['repo']['full_name']
-               else
+      def log_msg(req)
+        head = if has_head_repo(req)
                  req['head']['repo']['full_name']
+               else
+                 "(head deleted)"
                end
 
         <<-eos.gsub(/\s+/, " ").strip
-            GHTorrent: Pull request #{pullreq_id}
+            GHTorrent: Pull request #{req['number']}
             #{head} -> #{req['base']['repo']['full_name']}
         eos
       end
@@ -890,22 +943,26 @@ module GHTorrent
 
       base_commit = ensure_commit(retrieved['base']['repo']['name'],
                                   retrieved['base']['sha'],
-                                  retrieved['base']['repo']['owner']['login']
-      )
+                                  retrieved['base']['repo']['owner']['login'])
 
-      if is_intra_branch.call(retrieved)
+      if is_intra_branch(retrieved)
         head_repo = base_repo
-        head_commit =
-        warn "GHTorrent: Pull request is intra branch"
-      else
-
-        head_repo = ensure_repo(retrieved['head']['repo']['owner']['login'],
-                                retrieved['head']['repo']['name'],
-                                false, false, false)
-
-        head_commit = ensure_commit(retrieved['head']['repo']['name'],
+        head_commit = ensure_commit(retrieved['base']['repo']['name'],
                                     retrieved['head']['sha'],
-                                    retrieved['head']['repo']['owner']['login'])
+                                    retrieved['base']['repo']['owner']['login'])
+        info log_msg(retrieved) + " is intra branch"
+      else
+        head_repo = if has_head_repo(retrieved)
+                      ensure_repo(retrieved['head']['repo']['owner']['login'],
+                                  retrieved['head']['repo']['name'],
+                                  false, false, false)
+                    end
+
+        head_commit = if not head_repo.nil?
+                        ensure_commit(retrieved['head']['repo']['name'],
+                                      retrieved['head']['sha'],
+                                      retrieved['head']['repo']['owner']['login'])
+                      end
       end
 
       pull_req_user = ensure_user(retrieved['user']['login'], false, false)
@@ -923,24 +980,24 @@ module GHTorrent
             :base_commit_id => base_commit[:id],
             :user_id => pull_req_user[:id],
             :pullreq_id => pullreq_id,
-            :intra_branch => is_intra_branch.call(retrieved)
+            :intra_branch => is_intra_branch(retrieved)
         )
 
-        info log_msg.call(retrieved)
+        info log_msg(retrieved)
       else
-        debug log_msg.call(retrieved) + " exists"
+        debug log_msg(retrieved) + " exists"
       end
 
       pull_req = pulls_reqs.first(:base_repo_id => project[:id],
                                   :pullreq_id => pullreq_id)
 
-      add_history.call(pull_req[:id], date(retrieved['created_at']),
+      add_history(pull_req[:id], date(retrieved['created_at']),
                        retrieved[@ext_uniq], 'opened')
-      add_history.call(pull_req[:id], date(retrieved['merged_at']),
+      add_history(pull_req[:id], date(retrieved['merged_at']),
                        retrieved[@ext_uniq], 'merged') if merged
-      add_history.call(pull_req[:id], date(retrieved['closed_at']),
+      add_history(pull_req[:id], date(retrieved['closed_at']),
                        retrieved[@ext_uniq], 'closed') if closed
-      add_history.call(pull_req[:id], date(created_at), retrieved[@ext_uniq],
+      add_history(pull_req[:id], date(created_at), retrieved[@ext_uniq],
                        state) unless state.nil?
 
       ensure_pull_request_commits(owner, repo, pullreq_id) if commits
@@ -955,7 +1012,7 @@ module GHTorrent
       time = if created_at.nil? then currepo[:created_at] else Time.now() end
 
       if currepo.nil?
-        warn "Could not repository #{owner}/#{repo}"
+        warn "GHTorrent: Could not find repository #{owner}/#{repo}"
         return
       end
 
@@ -983,7 +1040,7 @@ module GHTorrent
       pull_req = ensure_pull_request(owner, repo, pullreq_id, false, true)
 
       if pull_req.nil?
-        warn "Could not retrieve pull req #{owner}/#{repo} -> #{pullreq_id}"
+        warn "GHTorrent: Could not retrieve pull req #{owner}/#{repo} -> #{pullreq_id}"
         return
       end
 
@@ -994,7 +1051,7 @@ module GHTorrent
         retrieved = retrieve_pull_req_comment(owner, repo, pullreq_id, comment_id)
 
         if retrieved.nil?
-          warn "Could not retrieve comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id}"
+          warn "GHTorrent: Could not retrieve comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id}"
           return
         end
 
@@ -1018,14 +1075,19 @@ module GHTorrent
             :ext_ref_id => retrieved[@ext_uniq]
         )
         debug "GHTorrent: Adding comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id}"
+        @db[:pull_request_comments].first(:pull_request_id => pull_req[:id],
+                                          :comment_id => comment_id)
       else
-        debug "GHTorrent: Updating comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id}"
+        debug "GHTorrent: Comment #{comment_id} for pullreq #{owner}/#{repo} -> #{pullreq_id} exists"
+        exists
       end
     end
 
     def ensure_pull_request_commits(owner, repo, pullreq_id)
-      retrieve_pull_req_commits(owner, repo, pullreq_id).map {|c|
-        ensure_commit(repo, c['sha'], owner, true)
+      retrieve_pull_req_commits(owner, repo, pullreq_id).reduce([]){|acc, c|
+        x = ensure_commit(repo, c['sha'], owner, true)
+        acc << x if not x.nil?
+        acc
       }.map { |c|
         pullreq = ensure_pull_request(owner, repo, pullreq_id, false, false)
         exists = @db[:pull_request_commits].first(:pull_request_id => pullreq[:id],
@@ -1050,26 +1112,28 @@ module GHTorrent
     # [repo]  The repository/project to find forks for
     def ensure_forks(owner, repo)
       currepo = ensure_repo(owner, repo, false, false, false)
-      time = currepo[:created_at]
 
       if currepo.nil?
         warn "Could not retrieve forks for #{owner}/#{repo}"
         return
       end
 
-      existing_forks = @db.from(:forks, :projects).\
+      existing_forks = @db.from(:forks, :projects, :users).\
           where(:forks__forked_project_id => :projects__id). \
-          where(:forks__forked_from_id => currepo[:id]).select(:name, :login).all
+          where(:users__id => :projects__owner_id). \
+          where(:forks__forked_from_id => currepo[:id]).select(:projects__name, :login).all
 
       retrieve_forks(owner, repo).reduce([]) do |acc, x|
         if existing_forks.find {|y|
-          y[:login] == x['owner']['login'] && y[:name] == x['name']
+          forked_repo_owner = x['full_name'].split(/\//)[0]
+          forked_repo_name = x['full_name'].split(/\//)[1]
+          y[:login] == forked_repo_owner && y[:name] == forked_repo_name
         }.nil?
           acc << x
         else
           acc
         end
-      end.map { |x| ensure_fork(owner, repo, x['id'], time) }
+      end.map { |x| ensure_fork(owner, repo, x['id']) }
     end
 
     ##
@@ -1081,8 +1145,8 @@ module GHTorrent
       fork_exists = forks.first(:fork_id => fork_id)
 
       if fork_exists.nil?
-        added = if date_added.nil? then Time.now else date_added end
         retrieved = retrieve_fork(owner, repo, fork_id)
+        added = if date_added.nil? then retrieved['created_at'] else date_added end
 
         if retrieved.nil?
           warn "GHTorrent: Fork #{fork_id} does not exist for #{owner}/#{repo}"
@@ -1106,6 +1170,7 @@ module GHTorrent
                      :ext_ref_id => retrieved[@ext_uniq])
         info "GHTorrent: Added #{forked_repo_owner}/#{forked_repo_name} as fork of  #{owner}/#{repo}"
       else
+        debug "GHTorrent: Fork #{fork_id} exists as fork of #{owner}/#{repo}"
         unless date_added.nil?
           forks.filter(:fork_id => fork_id)\
                .update(:created_at => date(date_added))
@@ -1114,36 +1179,253 @@ module GHTorrent
       end
     end
 
-    private
+    ##
+    # Make sure all issues exist for a project
+    def ensure_issues(owner, repo)
+      currepo = ensure_repo(owner, repo, false, false, false)
+      if currepo.nil?
+        warn "GHTorrent: Could not retrieve issues for #{owner}/#{repo}"
+        return
+      end
 
-    # Store a commit contained in a hash. First check whether the commit exists.
-    def store_commit(c, repo, user)
-      commits = @db[:commits]
-      commit = commits.first(:sha => c['sha'])
+      issues = @db[:issues].filter(:repo_id => currepo[:id]).all
 
-      if commit.nil?
-        author = commit_user(c['author'], c['commit']['author'])
-        commiter = commit_user(c['committer'], c['commit']['committer'])
+      retrieve_issues(owner, repo).reduce([]) do |acc, x|
+        if issues.find { |y| y[:issue_id] == x['number'] }.nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x| ensure_issue(owner, repo, x['number']) }
+    end
 
-        repository = ensure_repo(user, repo, false, false, false)
+    ##
+    # Make sure that the issue exists
+    def ensure_issue(owner, repo, issue_id, events = true, comments = true)
 
-        if repository.nil?
-          warn "Could not store commit #{user}/#{repo} #{c['sha']}"
+      issues = @db[:issues]
+      repository = ensure_repo(owner, repo, false, false, false)
+
+      if repo.nil?
+        warn "Cannot find repo #{owner}/#{repo}"
+        return
+      end
+
+      cur_issue = issues.first(:issue_id => issue_id,
+                               :repo_id => repository[:id])
+
+      if cur_issue.nil?
+        retrieved = retrieve_issue(owner, repo, issue_id)
+
+        if retrieved.nil?
+          warn "GHTorrent: Issue #{issue_id} does not exist for #{owner}/#{repo}"
           return
         end
 
-        commits.insert(:sha => c['sha'],
-                       :author_id => author[:id],
-                       :committer_id => commiter[:id],
-                       :project_id => repository[:id],
-                       :created_at => date(c['commit']['author']['date']),
-                       :ext_ref_id => c[@ext_uniq]
+        reporter = ensure_user(retrieved['user']['login'], false, false)
+        assignee = unless retrieved['assignee'].nil?
+                     ensure_user(retrieved['assignee']['login'], false, false)
+                   end
+
+        # Pull requests and issues share the same issue_id
+        pull_req = unless retrieved['pull_request'].nil? or retrieved['pull_request']['patch_url'].nil?
+                     ensure_pull_request(owner, repo, issue_id)
+                   end
+
+        issues.insert(:repo_id => repository[:id],
+                     :assignee_id => unless assignee.nil? then assignee[:id] end,
+                     :reporter_id => reporter[:id],
+                     :issue_id => issue_id,
+                     :pull_request => if pull_req.nil? then false else true end,
+                     :pull_request_id => unless pull_req.nil? then pull_req[:id] end,
+                     :created_at => date(retrieved['created_at']),
+                     :ext_ref_id => retrieved[@ext_uniq])
+
+        ensure_issue_events(owner, repo, issue_id) if events
+        ensure_issue_comments(owner, repo, issue_id) if comments and retrieved['comments'] > 0
+
+        info "GHTorrent: Added issue #{owner}/#{repo} -> #{issue_id}"
+        issues.first(:issue_id => issue_id,
+                     :repo_id => repository[:id])
+      else
+        info "GHTorrent: Issue #{owner}/#{repo}->#{issue_id} exists"
+        cur_issue
+      end
+    end
+
+    ##
+    # Retrieve and process all events for an issue
+    def ensure_issue_events(owner, repo, issue_id)
+      currepo = ensure_repo(owner, repo, true, true, false)
+      #time = if created_at.nil? then currepo[:created_at] else Time.now() end
+
+      if currepo.nil?
+        warn "GHTorrent: Could not find repository #{owner}/#{repo}"
+        return
+      end
+
+      issue = ensure_issue(owner, repo, issue_id, false, false)
+      if issue.nil?
+        warn "Could not retrieve issue #{owner}/#{repo} -> #{issue_id}"
+        return
+      end
+
+      retrieve_issue_events(owner, repo, issue_id).reduce([]) do |acc, x|
+
+        if @db[:issue_events].first(:issue_id => issue[:id],
+                                    :event_id => x['id']).nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x|
+        ensure_issue_event(owner, repo, issue_id, x['id'])
+      }
+    end
+
+    ##
+    # Retrieve and process +event_id+ for an +issue_id+
+    def ensure_issue_event(owner, repo, issue_id, event_id)
+      issue = ensure_issue(owner, repo, issue_id, false, false)
+
+      if issue.nil?
+        warn "GHTorrent: Could not retrieve issue #{owner}/#{repo} -> #{issue_id}"
+        return
+      end
+
+      issue_event_str = "#{owner}/#{repo} -> #{issue_id}/#{event_id}"
+
+      curevent = @db[:issue_events].first(:issue_id => issue[:id],
+                                          :event_id => event_id)
+      if curevent.nil?
+
+        retrieved = retrieve_issue_event(owner, repo, issue_id, event_id)
+
+        if retrieved.nil?
+          warn "GHTorrent: Could not retrieve issue event #{issue_event_str}"
+          return
+        elsif retrieved['actor'].nil?
+          warn "GHTorrent: Issue event #{issue_event_str} does not contain an actor"
+          return
+        end
+
+        actor = ensure_user(retrieved['actor']['login'], false, false)
+
+        action_specific = case retrieved['event']
+                            when "referenced" then retrieved['commit_id']
+                            when "merged" then retrieved['commit_id']
+                            when "closed" then retrieved['commit_id']
+                            else nil
+                          end
+
+        if retrieved['event'] == "assigned"
+
+          def update_assignee(owner, repo, issue, actor)
+            @db[:issues][:id => issue[:id]] = {:assignee_id => actor[:id]}
+            info "Updating #{owner}/#{repo} -> #{issue[:id]} assignee to #{actor[:id]}"
+          end
+
+          if issue[:assignee_id].nil? then
+            update_assignee(owner, repo, issue, actor)
+          else
+            existing = @db[:issue_events].\
+                        filter(:issue_id => issue[:id],:action => "assigned").\
+                        order(Sequel.desc(:created_at)).first
+            if existing.nil?
+              update_assignee(owner, repo, issue, actor)
+            elsif date(existing[:created_at]) < date(retrieved['created_at'])
+              update_assignee(owner, repo, issue, actor)
+            end
+          end
+        end
+
+        @db[:issue_events].insert(
+            :event_id => event_id,
+            :issue_id => issue[:id],
+            :actor_id => unless actor.nil? then actor[:id] end,
+            :action => retrieved['event'],
+            :action_specific => action_specific,
+            :created_at => date(retrieved['created_at']),
+            :ext_ref_id => retrieved[@ext_uniq]
         )
-        debug "GHTorrent: New commit #{user}/#{repo} -> #{c['sha']} "
-        commits.first(:sha => c['sha'])
+
+        info "GHTorrent: Added issue event #{issue_event_str}"
+        @db[:issue_events].first(:issue_id => issue[:id],
+                                 :event_id => event_id)
       else
-        debug "GHTorrent: Commit #{user}/#{repo} -> #{c['sha']} exists"
-        commit
+        debug "GHTorrent: Issue event #{issue_event_str} exists"
+        curevent
+      end
+    end
+
+    ##
+    # Retrieve and process all comments for an issue
+    def ensure_issue_comments(owner, repo, issue_id)
+      currepo = ensure_repo(owner, repo, true, true, false)
+
+      if currepo.nil?
+        warn "GHTorrent: Could not find repository #{owner}/#{repo}"
+        return
+      end
+
+      issue = ensure_issue(owner, repo, issue_id, false, false)
+      if issue.nil?
+        warn "Could not retrieve issue #{owner}/#{repo} -> #{issue_id}"
+        return
+      end
+
+      retrieve_issue_comments(owner, repo, issue_id).reduce([]) do |acc, x|
+
+        if @db[:issue_comments].first(:issue_id => issue[:id],
+                                    :comment_id => x['id']).nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x|
+        ensure_issue_comment(owner, repo, issue_id, x['id'])
+      }
+    end
+
+    ##
+    # Retrieve and process +comment_id+ for an +issue_id+
+    def ensure_issue_comment(owner, repo, issue_id, comment_id)
+      issue = ensure_issue(owner, repo, issue_id)
+
+      if issue.nil?
+        warn "GHTorrent: Could not retrieve issue #{owner}/#{repo} -> #{issue_id}"
+        return
+      end
+
+      issue_comment_str = "#{owner}/#{repo} -> #{issue_id}/#{comment_id}"
+
+      curcomment = @db[:issue_comments].first(:issue_id => issue[:id],
+                                          :comment_id => comment_id)
+      if curcomment.nil?
+
+        retrieved = retrieve_issue_comment(owner, repo, issue_id, comment_id)
+
+        if retrieved.nil?
+          warn "GHTorrent: Could not retrieve issue comment #{issue_comment_str}"
+          return
+        end
+
+        user = ensure_user(retrieved['user']['login'], false, false)
+
+        @db[:issue_comments].insert(
+            :comment_id => comment_id,
+            :issue_id => issue[:id],
+            :user_id => unless user.nil? then user[:id] end,
+            :created_at => date(retrieved['created_at']),
+            :ext_ref_id => retrieved[@ext_uniq]
+        )
+
+        info "GHTorrent: Added issue comment #{issue_comment_str}"
+        @db[:issue_comments].first(:issue_id => issue[:id],
+                                   :comment_id => comment_id)
+      else
+        debug "GHTorrent: Issue comment #{issue_comment_str} exists"
+        curcomment
       end
     end
 
@@ -1153,27 +1435,57 @@ module GHTorrent
       @db ||= get_db
       @persister ||= persister
 
+      result = nil
       start_time = Time.now
       begin
         @db.transaction(:rollback => :reraise, :isolation => :committed) do
-          yield block
+          result = yield block
         end
         total = Time.now.to_ms - start_time.to_ms
         debug "GHTorrent: Transaction committed (#{total} ms)"
+        result
       rescue Exception => e
         total = Time.now.to_ms - start_time.to_ms
         warn "GHTorrent: Transaction failed (#{total} ms)"
         raise e
       ensure
-        @db.disconnect
-        @persister.close
-
-        @db = nil
-        @persister = nil
         GC.start
       end
     end
 
+    private
+
+    # Store a commit contained in a hash. First check whether the commit exists.
+    def store_commit(c, repo, user)
+      commits = @db[:commits]
+      commit = commits.first(:sha => c['sha'])
+
+      if commit.nil?
+        author = commit_user(c['author'], c['commit']['author'])
+        commiter = commit_user(c['committer'], c['commit']['committer'])
+
+        repository = ensure_repo(user, repo, false, false, false)
+
+        if repository.nil?
+          warn "Could not store commit #{user}/#{repo} #{c['sha']}"
+          return
+        end
+
+        commits.insert(:sha => c['sha'],
+                       :author_id => author[:id],
+                       :committer_id => commiter[:id],
+                       :project_id => repository[:id],
+                       :created_at => date(c['commit']['author']['date']),
+                       :ext_ref_id => c[@ext_uniq]
+        )
+        debug "GHTorrent: New commit #{user}/#{repo} -> #{c['sha']} "
+        commits.first(:sha => c['sha'])
+      else
+        debug "GHTorrent: Commit #{user}/#{repo} -> #{c['sha']} exists"
+        commit
+      end
+    end
+
     ##
     # Convert a string value to boolean, the SQL way
     def boolean(arg)