ghtorrent 0.9 → 0.10

data/lib/ghtorrent/commands/ght_retrieve_user.rb

@@ -5,6 +5,7 @@ require 'ghtorrent/settings'
 require 'ghtorrent/logging'
 require 'ghtorrent/command'
 require 'ghtorrent/retriever'
+require 'ghtorrent/transacted_ghtorrent'
 require 'ghtorrent/commands/ght_retrieve_repo'
 
 class GHTRetrieveUser < GHTRetrieveRepo
@@ -20,7 +21,7 @@ An efficient way to get all data for a single user
 
   def validate
     super
-    Trollop::die "One argument are required" unless args[0] && !args[0].empty?
+    Trollop::die "One argument is required" unless args[0] && !args[0].empty?
   end
 
   def go
@@ -44,19 +45,19 @@ An efficient way to get all data for a single user
 
     functions = %w(ensure_user_followers ensure_orgs ensure_org)
 
-    if ARGV[2].nil?
+    if ARGV[1].nil?
       functions.each do |x|
         send_message(x, user)
       end
     else
-      Trollop::die("Not a valid function: #{ARGV[2]}") unless functions.include? ARGV[2]
-      send_message(ARGV[2], user)
+      Trollop::die("Not a valid function: #{ARGV[1]}") unless functions.include? ARGV[1]
+      send_message(ARGV[1], user)
     end
 
   end
 end
 
-class TransactedGHTorrent < GHTorrent::Mirror
+class TransactedGhtorrent 
 
   def ensure_user_followers(user)
     check_transaction do

data/lib/ghtorrent/ghtorrent.rb

@@ -23,6 +23,11 @@ module GHTorrent
       debug "GHTorrent: Using cache dir #{config(:cache_dir)}"
     end
 
+    def dispose
+      @db.disconnect unless @db.nil?
+      @persister.close unless @persister.nil?
+    end
+
     # Get a connection to the database
     def get_db
       return @db unless @db.nil?
@@ -199,35 +204,38 @@ module GHTorrent
 
     ##
     # Retrieve commits for a repository, starting from +sha+
-    # and going back to 30 * +num_pages+ commit log entries.
     # ==Parameters:
     # [user]  The user to whom the repo belongs.
     # [repo]  The repo to look for commits into.
-    # [sha]   The first commit to start retrieving from. If nil, then the
-    #         earliest stored commit will be used instead.
-    # [num_pages] The number of commit pages to retrieve
-    def ensure_commits(user, repo, refresh = false, sha = nil,
-                       num_pages = config(:mirror_commit_pages_new_repo)
-                       )
-      userid = @db[:users].filter(:login => user).first[:id]
-      repoid = @db[:projects].filter(:owner_id => userid,
-                                     :name => repo).first[:id]
-
-      latest = if sha.nil?
-                 @db[:commits].filter(:project_id => repoid).order(:created_at).last
-               else
-                 sha
-               end
+    # [sha]   The first commit to start retrieving from. If nil, then retrieval
+    #         starts from what the project considers as master branch.
+    # [return_retrieved] Should retrieved commits be returned? If not, memory is
+    #                    saved while processing them if this is false
+    def ensure_commits(user, repo, sha = nil, return_retrieved = false)
+
+      commits = ['foo'] # Dummy entry for simplifying the loop below
+      commit_acc = []
+      until commits.empty?
+        commits = retrieve_commits(repo, sha, user, 1)
+
+        # This means that we retrieved the last commit page again
+        if commits.size == 1 and commits[0]['sha'] == sha
+          commits = []
+        end
 
-      commits = if latest.nil?
-                  retrieve_commits(repo, "head", user, num_pages)
-                else
-                  retrieve_commits(repo, latest[:sha], user, num_pages)
-                end
+        retrieved = commits.map do |c|
+          sha = c['sha']
+          save{ensure_commit(repo, c['sha'], user)}
+        end
 
-      commits.map do |c|
-        save{ensure_commit(repo, c['sha'], user)}
-      end.select{|x| !x.nil?}
+        # Store retrieved commits to return, if client requested so
+        if return_retrieved
+          commit_acc = commit_acc << retrieved
+        end
+
+      end
+
+      commit_acc.select{|x| !x.nil?}
     end
 
     ##
@@ -293,7 +301,7 @@ module GHTorrent
             :project_id => project[:id],
             :commit_id => commitid
         )
-        info "GHTorrent: Associating commit  #{sha} with #{user}/#{repo}"
+        debug "GHTorrent: Associating commit  #{sha} with #{user}/#{repo}"
         @db[:project_commits].first(:project_id => project[:id],
                                     :commit_id => commitid)
       else
@@ -450,11 +458,18 @@ module GHTorrent
                      :company => u['company'],
                      :email => email,
                      :location => u['location'],
+                     :fake => false,
                      :type => user_type(u['type']),
                      :created_at => date(u['created_at']),
                      :ext_ref_id => u[@ext_uniq])
 
         info "GHTorrent: New user #{user}"
+
+        if user_type(u['type']) == 'ORG'
+          info "GHTorrent: User #{user} is an organization. Retrieving members"
+          ensure_org(u['login'], true)
+        end
+
         users.first(:login => user)
       else
         debug "GHTorrent: User #{user} exists"
@@ -557,6 +572,7 @@ module GHTorrent
           users.insert(:email => email,
                        :name => name,
                        :login => login,
+                       :fake => true,
                        :created_at => Time.now,
                        :ext_ref_id => "")
           info "GHTorrent: Added fake user #{login} -> #{email}"
@@ -569,6 +585,7 @@ module GHTorrent
                          :company => u['company'],
                          :email => u['email'],
                          :location => u['location'],
+                         :fake => false,
                          :created_at => date(u['created_at']),
                          :ext_ref_id => u[@ext_uniq])
             info "GHTorrent: Found #{email} through search API query"
@@ -577,6 +594,7 @@ module GHTorrent
                          :company => u['company'],
                          :email => u['email'],
                          :location => u['location'],
+                         :fake => false,
                          :created_at => date(u['created_at']),
                          :ext_ref_id => u[@ext_uniq])
             info "GHTorrent: User with email #{email} exists with username #{u['login']}"
@@ -599,8 +617,8 @@ module GHTorrent
     # == Returns:
     #  If the repo can be retrieved, it is returned as a Hash. Otherwise,
     #  the result is nil
-    def ensure_repo(user, repo, commits = false, project_members = false,
-                    watchers = false, forks = false, labels = false)
+    def ensure_repo(user, repo, commits = true, project_members = true, 
+                    watchers = true, forks = true, labels = true)
 
       repos = @db[:projects]
       curuser = ensure_user(user, false, false)
@@ -640,11 +658,96 @@ module GHTorrent
         end
 
         info "GHTorrent: New repo #{user}/#{repo}"
-        ensure_commits(user, repo) if commits
-        ensure_project_members(user, repo) if project_members
-        ensure_watchers(user, repo) if watchers
-        ensure_forks(user, repo) if forks
-        ensure_labels(user, repo) if labels
+
+        begin
+          watchdog = nil
+          unless parent.nil?
+            watchdog = Thread.new do
+              slept = 0
+              while true do
+                debug "GHTorrent: In ensure_repo_fork for #{slept} seconds"
+                sleep 1
+                slept += 1
+              end
+            end
+            # Fast path to project forking. Retrieve all commits page by page
+            # until we reach a commit that has been registered with the parent
+            # repository. Then, copy all remaining parent commits to this repo.
+            debug "GHTorrent: Retrieving commits for #{user}/#{repo} until we reach a commit shared with the parent"
+
+            sha = nil
+            # Refresh the latest commits for the parent.
+            retrieve_commits(parent_repo, sha, parent_owner, 1).each do |c|
+              sha = c['sha']
+              ensure_commit(parent_repo, sha, parent_owner, true)
+            end
+
+            sha = nil
+            found = false
+            while not found
+              processed = 0
+              commits = retrieve_commits(repo, sha, user, 1)
+
+              # If only one commit has been retrieved (and this is the same as
+              # the commit since which we query commits from) this mean that
+              # there are no more commits.
+              if commits.size == 1 and commits[0]['sha'] == sha
+                debug "GHTorrent: No shared commit found and no more commits for #{user}/#{repo}"
+                break
+              end
+
+              for c in commits
+                processed += 1
+                exists_in_parent =
+                    !@db.from(:project_commits, :commits).\
+                         where(:project_commits__commit_id => :commits__id).\
+                         where(:project_commits__project_id => parent[:id]).\
+                         where(:commits__sha => c['sha']).first.nil?
+
+                sha = c['sha']
+                if not exists_in_parent
+                  ensure_commit(repo, sha, user, true)
+                else
+                  found = true
+                  debug "GHTorrent: Found commit #{sha} shared with parent, switching to copying commits"
+                  break
+                end
+              end
+              if processed == 0
+                warn "No commits found for #{user}/#{repo}, repo deleted?"
+                found = true
+              end
+            end
+
+            if found
+              shared_commit = @db[:commits].first(:sha => sha)
+              forked_repo = repos.first(:owner_id => curuser[:id], :name => repo)
+
+              @db.from(:project_commits, :commits).\
+                  where(:project_commits__commit_id => :commits__id).\
+                  where(:project_commits__project_id => parent[:id]).\
+                  where('commits.created_at < ?', shared_commit[:created_at]).\
+                  select(:commits__id, :commits__sha).\
+                  each do |c|
+                    @db[:project_commits].insert(
+                        :project_id => forked_repo[:id],
+                        :commit_id => c[:id]
+                    )
+                  debug "GHTorrent: Copied commit #{c[:sha]} from #{parent_owner}/#{parent_repo} -> #{user}/#{repo}"
+              end
+            end
+          else
+            ensure_commits(user, repo) if commits
+          end
+          ensure_project_members(user, repo) if project_members
+          ensure_watchers(user, repo) if watchers
+          ensure_forks(user, repo) if forks
+          ensure_labels(user, repo) if labels
+        ensure
+          unless watchdog.nil?
+            watchdog.exit
+          end
+        end
         repos.first(:owner_id => curuser[:id], :name => repo)
       else
         debug "GHTorrent: Repo #{user}/#{repo} exists"
@@ -777,12 +880,11 @@ module GHTorrent
           warn "GHTorrent: Account #{organization} is not an organization"
           return nil
         end
-
-        if members
-          retrieve_org_members(organization).map do |x|
-            ensure_participation(ensure_user(x['login'], false, false)[:login],
-                                 organization, false)
-          end
+      end
+      if members
+        retrieve_org_members(organization).map do |x|
+          ensure_participation(ensure_user(x['login'], false, false)[:login],
+                               organization, false)
         end
       end
       org
@@ -819,7 +921,7 @@ module GHTorrent
         retrieved = retrieve_commit_comment(owner, repo, sha, comment_id)
 
         if retrieved.nil?
-          warn "GHTorrent: Commit comment #{sha}->#{id} deleted"
+          warn "GHTorrent: Commit comment #{sha}->#{comment_id} deleted"
           return
         end
 
@@ -837,7 +939,7 @@ module GHTorrent
         )
         info "GHTorrent: Added commit comment #{sha} -> #{retrieved['id']} by #{user[:login]}"
       else
-        info "GHTorrent: Commit comment #{sha} -> #{id} exists"
+        info "GHTorrent: Commit comment #{sha} -> #{comment_id} exists"
       end
       @db[:commit_comments].first(:comment_id => comment_id)
     end
@@ -942,6 +1044,42 @@ module GHTorrent
       raw_pull_reqs.map { |x| save { ensure_pull_request(owner, repo, x['number']) } }.select { |x| !x.nil? }
     end
 
+    # Adds a pull request history event
+    def ensure_pull_request_history(id, ts, unq, act, actor)
+      user = unless actor.nil?
+               ensure_user(actor, false, false)
+             end
+      pull_req_history = @db[:pull_request_history]
+
+      entry =  if ['opened', 'merged'].include? act
+                  pull_req_history.first(:pull_request_id => id,
+                                         :action => act)
+               else
+                 pull_req_history.first(:pull_request_id => id,
+                                        :created_at => (ts - 3)..(ts + 3),
+                                        :action => act)
+               end
+
+      if entry.nil?
+        pull_req_history.insert(:pull_request_id => id,
+                                :created_at => ts,
+                                :ext_ref_id => unq,
+                                :action => act,
+                                :actor_id => unless user.nil? then user[:id] end)
+        info "GHTorrent: New pull request (#{id}) event (#{act}) by (#{actor}) timestamp #{ts}"
+      else
+        info "GHTorrent: Pull request (#{id}) event (#{act}) by (#{actor}) timestamp #{ts} exists"
+        if entry[:actor_id].nil? and not user.nil?
+          pull_req_history.where(:pull_request_id => id,
+                               :created_at => (ts - 3)..(ts + 3),
+                               :action => act)\
+                          .update(:actor_id => user[:id])
+          debug "Pull request (#{id}) event (#{act}) timestamp #{ts} set actor -> #{user[:login]}"
+        end
+      end
+    end
+
+
     ##
     # Process a pull request
     def ensure_pull_request(owner, repo, pullreq_id,
@@ -955,26 +1093,6 @@ module GHTorrent
         return
       end
 
-      # Adds a pull request history event
-      def add_history(id, ts, unq, act, actor)
-        user = ensure_user(actor, false, false)
-        pull_req_history = @db[:pull_request_history]
-        entry = pull_req_history.first(:pull_request_id => id,
-                                       :created_at => (ts - 4)..(ts + 4),
-                                       :action => act)
-        if entry.nil?
-          pull_req_history.insert(:pull_request_id => id,
-                                  :created_at => ts,
-                                  :ext_ref_id => unq,
-                                  :action => act,
-                                  :actor_id => unless user.nil? then user[:id] end)
-          info "GHTorrent: New pull request (#{id}) event (#{act}) by (#{actor}) timestamp #{ts}"
-        else
-          entry.update(:actor_id => user[:id])
-          info "GHTorrent: Pull request (#{id}) history entry (#{act}) by (#{actor}) timestamp #{ts} exists"
-        end
-      end
-
       # Checks whether a pull request concerns two branches of the same
       # repository
       def is_intra_branch(req)
@@ -1054,10 +1172,8 @@ module GHTorrent
             :base_repo_id => base_repo[:id],
             :head_commit_id => if not head_commit.nil? then head_commit[:id] end,
             :base_commit_id => base_commit[:id],
-            :user_id => pull_req_user[:id],
             :pullreq_id => pullreq_id,
-            :intra_branch => is_intra_branch(retrieved),
-            :merged => merged
+            :intra_branch => is_intra_branch(retrieved)
         )
         info log_msg(retrieved) + ' was added'
       else
@@ -1088,18 +1204,18 @@ module GHTorrent
 
       if history
         # Actions on pull requests
-        actor = if actor.nil? then pull_req_user[:login] else actor end
         opener = pull_req_user[:login]
-        add_history(pull_req[:id], date(retrieved['created_at']),
+        ensure_pull_request_history(pull_req[:id], date(retrieved['created_at']),
                        retrieved[@ext_uniq], 'opened', opener)
-        # There is an additional merged_by field for merged pull requests
+
         merger = if retrieved['merged_by'].nil? then actor else retrieved['merged_by']['login'] end
-        add_history(pull_req[:id], date(retrieved['merged_at']),
+        ensure_pull_request_history(pull_req[:id], date(retrieved['merged_at']),
                          retrieved[@ext_uniq], 'merged', merger) if (merged && state != 'merged')
+
         closer = if merged then merger else actor end
-        add_history(pull_req[:id], date(retrieved['closed_at']),
+        ensure_pull_request_history(pull_req[:id], date(retrieved['closed_at']),
                          retrieved[@ext_uniq], 'closed', closer) if (closed && state != 'closed')
-        add_history(pull_req[:id], date(created_at), retrieved[@ext_uniq],
+        ensure_pull_request_history(pull_req[:id], date(created_at), retrieved[@ext_uniq],
                          state, actor) unless state.nil?
       end
       ensure_pull_request_commits(owner, repo, pullreq_id) if commits
@@ -1239,11 +1355,11 @@ module GHTorrent
           where(:projects__forked_from => currepo[:id]).select(:projects__name, :login).all
 
       retrieve_forks(owner, repo).reduce([]) do |acc, x|
-        if existing_forks.find {|y|
+        if existing_forks.find do |y|
           forked_repo_owner = x['full_name'].split(/\//)[0]
           forked_repo_name = x['full_name'].split(/\//)[1]
           y[:login] == forked_repo_owner && y[:name] == forked_repo_name
-        }.nil?
+        end.nil?
           acc << x
         else
           acc
@@ -1669,7 +1785,7 @@ module GHTorrent
       result = nil
       start_time = Time.now
       begin
-        @db.transaction(:rollback => :reraise, :isolation => :committed) do
+        @db.transaction(:rollback => :reraise, :isolation => :uncommitted) do
           result = yield block
         end
         total = Time.now.to_ms - start_time.to_ms
@@ -1698,8 +1814,6 @@ module GHTorrent
       end
     end
 
-    private
-
     # Store a commit contained in a hash. First check whether the commit exists.
     def store_commit(c, repo, user)
       commits = @db[:commits]

data/lib/ghtorrent/migrations/016_add_actor_pull_request_history.rb

@@ -10,7 +10,7 @@ Sequel.migration do
       add_foreign_key :actor_id, :users
     end
 
-    puts 'Remember to run the fixes/update_pull_request_history_actor.rb
+    puts 'Remember to run the fixes/update_pullreq_entries_from_events.rb
  script to mark deleted projects'
   end