ghtorrent 0.2 → 0.3

data/lib/ghtorrent/command.rb

@@ -26,7 +26,10 @@
 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
+require 'rubygems'
 require 'trollop'
+require 'daemons'
+require 'etc'
 
 # Base class for all GHTorrent command line utilities. Provides basic command
 # line argument parsing and command bootstraping support. The order of
@@ -38,7 +41,7 @@ require 'trollop'
 module GHTorrent
   class Command
 
-    attr_reader :args, :options
+    attr_reader :args, :options, :name
 
     # Specify the run method for subclasses.
     class << self
@@ -47,6 +50,29 @@ module GHTorrent
         command.process_options
         command.validate
 
+        if command.options[:daemon]
+          if Process.uid == 0
+            # Daemonize as a proper system daemon
+            Daemons.daemonize(:app_name   => File.basename($0),
+                              :dir_mode   => :system,
+                              :log_dir    => "/var/log",
+                              :backtrace  => true,
+                              :log_output => true)
+            STDERR.puts "Became a daemon"
+            # Change effective user id for the process
+            unless command.options[:user].nil?
+              Process.euid = Etc.getpwnam(command.options[:user]).uid
+            end
+          else
+            # Daemonize, but output in current directory
+            Daemons.daemonize(:app_name   => File.basename($0),
+                              :dir_mode   => :normal,
+                              :dir        => Dir.getwd,
+                              :backtrace  => true,
+                              :log_output => true)
+          end
+        end
+
         begin
           command.go
         rescue => e
@@ -63,6 +89,7 @@ module GHTorrent
 
     def initialize(args)
       @args = args
+      @name = self.class.name
     end
 
     # Specify and parse supported command line options.
@@ -79,6 +106,9 @@ Standard options:
         opt :config, 'config.yaml file location', :short => 'c',
             :default => 'config.yaml'
         opt :verbose, 'verbose mode', :short => 'v'
+        opt :daemon, 'run as daemon', :short => 'd'
+        opt :user, 'run as the specified user (only when started as root)',
+            :short => 'u', :type => String
       end
 
       @args = @args.dup
@@ -103,11 +133,22 @@ Standard options:
         unless (file_exists?("config.yaml") or file_exists?("/etc/ghtorrent/config.yaml"))
           Trollop::die "No config file in default locations (., /etc/ghtorrent)
                       you need to specify the #{:config} parameter. Read the
-                      documnetation on how to create a config.yaml file."
+                      documentation on how to create a config.yaml file."
         end
       else
         Trollop::die "Cannot find file #{options[:config]}" unless file_exists?(options[:config])
       end
+
+      unless @options[:user].nil?
+        if not Process.uid == 0
+          Trollop::die "Option --user (-u) cannot be specified by normal users"
+        end
+          begin
+            Etc.getpwnam(@options[:user])
+          rescue ArgumentError
+            Trollop::die "No such user: #{@options[:user]}"
+          end
+      end
     end
 
     # Name of the command that is currently being executed.

data/lib/ghtorrent/ghtorrent.rb

@@ -78,39 +78,71 @@ module GHTorrent
         return
       end
 
+      transaction do
+        ensure_repo(user, repo)
+        ensure_commit(repo, sha, user)
+      end
+    end
+
+    ##
+    # Make sure a commit exists
+    def ensure_commit(repo, sha, user)
+      c = retrieve_commit(repo, sha, user)
+      store_commit(c, repo, user)
+      ensure_commit_comments(user, repo, sha)
+      ensure_parents(c)
+      c
+    end
+
+    ##
+    # Get as many commits for a repository as allowed by Github
+    #
+    # ==Parameters:
+    # [user]  The user to whom the repo belongs.
+    # [repo]  The repo to look for commits into.
+    def ensure_commits(user, repo)
+      userid = @db[:users].filter(:login => user).first[:id]
+      repoid = @db[:projects].filter(:owner_id => userid,
+                                     :name => repo).first[:id]
+
+      latest = @db[:commits].filter(:project_id => repoid).order(:created_at).last
+      commits = if latest.nil?
+                  retrieve_commits(repo, nil, user)
+                else
+                  retrieve_commits(repo, latest[:sha], user)
+                end
+
+      commits.map do |c|
+        ensure_commit(repo, c['sha'], user)
+      end
+    end
+
+    ##
+    # Get the parents for a specific commit. The commit must be first stored
+    # in the database.
+    def ensure_parents(commit)
       commits = @db[:commits]
-      commit = commits.first(:sha => sha)
+      commit['parents'].each do |p|
+          parents =  @db[:commit_parents]
+          url = p['url'].split(/\//)
+          this = commits.first(:sha => commit['sha'])
+          parent = commits.first(:sha => url[7])
+
+          if parent.nil?
+            store_commit(retrieve_commit(url[5], url[7], url[4]), url[5], url[4])
+            parent = commits.first(:sha => url[7])
+          end
 
-      if commit.nil?
-        @db.transaction(:rollback => :reraise) do
-          ensure_repo(user, repo)
-          c = retrieve_commit(repo, sha, user)
-
-          author = commit_user(c['author'], c['commit']['author'])
-          commiter = commit_user(c['committer'], c['commit']['committer'])
-
-          commits.insert(:sha => sha,
-                         :author_id => author[:id],
-                         :committer_id => commiter[:id],
-                         :created_at => date(c['commit']['author']['date']),
-                         :ext_ref_id => c[@ext_uniq]
-          )
+          if parents.first(:commit_id => this[:id],
+                           :parent_id => parent[:id]).nil?
 
-          #c['parents'].each do |p|
-          #  url = p['url'].split(/\//)
-          #  get_commit url[4], url[5], url[7]
-          #
-          #  commit = commits.first(:sha => sha)
-          #  parent = commits.first(:sha => url[7])
-          #  @db[:commit_parents].insert(:commit_id => commit[:id],
-          #                              :parent_id => parent[:id])
-          #  @log.info "Added parent #{parent[:sha]} to commit #{sha}"
-          #end
+            parents.insert(:commit_id => this[:id],
+                           :parent_id => parent[:id])
+            info "Added parent #{parent[:sha]} to commit #{this[:sha]}"
+          else
+            info "Parent #{parent[:sha]} for commit #{this[:sha]} exists"
+          end
         end
-        debug "GHTorrent: Transaction committed"
-      else
-        debug "GHTorrent: Commit #{sha} exists"
-      end
     end
 
     ##
@@ -120,10 +152,8 @@ module GHTorrent
     # Resolution of how
     #
     # ==Parameters:
-    #  githubuser::
-    #     A hash containing the user's Github login
-    #  commituser::
-    #     A hash containing the Git commit's user name and email
+    # [githubuser]  A hash containing the user's Github login
+    # [commituser]  A hash containing the Git commit's user name and email
     # == Returns:
     # The (added/modified) user entry as a Hash.
     def commit_user(githubuser, commituser)
@@ -139,13 +169,13 @@ module GHTorrent
       login = githubuser['login'] unless githubuser.nil?
 
       if login.nil?
-        ensure_user("#{name}<#{email}>", true)
+        ensure_user("#{name}<#{email}>", true, false)
       else
         dbuser = users.first(:login => login)
         byemail = users.first(:email => email)
         if dbuser.nil?
           # We do not have the user in the database yet. Add him
-          added = ensure_user(login, true)
+          added = ensure_user(login, true, false)
           if byemail.nil?
             #
             users.filter(:login => login).update(:name => name) if added[:name].nil?
@@ -177,28 +207,31 @@ module GHTorrent
     ##
     # Ensure that a user exists, or fetch its latest state from Github
     # ==Parameters:
-    #  user::
-    #     The full email address in RFC 822 format
-    #     or a login name to lookup the user by
-    #  followers::
-    #     A boolean value indicating whether to retrieve the user's followers
-    # == Returns:
+    #  [user] The full email address in RFC 822 format or a login name to lookup
+    #         the user by
+    #  [followers] A boolean value indicating whether to retrieve the user's
+    #              followers
+    #  [orgs] A boolean value indicating whether to retrieve the organizations
+    #         the user participates into
+    # ==Returns:
     # If the user can be retrieved, it is returned as a Hash. Otherwise,
     # the result is nil
-    def ensure_user(user, followers)
+    def ensure_user(user, followers, orgs)
       # Github only supports alpa-nums and dashes in its usernames.
       # All other sympbols are treated as emails.
-      u = if not user.match(/^[A-Za-z0-9\-]*$/)
-            begin
-              name, email = user.split("<")
-              email = email.split(">")[0]
-            rescue Exception
-              raise new GHTorrentException("Not a valid email address: #{user}")
-            end
-            ensure_user_byemail(email.strip, name.strip, followers)
-          else
-            ensure_user_byuname(user, followers)
-          end
+      if not user.match(/^[A-Za-z0-9\-]*$/)
+        begin
+          name, email = user.split("<")
+          email = email.split(">")[0]
+        rescue Exception
+          raise new GHTorrentException("Not a valid email address: #{user}")
+        end
+        u = ensure_user_byemail(email.strip, name.strip)
+      else
+        u = ensure_user_byuname(user)
+        ensure_user_followers(user) if followers
+        ensure_orgs(user) if orgs
+      end
       return u
     end
 
@@ -211,7 +244,7 @@ module GHTorrent
     # == Returns:
     # If the user can be retrieved, it is returned as a Hash. Otherwise,
     # the result is nil
-    def ensure_user_byuname(user, followers)
+    def ensure_user_byuname(user)
       users = @db[:users]
       usr = users.first(:login => user)
 
@@ -232,14 +265,12 @@ module GHTorrent
                      :hireable => boolean(u['hirable']),
                      :bio => u['bio'],
                      :location => u['location'],
+                     :type =>  user_type(u['type']),
                      :created_at => date(u['created_at']),
                      :ext_ref_id => u[@ext_uniq])
 
         info "GHTorrent: New user #{user}"
 
-        # Get the user's followers
-        ensure_user_followers(user) if followers
-
         users.first(:login => user)
       else
         debug "GHTorrent: User #{user} exists"
@@ -256,11 +287,11 @@ module GHTorrent
     # [user]  The user login to find followers by
     def ensure_user_followers(user, ts = Time.now)
 
-      followers = retrieve_new_user_followers(user)
+      followers = retrieve_user_followers(user)
       followers.each { |f|
         follower = f['login']
-        ensure_user(user, false)
-        ensure_user(follower, false)
+        ensure_user(user, false, false)
+        ensure_user(follower, false, false)
 
         userid = @db[:users].select(:id).first(:login => user)[:id]
         followerid = @db[:users].select(:id).first(:login => follower)[:id]
@@ -284,13 +315,13 @@ module GHTorrent
     # Github API v2 if unsuccessful.
     #
     # ==Parameters:
-    #  user::
-    #     The email to lookup the user by
-    #
+    # [email]  The email to lookup the user by
+    # [email]  The user's name
+    # [followers]  If true, the user's followers will be retrieved
     # == Returns:
     # If the user can be retrieved, it is returned as a Hash. Otherwise,
     # the result is nil
-    def ensure_user_byemail(email, name, followers)
+    def ensure_user_byemail(email, name)
       users = @db[:users]
       usr = users.first(:email => email)
 
@@ -318,7 +349,6 @@ module GHTorrent
                        :created_at => date(u['user']['created_at']),
                        :ext_ref_id => u[@ext_uniq])
           debug "GHTorrent: Found #{email} through API v2 query"
-          ensure_user_followers(user) if followers
           users.first(:email => email)
         end
       else
@@ -334,11 +364,12 @@ module GHTorrent
     # [user]  The email or login name to which this repo belongs
     # [repo]  The repo name
     #
-    # == Returns: If the repo can be retrieved, it is returned as a Hash.
-    #             Otherwise, the result is nil
+    # == Returns:
+    #  If the repo can be retrieved, it is returned as a Hash. Otherwise,
+    #  the result is nil
     def ensure_repo(user, repo)
 
-      ensure_user(user, false)
+      ensure_user(user, true, true)
       repos = @db[:projects]
       currepo = repos.first(:name => repo)
 
@@ -353,6 +384,7 @@ module GHTorrent
                      :ext_ref_id => r[@ext_uniq])
 
         info "GHTorrent: New repo #{repo}"
+        ensure_commits(user, repo)
         repos.first(:name => repo)
       else
         debug "GHTorrent: Repo #{repo} exists"
@@ -360,7 +392,165 @@ module GHTorrent
       end
     end
 
-  private
+    ##
+    # Make sure that the organizations the user participates into exist
+    #
+    # ==Parameters:
+    # [user]  The login name of the user to check the organizations for
+    #
+    def ensure_orgs(user)
+      usr = @db[:users].first(:login => user)
+      retrieve_orgs(user).map{|o| ensure_participation(user, o['login'])}
+    end
+
+    ##
+    # Make sure that a user belongs to the provided organization
+    #
+    # ==Parameters:
+    # [user] The login name of the user to check the organizations for
+    # [org]  The login name of the organization to check whether the user
+    #        belongs in
+    #
+    def ensure_participation(user, organization)
+      org = ensure_org(organization)
+      usr = ensure_user(user, false, false)
+
+      org_members = @db[:organization_members]
+      participates = org_members.first(:user_id => usr[:id], :org_id => org[:id])
+
+      if participates.nil?
+        org_members.insert(:user_id => usr[:id],
+                           :org_id => org[:id])
+        info "GHTorrent: Added participation #{organization} -> #{user}"
+        org_members.first(:user_id => usr[:id], :org_id => org[:id])
+      else
+        debug "GHTorrent: Participation #{organization} -> #{user} exists"
+        participates
+      end
+
+    end
+
+    ##
+    # Make sure that an organization exists
+    #
+    # ==Parameters:
+    # [org]  The login name of the organization
+    #
+    def ensure_org(organization)
+      org = @db[:users].find(:login => organization, :type => 'org')
+
+      if org.nil?
+        ensure_user(org, false, false)
+      else
+        debug "GHTorrent: Organization #{organization} exists"
+        org.first
+      end
+    end
+
+
+    ##
+    # Get all comments for a commit
+    #
+    # ==Parameters:
+    # [user]  The login name of the organization
+    def ensure_commit_comments(user, repo, sha)
+      commit_id = @db[:commits].first(:sha => sha)
+      stored_comments = @db[:commit_comments].find(:commit_id => commit_id)
+      commit_commets = retrieve_commit_comments(user, repo, sha)
+      user_id = @db[:users].first(:login => user)[:id]
+
+      not_saved = commit_commets.reduce([]) do |acc, x|
+        if stored_comments.find{|y| y[:comment_id] == x['comment_id']}.nil?
+          acc << x
+        else
+          acc
+        end
+      end
+
+      not_saved.each do |c|
+        @db[:commit_comments].insert(
+          :commit_id => commit_id,
+          :user_id => user_id,
+          :body => c['body'],
+          :line => c['line'],
+          :position => c['position'],
+          :comment_id => c['id'],
+          :ext_ref_id => c['ext_ref_id'],
+          :created_at => date(c['created_at'])
+        )
+        info "GHTorrent: Added commit comment #{sha} -> #{user}"
+      end
+    end
+
+    ##
+    # Get a specific comment
+    #
+    # ==Parameters:
+    # [user]  The login name of the organization
+    def ensure_commit_comment(user, repo, id)
+      stored_comment = @db[:commit_comments].first(:comment_id => id)
+
+      if stored_comment.nil?
+        retrieved = retrieve_commit_comment(user, repo, id)
+        commit = ensure_commit(repo, retrieved['commit_id'], user)
+        user = ensure_user(user, false, false)
+        @db[:commit_comments].insert(
+            :commit_id => commit[:id],
+            :user_id => user[:id],
+            :body => retrieved['body'],
+            :line => retrieved['line'],
+            :position => retrieved['position'],
+            :comment_id => retrieved['id'],
+            :ext_ref_id => retrieved['ext_ref_id'],
+            :created_at => date(retrieved['created_at'])
+        )
+        @db[:commit_comments].first(:comment_id => id)
+        info "GHTorrent: Added commit comment #{commit[:sha]} -> #{user}"
+      else
+        info "GHTorrent: Commit comment #{id} exists"
+        stored_comment
+      end
+    end
+
+    private
+
+    # Store a commit contained in a hash. First check whether the commit exists.
+    def store_commit(c, repo, user)
+      commits = @db[:commits]
+      commit = commits.first(:sha => c['sha'])
+
+      if commit.nil?
+        author = commit_user(c['author'], c['commit']['author'])
+        commiter = commit_user(c['committer'], c['commit']['committer'])
+
+
+        userid = @db[:users].filter(:login => user).first[:id]
+        repoid = @db[:projects].filter(:owner_id => userid,
+                                       :name => repo).first[:id]
+
+        commits.insert(:sha => c['sha'],
+                       :author_id => author[:id],
+                       :committer_id => commiter[:id],
+                       :project_id => repoid,
+                       :created_at => date(c['commit']['author']['date']),
+                       :ext_ref_id => c[@ext_uniq]
+        )
+        debug "GHTorrent: New commit #{repo} -> #{c['sha']} "
+      else
+        debug "GHTorrent: Commit #{repo} -> #{c['sha']} exists"
+      end
+    end
+
+    # Run a block in a DB transaction. Exceptions trigger transaction rollback
+    # and are rethrown.
+    def transaction(&block)
+      start_time = Time.now
+      @db.transaction(:rollback => :reraise, :isolation => :committed) do
+        yield block
+      end
+      total = Time.now.to_ms - start_time.to_ms
+      debug "GHTorrent: Transaction committed (#{total} ms)"
+    end
 
     ##
     # Convert a string value to boolean, the SQL way
@@ -379,7 +569,7 @@ module GHTorrent
     # - yyyy-mm-ddThh:mm:ssZ
     # - yyyy/mm/dd hh:mm:ss {+/-}hhmm
     def date(arg)
-      Time.parse(arg).to_i
+      Time.parse(arg)#.to_i
     end
 
     def is_valid_email(email)
@@ -388,9 +578,13 @@ module GHTorrent
   end
   # Base exception for all GHTorrent exceptions
   class GHTorrentException < Exception
-
   end
+end
 
+class Time
+  def to_ms
+    (self.to_f * 1000.0).to_i
+  end
 end
 
 # vim: set sta sts=2 shiftwidth=2 sw=2 et ai :