ghtorrent 0.3.1 → 0.4

data/lib/ghtorrent/api_client.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'net/http'
 require 'set'
 require 'open-uri'
@@ -132,6 +104,7 @@ module GHTorrent
             STDERR.puts "#{url}: #{e.io.status[1]}"
             return nil
           else # Server error or HTTP conditions that Github does not report
+            STDERR.puts "#{url}"
             raise e
         end
       end

data/lib/ghtorrent/call_stack.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 module GHTorrent
   class CallStack
 

data/lib/ghtorrent/command.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'rubygems'
 require 'trollop'
 require 'daemons'

data/lib/ghtorrent/ghtorrent.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'sequel'
 
 module GHTorrent
@@ -65,12 +37,7 @@ module GHTorrent
     ##
     # Ensure that a user exists, or fetch its latest state from Github
     # ==Parameters:
-    #  user::
-    #     The email or login name to lookup the user by
-    #
-    # == Returns:
-    # If the user can be retrieved, it is returned as a Hash. Otherwise,
-    # the result is nil
+    #  [user] The email or login name to lookup the user by
     def get_commit(user, repo, sha)
 
       unless sha.match(/[a-f0-9]{40}$/)
@@ -84,14 +51,71 @@ module GHTorrent
       end
     end
 
+    ##
+    # Add a user as member to a project
+    # ==Parameters:
+    #  [owner] The login of the repository owner
+    #  [repo] The name of the repository
+    #  [new_member] The login of the member to add
+    #  [date_added] The timestamp that the add event took place
+    def get_project_member(owner, repo, new_member, date_added)
+      transaction do
+        ensure_repo(owner, repo)
+        ensure_project_member(owner, repo, new_member, date_added)
+      end
+    end
+
+    ##
+    # Add a commit comment to a commit
+    # ==Parameters:
+    #  [user] The login of the repository owner
+    #  [repo] The name of the repository
+    #  [comment_id] The login of the member to add
+    #  [date_added] The timestamp that the add event took place
+    def get_commit_comment(user, repo, comment_id, date_added)
+      transaction do
+        ensure_repo(user, repo)
+        ensure_commit_comment(user, repo, comment_id, date_added)
+      end
+    end
+
+    ##
+    # Add a watcher to a repository
+    # ==Parameters:
+    #  [owner] The login of the repository owner
+    #  [repo] The name of the repository
+    #  [watcher] The login of the member to add
+    #  [date_added] The timestamp that the add event took place
+    def get_watcher(owner, repo, watcher, date_added)
+      transaction do
+        ensure_repo(owner, repo)
+        ensure_watcher(owner, repo, watcher, date_added)
+      end
+    end
+
+    ##
+    # Add a follower to user
+    # ==Parameters:
+    #  [follower] The login of the repository owner
+    #  [followed] The name of the repository
+    #  [date_added] The timestamp that the add event took place
+    def get_follower(follower, followed, date_added)
+      transaction do
+        ensure_user(follower, false, false)
+        ensure_user(followed, false, false)
+        ensure_user_followers(followed, date_added)
+      end
+    end
+
     ##
     # Make sure a commit exists
-    def ensure_commit(repo, sha, user)
+    #
+    def ensure_commit(repo, sha, user, comments = true)
       c = retrieve_commit(repo, sha, user)
-      store_commit(c, repo, user)
-      ensure_commit_comments(user, repo, sha)
+      stored = store_commit(c, repo, user)
       ensure_parents(c)
-      c
+      ensure_commit_comments(user, repo, sha) if comments
+      stored
     end
 
     ##
@@ -235,6 +259,7 @@ module GHTorrent
       return u
     end
 
+
     ##
     # Ensure that a user exists, or fetch its latest state from Github
     # ==Parameters:
@@ -258,19 +283,49 @@ module GHTorrent
                   end
                 end
 
-        users.insert(:login => u['login'],
-                     :name => u['name'],
-                     :company => u['company'],
-                     :email => email,
-                     :hireable => boolean(u['hirable']),
-                     :bio => u['bio'],
-                     :location => u['location'],
-                     :type =>  user_type(u['type']),
-                     :created_at => date(u['created_at']),
-                     :ext_ref_id => u[@ext_uniq])
-
-        info "GHTorrent: New user #{user}"
+        if not email.nil?
+          # Check whether a user has been added by email before
+          byemail = users.first(:email => email)
+          unless byemail.nil?
+            users.filter(:email => email).update(:login => u['login'],
+                           :name => u['name'],
+                           :company => u['company'],
+                           :hireable => boolean(u['hirable']),
+                           :bio => u['bio'],
+                           :location => u['location'],
+                           :type => user_type(u['type']),
+                           :created_at => date(u['created_at']),
+                           :ext_ref_id => u[@ext_uniq]
+            )
+            info "GHTorrent: Updating user #{user} (email #{email})"
+          else
+            users.insert(:login => u['login'],
+                         :name => u['name'],
+                         :company => u['company'],
+                         :email => email,
+                         :hireable => boolean(u['hirable']),
+                         :bio => u['bio'],
+                         :location => u['location'],
+                         :type => user_type(u['type']),
+                         :created_at => date(u['created_at']),
+                         :ext_ref_id => u[@ext_uniq])
+
+            info "GHTorrent: New user #{user}"
+          end
+        else
+          users.insert(:login => u['login'],
+                       :name => u['name'],
+                       :company => u['company'],
+                       :email => email,
+                       :hireable => boolean(u['hirable']),
+                       :bio => u['bio'],
+                       :location => u['location'],
+                       :type => user_type(u['type']),
+                       :created_at => date(u['created_at']),
+                       :ext_ref_id => u[@ext_uniq])
 
+          info "GHTorrent: New user #{user}"
+        end
         users.first(:login => user)
       else
         debug "GHTorrent: User #{user} exists"
@@ -285,34 +340,42 @@ module GHTorrent
     #
     # ==Parameters:
     # [user]  The user login to find followers by
-    def ensure_user_followers(user, ts = Time.now)
+    def ensure_user_followers(user, date_added = nil)
+      followers = @db[:followers]
+      userid = @db[:users].first(:login => user)[:id]
 
-      followers = retrieve_user_followers(user)
-      followers.each { |f|
+      retrieved = retrieve_user_followers(user)
+      retrieved.each { |f|
         follower = f['login']
         ensure_user(user, false, false)
         ensure_user(follower, false, false)
 
-        userid = @db[:users].select(:id).first(:login => user)[:id]
-        followerid = @db[:users].select(:id).first(:login => follower)[:id]
-        followers = @db[:followers]
+        followerid = @db[:users].first(:login => follower)[:id]
+
 
         if followers.first(:user_id => userid, :follower_id => followerid).nil?
-          @db[:followers].insert(:user_id => userid,
-                                 :follower_id => followerid,
-                                 :created_at => ts,
-                                 :ext_ref_id => f[@ext_uniq]
+          added = if date_added.nil? then Time.now else date_added end
+          followers.insert(:user_id => userid,
+                           :follower_id => followerid,
+                           :created_at => added,
+                           :ext_ref_id => f[@ext_uniq]
           )
           info "GHTorrent: User #{follower} follows #{user}"
         else
-          info "User #{follower} already follows #{user}"
+          unless date_added.nil?
+            followers.filter(:user_id => userid,
+                             :follower_id => followerid).\
+                      update(:created_at => date(date_added))
+            info "GHTorrent: Updated follower #{follower} -> #{user}"
+          end
+          debug "GHTorrent: User #{follower} already follows #{user}"
         end
       }
     end
 
     ##
     # Try to retrieve a user by email. Search the DB first, fall back to
-    # Github API v2 if unsuccessful.
+    # Github search API if unsuccessful.
     #
     # ==Parameters:
     # [email]  The email to lookup the user by
@@ -330,7 +393,7 @@ module GHTorrent
         u = retrieve_user_byemail(email, name)
 
         if u.nil? or u['user'].nil? or u['user']['login'].nil?
-          debug "GHTorrent: Cannot find #{email} through API v2 query"
+          debug "GHTorrent: Cannot find #{email} through search API query"
           users.insert(:email => email,
                        :name => name,
                        :login => (0...8).map { 65.+(rand(25)).chr }.join,
@@ -348,7 +411,7 @@ module GHTorrent
                        :location => u['user']['location'],
                        :created_at => date(u['user']['created_at']),
                        :ext_ref_id => u[@ext_uniq])
-          debug "GHTorrent: Found #{email} through API v2 query"
+          debug "GHTorrent: Found #{email} through search API query"
           users.first(:email => email)
         end
       else
@@ -371,7 +434,8 @@ module GHTorrent
 
       ensure_user(user, true, true)
       repos = @db[:projects]
-      currepo = repos.first(:name => repo)
+      curuser = @db[:users].first(:login => user)
+      currepo = repos.first(:owner_id => curuser[:id], :name => repo)
 
       if currepo.nil?
         r = retrieve_repo(user, repo)
@@ -385,13 +449,63 @@ module GHTorrent
 
         info "GHTorrent: New repo #{repo}"
         ensure_commits(user, repo)
-        repos.first(:name => repo)
+        ensure_project_members(user, repo)
+        ensure_watchers(user, repo)
+        repos.first(:owner_id => curuser[:id], :name => repo)
       else
         debug "GHTorrent: Repo #{repo} exists"
         currepo
       end
     end
 
+    ##
+    # Make sure that a project has all the registered members defined
+    def ensure_project_members(user, repo)
+      curuser = @db[:users].first(:login => user)
+      currepo = @db[:projects].first(:owner_id => curuser[:id], :name => repo)
+      project_members = @db[:project_members].filter(:user_id => curuser[:id],
+                                                   :repo_id => currepo[:id])
+
+      retrieve_repo_collaborators(user, repo).reduce([]) do |acc, x|
+        if project_members.find { |y| y[:login] == x['login'] }.nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x| ensure_project_member(user, repo, x['login'], nil) }
+    end
+
+    ##
+    # Make sure that a project member exists in a project
+    def ensure_project_member(owner, repo, new_member, date_added)
+      pr_members = @db[:project_members]
+      new_user = ensure_user(new_member, false, false)
+      owner_id = @db[:users].first(:login => owner)[:id]
+      project = @db[:projects].first(:owner_id => owner_id, :name => repo)
+
+      memb_exist = pr_members.first(:user_id => new_user[:id],
+                                    :repo_id => project[:id])
+
+      if memb_exist.nil?
+        added = if date_added.nil? then Time.now else date_added end
+        retrieved = retrieve_repo_collaborator(owner, repo, new_member)
+        pr_members.insert(
+            :user_id => new_user[:id],
+            :repo_id => project[:id],
+            :created_at => date(added),
+            :ext_ref_id => retrieved[@ext_uniq]
+        )
+        info "GHTorrent: Added project member #{repo} -> #{new_member}"
+      else
+        unless date_added.nil?
+          pr_members.filter(:user_id => new_user[:id],
+                            :repo_id => project[:id])\
+                    .update(:created_at => date(date_added))
+          info "GHTorrent: Updating  #{repo} -> #{new_member}"
+        end
+      end
+    end
+
     ##
     # Make sure that the organizations the user participates into exist
     #
@@ -404,7 +518,7 @@ module GHTorrent
     end
 
     ##
-    # Make sure that a user belongs to the provided organization
+    # Make sure that a user participates to the provided organization
     #
     # ==Parameters:
     # [user] The login name of the user to check the organizations for
@@ -434,7 +548,7 @@ module GHTorrent
     # Make sure that an organization exists
     #
     # ==Parameters:
-    # [org]  The login name of the organization
+    # [organization]  The login name of the organization
     #
     def ensure_org(organization)
       org = @db[:users].find(:login => organization, :type => 'org')
@@ -447,39 +561,28 @@ module GHTorrent
       end
     end
 
-
     ##
     # Get all comments for a commit
     #
     # ==Parameters:
     # [user]  The login name of the organization
+    # [user]  The repository containing the commit whose comments will be retrieved
+    # [sha]  The commit sha to retrieve comments for
     def ensure_commit_comments(user, repo, sha)
-      commit_id = @db[:commits].first(:sha => sha)
-      stored_comments = @db[:commit_comments].find(:commit_id => commit_id)
-      commit_commets = retrieve_commit_comments(user, repo, sha)
-      user_id = @db[:users].first(:login => user)[:id]
+      commit_id = @db[:commits].first(:sha => sha)[:id]
+      stored_comments = @db[:commit_comments].filter(:commit_id => commit_id)
+      commit_comments = retrieve_commit_comments(user, repo, sha)
+      #user_id = @db[:users].first(:login => user)[:id]
 
-      not_saved = commit_commets.reduce([]) do |acc, x|
-        if stored_comments.find{|y| y[:comment_id] == x['comment_id']}.nil?
+      not_saved = commit_comments.reduce([]) do |acc, x|
+        if stored_comments.find{|y| y[:comment_id] == x['id']}.nil?
           acc << x
         else
           acc
         end
       end
 
-      not_saved.each do |c|
-        @db[:commit_comments].insert(
-          :commit_id => commit_id,
-          :user_id => user_id,
-          :body => c['body'],
-          :line => c['line'],
-          :position => c['position'],
-          :comment_id => c['id'],
-          :ext_ref_id => c['ext_ref_id'],
-          :created_at => date(c['created_at'])
-        )
-        info "GHTorrent: Added commit comment #{sha} -> #{user}"
-      end
+      not_saved.map{|x| ensure_commit_comment(user, repo, x['id'], nil)}
     end
 
     ##
@@ -487,12 +590,21 @@ module GHTorrent
     #
     # ==Parameters:
     # [user]  The login name of the organization
-    def ensure_commit_comment(user, repo, id)
+    # [repo]  The repository containing the commit whose comment will be retrieved
+    # [id]  The comment id to retrieve
+    # [created_at]  The timestamp that the comment was made.
+    def ensure_commit_comment(user, repo, id, created_at)
       stored_comment = @db[:commit_comments].first(:comment_id => id)
 
       if stored_comment.nil?
         retrieved = retrieve_commit_comment(user, repo, id)
-        commit = ensure_commit(repo, retrieved['commit_id'], user)
+
+        if retrieved.nil?
+          debug "GHTorrent: Commit comment #{id} deleted"
+          return
+        end
+
+        commit = ensure_commit(repo, retrieved['commit_id'], user, comments = false)
         user = ensure_user(user, false, false)
         @db[:commit_comments].insert(
             :commit_id => commit[:id],
@@ -501,17 +613,66 @@ module GHTorrent
             :line => retrieved['line'],
             :position => retrieved['position'],
             :comment_id => retrieved['id'],
-            :ext_ref_id => retrieved['ext_ref_id'],
+            :ext_ref_id => retrieved[@ext_uniq],
             :created_at => date(retrieved['created_at'])
         )
+        info "GHTorrent: Added commit comment #{commit[:sha]} -> #{user[:login]}"
         @db[:commit_comments].first(:comment_id => id)
-        info "GHTorrent: Added commit comment #{commit[:sha]} -> #{user}"
       else
         info "GHTorrent: Commit comment #{id} exists"
         stored_comment
       end
     end
 
+    ##
+    # Make sure that
+    def ensure_watchers(owner, repo)
+      curuser = @db[:users].first(:login => owner)
+      currepo = @db[:projects].first(:owner_id => curuser[:id],
+                                     :name => repo)
+      watchers = @db[:watchers].filter(:user_id => curuser[:id],
+                                       :repo_id => currepo[:id])
+
+      retrieve_watchers(owner, repo).reduce([]) do |acc, x|
+        if watchers.find { |y| y[:login] == x['login'] }.nil?
+          acc << x
+        else
+          acc
+        end
+      end.map { |x| ensure_watcher(owner, repo, x['login']) }
+    end
+
+    ##
+    # Make sure that a project member exists in a project
+    def ensure_watcher(owner, repo, watcher, date_added = nil)
+      watchers = @db[:watchers]
+      new_watcher = ensure_user(watcher, false, false)
+      owner_id = @db[:users].first(:login => owner)[:id]
+      project = @db[:projects].first(:owner_id => owner_id, :name => repo)
+
+      memb_exist = watchers.first(:user_id => new_watcher[:id],
+                                    :repo_id => project[:id])
+
+      if memb_exist.nil?
+        added = if date_added.nil? then Time.now else date_added end
+        retrieved = retrieve_watcher(owner, repo, watcher)
+        watchers.insert(
+            :user_id => new_watcher[:id],
+            :repo_id => project[:id],
+            :created_at => date(added),
+            :ext_ref_id => retrieved[@ext_uniq]
+        )
+        info "GHTorrent: Added watcher #{repo} -> #{watcher}"
+      else
+        unless date_added.nil?
+          watchers.filter(:user_id => new_watcher[:id],
+                          :repo_id => project[:id])\
+                  .update(:created_at => date(date_added))
+          info "GHTorrent: Updating  #{repo} -> #{watcher}"
+        end
+      end
+    end
+
     private
 
     # Store a commit contained in a hash. First check whether the commit exists.
@@ -523,7 +684,6 @@ module GHTorrent
         author = commit_user(c['author'], c['commit']['author'])
         commiter = commit_user(c['committer'], c['commit']['committer'])
 
-
         userid = @db[:users].filter(:login => user).first[:id]
         repoid = @db[:projects].filter(:owner_id => userid,
                                        :name => repo).first[:id]
@@ -535,9 +695,11 @@ module GHTorrent
                        :created_at => date(c['commit']['author']['date']),
                        :ext_ref_id => c[@ext_uniq]
         )
+        commits.first(:sha => c['sha'])
         debug "GHTorrent: New commit #{repo} -> #{c['sha']} "
       else
         debug "GHTorrent: Commit #{repo} -> #{c['sha']} exists"
+        commit
       end
     end
 
@@ -569,7 +731,11 @@ module GHTorrent
     # - yyyy-mm-ddThh:mm:ssZ
     # - yyyy/mm/dd hh:mm:ss {+/-}hhmm
     def date(arg)
-      Time.parse(arg)#.to_i
+      if arg.class != Time
+        Time.parse(arg)#.to_i
+      else
+        arg
+      end
     end
 
     def is_valid_email(email)