ghtorrent 0.3.1 → 0.4

data/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2012, Georgios Gousios 
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

data/bin/ght-data-retrieval

@@ -1,33 +1,3 @@
-#!/usr/bin/env ruby
-
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'rubygems'
 require 'amqp'
 require 'json'
@@ -37,6 +7,7 @@ require 'pp'
 class GHTDataRetrieval < GHTorrent::Command
 
   include GHTorrent::Settings
+  include GHTorrent::Logging
 
   attr_reader :settings, :name
 
@@ -54,21 +25,45 @@ class GHTDataRetrieval < GHTorrent::Command
 
   def WatchEvent(evt)
     data = parse evt
-    user = data['actor']['login']
-    #@gh.get_watched user, evt
+    owner = data['repo']['name'].split(/\//)[0]
+    repo = data['repo']['name'].split(/\//)[1]
+    watcher = data['actor']['login']
+    created_at = data['created_at']
+
+    @gh.get_watcher owner, repo, watcher, created_at
   end
 
   def FollowEvent(evt)
+    data = parse evt
+    follower = data['actor']['login']
+    followed = data['payload']['target']['login']
+    created_at = data['created_at']
+
+    @gh.get_follower(follower, followed, created_at)
+  end
+
+  def MemberEvent(evt)
+    data = parse evt
+    owner = data['actor']['login']
+    repo = data['repo']['name'].split(/\//)[1]
+    new_member = data['payload']['member']['login']
+    created_at = data['created_at']
+
+    @gh.get_project_member(owner, repo, new_member, created_at)
+  end
+
+  def CommitCommentEvent(evt)
     data = parse evt
     user = data['actor']['login']
-    #@gh.get_followed user
+    repo = data['repo']['name'].split(/\//)[1]
+    id = data['payload']['comment']['id']
+    created_at = data['created_at']
 
-    followed = data['payload']['target']['login']
-    #@gh.get_followers followed
+    @gh.get_commit_comment(user, repo, id, created_at)
   end
 
   def handlers
-    %w(PushEvent WatchEvent FollowEvent)
+    %w(PushEvent WatchEvent FollowEvent MemberEvent CommitCommentEvent)
   end
 
   def prepare_options(options)
@@ -81,11 +76,11 @@ class GHTDataRetrieval < GHTorrent::Command
 
     # Graceful exit
     Signal.trap('INT') {
-      info ("Received SIGINT, exiting")
+      info "Received SIGINT, exiting"
       AMQP.stop { EM.stop }
     }
     Signal.trap('TERM') {
-      info ("Received SIGTERM, exiting")
+      info "Received SIGTERM, exiting"
       AMQP.stop { EM.stop }
     }
 
@@ -102,7 +97,7 @@ class GHTDataRetrieval < GHTorrent::Command
         queue = channel.queue("#{h}s", {:durable => true})\
                        .bind(exchange, :routing_key => "evt.#{h}")
 
-        puts "Binding handler #{h} to routing key evt.#{h}"
+        info "Binding handler #{h} to routing key evt.#{h}"
 
         queue.subscribe(:ack => true) do |headers, msg|
           begin

data/bin/ght-load

@@ -1,36 +1,3 @@
-#!/usr/bin/env ruby
-#
-# Loads items from Mongo to the queue for further processing
-#
-#
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'rubygems'
 require 'ghtorrent-old'
 require 'mongo'

data/bin/ght-mirror-events

@@ -1,33 +1,3 @@
-#!/usr/bin/env ruby
-#
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'rubygems'
 require 'yaml'
 require 'amqp'

data/bin/ght-rm-dupl

@@ -1,35 +1,3 @@
-#!/usr/bin/env ruby
-#
-# Knows how to remove duplicate entries from various collections.
-#
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'rubygems'
 require 'mongo'
 require 'ghtorrent-old'

data/bin/ght-torrent-index

@@ -1,33 +1,3 @@
-#!/usr/bin/env ruby
-#
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'rubygems'
 require 'erb'
 require 'set'

data/lib/ghtorrent.rb

@@ -1,7 +1,25 @@
-#require 'ghtorrent-old/ghtorrent-old'
-
 module GHTorrent
-  VERSION = '0.3.1'
+  VERSION = '0.4'
+
+  # Route keys used for setting up queues for events, using GHTorrent
+  ROUTEKEY_CREATE = "evt.CreateEvent"
+  ROUTEKEY_DELETE = "evt.DeleteEvent"
+  ROUTEKEY_DOWNLOAD = "evt.DownloadEvent"
+  ROUTEKEY_FOLLOW = "evt.FollowEvent"
+  ROUTEKEY_FORK = "evt.ForkEvent"
+  ROUTEKEY_FORK_APPLY = "evt.ForkApplyEvent"
+  ROUTEKEY_GIST = "evt.GistEvent"
+  ROUTEKEY_GOLLUM = "evt.GollumEvent"
+  ROUTEKEY_ISSUE_COMMENT = "evt.IssueCommentEvent"
+  ROUTEKEY_ISSUES = "evt.IssuesEvent"
+  ROUTEKEY_MEMBER = "evt.MemberEvent"
+  ROUTEKEY_PUBLIC = "evt.PublicEvent"
+  ROUTEKEY_PULL_REQUEST = "evt.PullRequestEvent"
+  ROUTEKEY_PULL_REQUEST_REVIEW_COMMENT = "evt.PullRequestReviewCommentEvent"
+  ROUTEKEY_PUSH = "evt.PushEvent"
+  ROUTEKEY_TEAM_ADD = "evt.TeamAddEvent"
+  ROUTEKEY_WATCH = "evt.WatchEvent"
+
 end
 
 require 'ghtorrent/command'

data/lib/ghtorrent/adapters/base_adapter.rb

@@ -1,37 +1,9 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 module GHTorrent
 
   class BaseAdapter
 
     ENTITIES = [:users, :commits, :followers, :repos, :events, :org_members,
-        :commit_comments
+        :commit_comments, :repo_collaborators, :watchers
     ]
 
     # Stores +data+ into +entity+. Returns a unique key for the stored entry.

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 require 'mongo'
 
 module GHTorrent
@@ -65,10 +37,13 @@ module GHTorrent
           :followers => get_collection("followers"),
           :events => get_collection("events"),
           :org_members => get_collection("org_members"),
-          :commit_comments => get_collection("commit_comments")
+          :commit_comments => get_collection("commit_comments"),
+          :repo_collaborators => get_collection("repo_collaborators"),
+          :watchers => get_collection("watchers")
       }
 
       # Ensure that the necessary indexes exist
+      ensure_index(:events, "id")
       ensure_index(:users, "login")
       ensure_index(:commits, "sha")
       ensure_index(:repos, "name")
@@ -77,9 +52,14 @@ module GHTorrent
       ensure_index(:commit_comments, "repo")
       ensure_index(:commit_comments, "user")
       ensure_index(:commit_comments, "commit_id")
+      ensure_index(:repo_collaborators, "repo")
+      ensure_index(:repo_collaborators, "owner")
+      ensure_index(:repo_collaborators, "login")
+      ensure_index(:watchers, "repo")
+      ensure_index(:watchers, "owner")
+      ensure_index(:watchers, "login")
     end
 
-
     def store(entity, data = {})
       super
       get_entity(entity).insert(data).to_s

data/lib/ghtorrent/adapters/noop_persister.rb

@@ -1,31 +1,3 @@
-# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-#   1. Redistributions of source code must retain the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer.
-#
-#   2. Redistributions in binary form must reproduce the above
-#      copyright notice, this list of conditions and the following
-#      disclaimer in the documentation and/or other materials
-#      provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
 module GHTorrent
 
   # Persister adapter that does not store any data.