fluent-plugin-github-activities 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 890661f8c3f6d0ba720cfe6ffaabfb69d6396c76
-  data.tar.gz: c7db01ecf49764a715a5f325c3cdd1c0f3cd0c91
+  metadata.gz: 86e9e9a0a689ec6d7ad2c9e0f9dcaa05d932ce69
+  data.tar.gz: c7a2502403c7cc773fced9752fa1590fc616cba6
 SHA512:
-  metadata.gz: e59f0b3d9ce18f8ccd38e425523fc4a5a78612d3ffd4e5fa46f6620b541c4ee3c04213199cf37f97ccbb1f283762c6dd14951739bd3355dd0b9c047efb408c3d
-  data.tar.gz: e642b72842ea663e14bb93ce1ef56869787956b0db982dbd1bbc1e09a6ecab79e059392a1bb6b9d6d609d64f7f17ca4a517b7de07d93694dcdff09366ee89a55
+  metadata.gz: a7abc34e2f2bd886d51c46b21c7b304647f58682234f2f5f62f66d811cf8b9450966d3f7b773475c22a0890f2ecac6654b6d2531abc071ecbb09837cc5a82730
+  data.tar.gz: 5515a6b80db99bc93ffdea84a1d62059fc6762c1c84716b57d8efc468fa48cde3816901ad9e2b513643baf2a059b109cd75801991231e35abd5965d686c87fb5

data/README.md

@@ -36,8 +36,43 @@ Notes:
  * Because a "push" activity doesn't include full information of each commit, commits are separately forwarded [commits](https://developer.github.com/v3/git/commits/) as pseudo `commit` activities.
  * All forwarded records have an extra property `$github-activities-related-avatar`.
    It will be useful to get the URI of the avatar image easily, for both activity events and commits.
+ * All forwarded records have an extra property `$github-activities-related-organization-icon`.
+   It will be useful to get the URI of the logo image of the organization easily, for both activity events and commits.
  * Unsupported activities are also forwarded with their raw event type like `StatusEvent`.
 
+## Authentication
+
+The configuration item `access_token` is optional but strongly recommended to be configured, because there is a rate limit: 60requests/hour by default.
+By an authenticated crawler, you can crawl 5000requests/hour (means about 80requests/minute).
+
+A new access token for your instance can be generated by a simple BASIC authentication, like:
+
+~~~
+$ curl -u 'your-account' -d '{"scopes":[],"note":"Sharetary"}' https://api.github.com/authorizations
+Enter host password for user 'your-account':
+{
+  "id": 1234567,
+  "url": "https://api.github.com/authorizations/1234567",
+  "app": {
+    "name": "Sharetary (API)",
+    "url": "https://developer.github.com/v3/oauth_authorizations/",
+    "client_id": "00000000000000000000"
+  },
+  "token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+  "hashed_token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+  "token_last_eight": "xxxxxxxx",
+  "note": "Sharetary",
+  "note_url": null,
+  "created_at": "2015-06-02T03:17:46Z",
+  "updated_at": "2015-06-02T03:17:46Z",
+  "scopes": [
+
+  ],
+  "fingerprint": null
+}
+~~~
+
+Then the value of the `token` field is the access key to be written to the configuration file.
 
 ## Configurations
 
@@ -45,13 +80,8 @@ Notes:
 <source>
   type github-activities
 
-  # Authentication settings.
-  # They are optional but strongly recommended to be configured,
-  # because there is a rate limit: 60requests/hour by default.
-  # By an authenticated crawler, you can crawl 5000requests/hour
-  # (means about 80requests/minute).
-  basic_username your-user-name-of-github
-  basic_password your-password-of-github
+  # Access token for this instance. See also the previous section.
+  access_token access-token-for-this-instance
 
   # Interval seconds for requests. This is `1` by default.
   interval 1

data/fluent-plugin-github-activities.gemspec

@@ -19,7 +19,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-github-activities"
-  spec.version = "0.2.0"
+  spec.version = "0.3.0"
   spec.authors = ["YUKI Hiroshi"]
   spec.email = ["yuki@clear-code.com"]
   spec.summary = "Fluentd plugin to crawl public activities on the GitHub."

data/lib/fluent/plugin/github-activities/crawler.rb

@@ -21,6 +21,7 @@ require "uri"
 require "net/https"
 require "json"
 require "pathname"
+require "time"
 
 require "fluent/plugin/github-activities/safe_file_writer"
 
@@ -36,13 +37,13 @@ module Fluent
       DEFAULT_LAST_EVENT_TIMESTAMP = -1
 
       RELATED_USER_IMAGE_KEY = "$github-activities-related-avatar"
+      RELATED_ORGANIZATION_IMAGE_KEY = "$github-activities-related-organization-logo"
 
       attr_writer :on_emit
       attr_reader :request_queue, :interval_for_next_request
 
       def initialize(options={})
-        @username = options[:username]
-        @password = options[:password]
+        @access_token = options[:access_token]
 
         @watching_users = options[:watching_users] || []
 
@@ -54,14 +55,11 @@ module Fluent
         @pos_file = Pathname(@pos_file) if @pos_file
         load_positions
 
-        @avatars = {}
-
         @request_queue = options[:request_queue] || []
 
         @default_interval = options[:default_interval] || DEFAULT_INTERVAL
 
         @watching_users.each do |user|
-          fetch_avatar(user)
           reserve_user_events(user)
         end
       end
@@ -70,7 +68,7 @@ module Fluent
         raise EmptyRequestQueue.new if @request_queue.empty?
 
         request = @request_queue.shift
-        $log.info("GithubActivities::Crawler: processing request: #{request.inspect}")
+        $log.info("GithubActivities::Crawler: processing request: #{request.inspect}") if $log
         if request[:process_after] and
              Time.now.to_i < request[:process_after]
           @request_queue.push(request)
@@ -81,18 +79,18 @@ module Fluent
         uri = request_uri(request)
         extra_headers = extra_request_headers(request)
 
-        $log.info("GithubActivities::Crawler: requesting to #{uri.inspect}")
+        $log.info("GithubActivities::Crawler: requesting to #{uri.inspect}") if $log
         response = http_get(uri, extra_headers)
-        $log.info("GithubActivities::Crawler: response: #{response.inspect}")
+        $log.info("GithubActivities::Crawler: response: #{response.inspect}") if $log
 
         case response
         when Net::HTTPSuccess
           body = JSON.parse(response.body)
-          $log.info("GithubActivities::Crawler: request type: #{request[:type]}")
+          $log.info("GithubActivities::Crawler: request type: #{request[:type]}") if $log
           case request[:type]
           when TYPE_EVENTS
             events = body
-            $log.info("GithubActivities::Crawler: events size: #{events.size}")
+            $log.info("GithubActivities::Crawler: events size: #{events.size}") if $log
             process_user_events(request[:user], events)
             reserve_user_events(request[:user], :previous_response => response)
             save_user_position(request[:user], :entity_tag => response["ETag"])
@@ -108,6 +106,15 @@ module Fluent
           end
           @interval_for_next_request = NO_INTERVAL
           return true
+        when Net::HTTPNotFound
+          case request[:type]
+          when TYPE_COMMIT
+            fake_body = {
+              "sha"    => request[:sha],
+              "author" => {},
+            }
+            process_commit(fake_body, request[:push])
+          end
         end
         @interval_for_next_request = @default_interval
         return true
@@ -169,7 +176,10 @@ module Fluent
 
       def process_user_event(user, event)
         # see also: https://developer.github.com/v3/activity/events/types/
-        event[RELATED_USER_IMAGE_KEY] = @avatars[user]
+        event[RELATED_USER_IMAGE_KEY] = event["actor"]["avatar_url"]
+        if event["org"]
+          event[RELATED_ORGANIZATION_IMAGE_KEY] = event["org"]["avatar_url"]
+        end
         case event["type"]
         when "PushEvent"
           process_push_event(event)
@@ -200,23 +210,27 @@ module Fluent
         commit_refs.reverse.each do |commit_ref|
           @request_queue.unshift(:type => TYPE_COMMIT,
                                  :uri  => commit_ref["url"],
+                                 :sha  => commit_ref["sha"],
                                  :push => event)
         end
         # emit("push", event)
       end
 
       def process_commit(commit, push_event)
+        $log.info("GithubActivities::Crawler: processing commit #{commit["sha"]}") if $log
         user = commit["author"]["login"]
-        fetch_avatar(user)
 
-        if @include_foreign_commits or watching_user?(user)
-          commit[RELATED_USER_IMAGE_KEY] = @avatars[user]
+        if user and (@include_foreign_commits or watching_user?(user))
+          commit[RELATED_USER_IMAGE_KEY] = push_event["actor"]["avatar_url"]
+          if push_event["org"]
+            commit[RELATED_ORGANIZATION_IMAGE_KEY] = push_event["org"]["avatar_url"]
+          end
           emit("commit", commit)
         end
 
         commit_refs = push_event["payload"]["commits"]
         target_commit_ref = commit_refs.find do |commit_ref|
-          commit_ref["url"] == commit["url"]
+          commit_ref["sha"] == commit["sha"]
         end
         target_commit_ref["commit"] = commit if target_commit_ref
 
@@ -227,7 +241,7 @@ module Fluent
       end
 
       def watching_user?(user)
-        @watching_users.include(user)
+        @watching_users.include?(user)
       end
 
       def process_issue_event(event)
@@ -299,13 +313,6 @@ module Fluent
         end
       end
 
-      def fetch_avatar(user)
-        return if @avatars.key?(user)
-        response = http_get(user_info(user))
-        fetched_user_info = JSON.parse(response.body)
-        @avatars[user] = fetched_user_info["avatar_url"]
-      end
-
       private
       def user_activities(user)
         "https://api.github.com/users/#{user}/events/public"
@@ -316,7 +323,7 @@ module Fluent
       end
 
       def emit(tag, record)
-        $log.trace("GithubActivities::Crawler: emit => #{tag}, #{record.inspect}")
+        $log.trace("GithubActivities::Crawler: emit => #{tag}, #{record.inspect}") if $log
         @on_emit.call(tag, record) if @on_emit
       end
 
@@ -327,8 +334,8 @@ module Fluent
         http.use_ssl = parsed_uri.is_a?(URI::HTTPS)
         http.start do |http|
           http_request = Net::HTTP::Get.new(parsed_uri.path, extra_headers)
-          if @username and @password
-            http_request.basic_auth(@username, @password)
+          if @access_token
+            extra_headers["Authorization"] = "token #{@access_token}"
           end
           response = http.request(http_request)
         end

data/lib/fluent/plugin/in_github-activities.rb

@@ -23,8 +23,7 @@ module Fluent
 
     Plugin.register_input("github-activities", self)
 
-    config_param :basic_username, :string, :default => nil
-    config_param :basic_password, :string, :default => nil
+    config_param :access_token, :string, :default => nil
     config_param :users, :string, :default => nil
     config_param :users_list, :string, :default => nil
     config_param :include_commits_from_pull_request, :bool, :default => false
@@ -45,8 +44,7 @@ module Fluent
       @base_tag = @base_tag.sub(/\.\z/, "")
       @thread = Thread.new do
         crawler_options = {
-          :username => @basic_username,
-          :password => @basic_password,
+          :access_token => @access_token,
           :watching_users => prepare_users_list,
           :include_commits_from_pull_request => @include_commits_from_pull_request,
           :include_foreign_commits => @include_foreign_commits,

data/test/fixture/push-event.json

@@ -0,0 +1,45 @@
+{
+  "id": "2823041920",
+  "type": "PushEvent",
+  "actor": {
+    "id": 70062,
+    "login": "piroor",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/piroor",
+    "avatar_url": "https://avatars.githubusercontent.com/u/70062?"
+  },
+  "repo": {
+    "id": 35922279,
+    "name": "clear-code/fluent-plugin-github-activities",
+    "url": "https://api.github.com/repos/clear-code/fluent-plugin-github-activities"
+  },
+  "payload": {
+    "push_id": 670478406,
+    "size": 1,
+    "distinct_size": 1,
+    "ref": "refs/heads/master",
+    "head": "c908f319c7b6d5c5a69c8b675bde40dd990ee364",
+    "before": "e77acc068b5568f8ea55605db2e3a9005805c898",
+    "commits": [
+      {
+        "sha": "8e90721ff5d89f52b5b3adf0b86db01f03dc5588",
+        "author": {
+          "email": "shimoda@clear-code.com",
+          "name": "YUKI Hiroshi"
+        },
+        "message": "Add missing constant",
+        "distinct": true,
+        "url": "https://api.github.com/repos/clear-code/fluent-plugin-github-activities/commits/8e90721ff5d89f52b5b3adf0b86db01f03dc5588"
+      }
+    ]
+  },
+  "public": true,
+  "created_at": "2015-05-21T05:37:34Z",
+  "org": {
+    "id": 176515,
+    "login": "clear-code",
+    "gravatar_id": "",
+    "url": "https://api.github.com/orgs/clear-code",
+    "avatar_url": "https://avatars.githubusercontent.com/u/176515?"
+  }
+}

data/test/test_crawler.rb

@@ -25,11 +25,38 @@ class CrawlerTest < Test::Unit::TestCase
   def setup
     @emitted_records = []
 
-    @crawler = ::Fluent::GithubActivities::Crawler.new
+    @crawler = ::Fluent::GithubActivities::Crawler.new(crawler_options)
     @crawler.on_emit = lambda do |tag, record|
       @emitted_records << { :tag    => tag,
                             :record => record }
     end
+    @crawler.request_queue.clear
+  end
+
+  def crawler_options
+    {
+      :include_commits_from_pull_request => false,
+      :include_foreign_commits => false,
+      :watching_users => [
+        'piroor',
+      ],
+    }
+  end
+
+  def fill_extra_fields(event, parent_event=nil)
+    parent_event ||= event
+
+    event = event.merge(
+      "$github-activities-related-avatar" => parent_event["actor"]["avatar_url"],
+    )
+
+    if parent_event["org"]
+      event = event.merge(
+        "$github-activities-related-organization-logo" => parent_event["org"]["avatar_url"],
+      )
+    end
+
+    event
   end
 
   REQUEST_PATTERNS = {
@@ -54,7 +81,7 @@ class CrawlerTest < Test::Unit::TestCase
   data(REQUEST_PATTERNS)
   def test_request_uri(data)
     uri = @crawler.request_uri(data[:request])
-    assert_equal(URI(data[:uri]), uri)
+    assert_equal(data[:uri], uri)
   end
 
   data(REQUEST_PATTERNS)
@@ -97,12 +124,17 @@ class CrawlerTest < Test::Unit::TestCase
 
   class UserEventTest < self
     def test_generic
-      @crawler.process_user_event("username", { "type" => "test" })
+      event = {
+        "type" => "test",
+        "actor" => {},
+        "created_at" => "2015-05-21T05:37:34Z",
+      }
+      @crawler.process_user_event("username", event)
       expected = {
         :request_queue => [],
         :emitted_records => [
           { :tag    => "test",
-            :record => { "type" => "test" } },
+            :record => fill_extra_fields(event) },
         ],
       }
       assert_equal(expected,
@@ -113,12 +145,17 @@ class CrawlerTest < Test::Unit::TestCase
 
   class UserEventsTest < self
     def test_generic
-      @crawler.process_user_events("username", [{ "type" => "test" }])
+      event = {
+        "type" => "test",
+        "actor" => {},
+        "created_at" => "2015-05-21T05:37:34Z",
+      }
+      @crawler.process_user_events("username", [event])
       expected = {
         :request_queue => [],
         :emitted_records => [
           { :tag    => "test",
-            :record => { "type" => "test" } },
+            :record => fill_extra_fields(event) },
         ],
       }
       assert_equal(expected,
@@ -128,26 +165,69 @@ class CrawlerTest < Test::Unit::TestCase
   end
 
   class PushEventTest < self
+    def test_single_commit
+      push = JSON.parse(fixture_data("push-event.json"))
+      push = fill_extra_fields(push)
+      base = "https://api.github.com/repos/clear-code/fluent-plugin-github-activities/commits"
+      @crawler.process_user_event("user", push)
+
+      expected_commit = JSON.parse(fixture_data("commit.json"))
+      expected_push = JSON.parse(fixture_data("push-event.json"))
+
+      expected_commit = fill_extra_fields(expected_commit, expected_push)
+
+      expected_push = fill_extra_fields(expected_push)
+      expected = {
+        :request_queue => [
+          { :type => ::Fluent::GithubActivities::TYPE_COMMIT,
+            :uri  => "#{base}/8e90721ff5d89f52b5b3adf0b86db01f03dc5588",
+            :sha  => "8e90721ff5d89f52b5b3adf0b86db01f03dc5588",
+            :push => expected_push },
+        ],
+        :emitted_records => [
+        ],
+      }
+      assert_equal(expected,
+                   { :request_queue   => @crawler.request_queue,
+                     :emitted_records => @emitted_records })
+    end
+
     def test_multiple_commits
-      event = JSON.parse(fixture_data("push-event-multiple-commits.json"))
+      push = JSON.parse(fixture_data("push-event-multiple-commits.json"))
+      push = fill_extra_fields(push)
       base = "https://api.github.com/repos/clear-code/fluent-plugin-github-activities/commits"
-      @crawler.process_user_event("user", event)
+      @crawler.process_user_event("user", push)
+
+      expected_commit = JSON.parse(fixture_data("commit.json"))
+      expected_push = JSON.parse(fixture_data("push-event-multiple-commits.json"))
+
+      expected_commit = fill_extra_fields(expected_commit, expected_push)
+
+      expected_push = fill_extra_fields(expected_push)
       expected = {
         :request_queue => [
           { :type => ::Fluent::GithubActivities::TYPE_COMMIT,
-            :uri  => "#{base}/8e90721ff5d89f52b5b3adf0b86db01f03dc5588"},
+            :uri  => "#{base}/8e90721ff5d89f52b5b3adf0b86db01f03dc5588",
+            :sha  => "8e90721ff5d89f52b5b3adf0b86db01f03dc5588",
+            :push => expected_push },
           { :type => ::Fluent::GithubActivities::TYPE_COMMIT,
-            :uri  => "#{base}/63e085b7607a3043cfbf9a866561807fbdda8a10"},
+            :uri  => "#{base}/63e085b7607a3043cfbf9a866561807fbdda8a10",
+            :sha  => "63e085b7607a3043cfbf9a866561807fbdda8a10",
+            :push => expected_push },
           { :type => ::Fluent::GithubActivities::TYPE_COMMIT,
-            :uri  => "#{base}/c85e33bace040b7b42983e14d2b11a491d102072"},
+            :uri  => "#{base}/c85e33bace040b7b42983e14d2b11a491d102072",
+            :sha  => "c85e33bace040b7b42983e14d2b11a491d102072",
+            :push => expected_push },
           { :type => ::Fluent::GithubActivities::TYPE_COMMIT,
-            :uri  => "#{base}/8ce6de7582376187e17e233dbae13575311a8c0b"},
+            :uri  => "#{base}/8ce6de7582376187e17e233dbae13575311a8c0b",
+            :sha  => "8ce6de7582376187e17e233dbae13575311a8c0b",
+            :push => expected_push },
           { :type => ::Fluent::GithubActivities::TYPE_COMMIT,
-            :uri  => "#{base}/c908f319c7b6d5c5a69c8b675bde40dd990ee364"},
+            :uri  => "#{base}/c908f319c7b6d5c5a69c8b675bde40dd990ee364",
+            :sha  => "c908f319c7b6d5c5a69c8b675bde40dd990ee364",
+            :push => expected_push },
         ],
         :emitted_records => [
-          { :tag    => "push",
-            :record => event }
         ],
       }
       assert_equal(expected,
@@ -157,14 +237,14 @@ class CrawlerTest < Test::Unit::TestCase
   end
 
   class IssuesEventTest < self
-    def test_multiple_commits
+    def test_issue_open
       event = JSON.parse(fixture_data("issues-event.json"))
       @crawler.process_user_event("user", event)
       expected = {
         :request_queue => [],
         :emitted_records => [
-          { :tag    => "issues",
-            :record => event }
+          { :tag    => "issue-open",
+            :record => fill_extra_fields(event) }
         ],
       }
       assert_equal(expected,
@@ -174,14 +254,14 @@ class CrawlerTest < Test::Unit::TestCase
   end
 
   class IssueCommentEventTest < self
-    def test_multiple_commits
+    def test_issue_comment
       event = JSON.parse(fixture_data("issue-comment-event.json"))
       @crawler.process_user_event("user", event)
       expected = {
         :request_queue => [],
         :emitted_records => [
           { :tag    => "issue-comment",
-            :record => event }
+            :record => fill_extra_fields(event) }
         ],
       }
       assert_equal(expected,
@@ -198,7 +278,7 @@ class CrawlerTest < Test::Unit::TestCase
         :request_queue => [],
         :emitted_records => [
           { :tag    => "commit-comment",
-            :record => event }
+            :record => fill_extra_fields(event) }
         ],
       }
       assert_equal(expected,
@@ -215,7 +295,7 @@ class CrawlerTest < Test::Unit::TestCase
         :request_queue => [],
         :emitted_records => [
           { :tag    => "fork",
-            :record => event }
+            :record => fill_extra_fields(event) }
         ],
       }
       assert_equal(expected,
@@ -232,7 +312,7 @@ class CrawlerTest < Test::Unit::TestCase
         :request_queue => [],
         :emitted_records => [
           { :tag    => "pull-request",
-            :record => event }
+            :record => fill_extra_fields(event) }
         ],
       }
       assert_equal(expected,
@@ -242,14 +322,29 @@ class CrawlerTest < Test::Unit::TestCase
   end
 
   class CommitTest < self
-    def test_multiple_commits
+    def test_commit
       commit = JSON.parse(fixture_data("commit.json"))
-      @crawler.process_commit(commit)
+      push = JSON.parse(fixture_data("push-event.json"))
+      push = fill_extra_fields(push)
+      @crawler.process_commit(commit, push)
+
+      expected_commit = JSON.parse(fixture_data("commit.json"))
+      expected_push = JSON.parse(fixture_data("push-event.json"))
+
+      expected_commit = fill_extra_fields(expected_commit, expected_push)
+
+      expected_push["payload"]["commits"].each do |commit|
+        commit["commit"] = expected_commit
+      end
+      expected_push = fill_extra_fields(expected_push)
+
       expected = {
         :request_queue => [],
         :emitted_records => [
           { :tag    => "commit",
-            :record => commit }
+            :record =>  expected_commit },
+          { :tag    => "push",
+            :record => expected_push }
         ],
       }
       assert_equal(expected,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-github-activities
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - YUKI Hiroshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-29 00:00:00.000000000 Z
+date: 2015-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,6 +121,7 @@ files:
 - test/fixture/pull-request-comment-event.json
 - test/fixture/pull-request-event.json
 - test/fixture/push-event-multiple-commits.json
+- test/fixture/push-event.json
 - test/run-test.rb
 - test/test_crawler.rb
 homepage: https://github.com/groonga/fluent-plugin-groonga
@@ -153,6 +154,7 @@ test_files:
 - test/run-test.rb
 - test/fixture/pull-request-event.json
 - test/fixture/push-event-multiple-commits.json
+- test/fixture/push-event.json
 - test/fixture/issue-comment-event.json
 - test/fixture/fork-event.json
 - test/fixture/line-note-event.json