fluent-plugin-github-activities 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5dd3655e789cb8a28e01800ea4830c625e51448e
-  data.tar.gz: b1340b9cdbf980e093042c4a9337fe3d4ad17e14
+  metadata.gz: 5566e97e3de09ccc09711e64cf7fae99072925f5
+  data.tar.gz: b114642700ecda10abe35598ce12bbc988879176
 SHA512:
-  metadata.gz: cba2954b4c6014fc8623d7303a53b890f1d809ebcd8b1b52a24fd9978be6bdc06e806cf31f1a7bc977dcfa07965f42c8f0b6bfe9e85551ff662666fd2600ac55
-  data.tar.gz: 6d21cf6f0243a5b0d4f184b45eee9ce926a2e48902f53c4ab96580c65789c13d794f04d235bde8f5fae36907aac2b6776d627ad363d0209185e516f468bd133a
+  metadata.gz: 1f2a08171cf6e4dbf37163a41d1d77185e60123377b3bfcda9f75a53e203b29eee3d6f30d50da607c6fda7dde13162028ab01ef14bf4871f5738fa1a161b7dd1
+  data.tar.gz: f85cc6dc814b6349c1f86d8ecd74b1f30c484aed23bb9000a011cbb43ab0e58c3af6d7d47ade7ce705a958cce6bd5735c2ea61508324491bfec5f2ef057305c7

data/README.md

@@ -6,6 +6,29 @@
 Provides ability to watch public activities on GitHub.
 This crawls GitHub activities of specified users and forward each activity as a record.
 
+## Requirements
+
+| fluent-plugin-github-activities | fluentd    | ruby   |
+|---------------------------------|------------|--------|
+| >= 0.7.0                        | >= v0.14.0 | >= 2.1 |
+| < 0.7.0                         | >= v0.12.0 | >= 1.9 |
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-github-activities'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-github-activities
+
 ## Supported activity types
 
  * Activities related to commits
@@ -94,11 +117,6 @@ Then the value of the `token` field is the access key to be written to the confi
   # Number of clients. This is `4` by default.
   clients 1
 
-  # Path to a file to store timestamp of last crawled activity
-  # for each user. If you don't specify this option, same records
-  # can be forwarded after the fluentd is restarted.
-  pos_file /tmp/github-activities.json
-
   # Base tag of forwarded records. It will be used as
   # <base_tag>.<activity type>, like: "github-activity.push",
   # "github-activity.StatusEvent", etc.
@@ -123,3 +141,20 @@ Then the value of the `token` field is the access key to be written to the confi
   #include_foreign_commits true
 </source>
 ~~~
+
+### Breaking changes
+
+`pos_file` is obsoleted since 0.7.0. Use storage instead as following:
+
+```
+<source>
+  @type github-activities
+  <storage>
+    @type local
+    persistent true
+    # or if you want to take over old pos_file,
+    # path parameter must be end with ".json".
+    # path /path/to/pos_file.json
+  </storage>
+</source>
+```

data/fluent-plugin-github-activities.gemspec

@@ -19,7 +19,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-github-activities"
-  spec.version = "0.6.1"
+  spec.version = "0.7.0"
   spec.authors = ["YUKI Hiroshi"]
   spec.email = ["yuki@clear-code.com"]
   spec.summary = "Fluentd plugin to crawl public activities on the GitHub."
@@ -33,11 +33,12 @@ Gem::Specification.new do |spec|
   spec.test_files += Dir.glob("test/**/*")
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency("fluentd", [">= 0.12.0", "< 0.14.0"])
+  spec.add_runtime_dependency("fluentd", [">= 0.14.13", "< 2"])
 
   spec.add_development_dependency("rake")
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("packnga", ">= 1.0.1")
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("test-unit-notify")
+  spec.add_development_dependency("webmock")
 end

data/lib/fluent/plugin/github-activities.rb

@@ -21,8 +21,10 @@ require "fluent/plugin/github-activities/users_manager"
 require "fluent/plugin/github-activities/crawler"
 
 module Fluent
-  module GithubActivities
-    TYPE_EVENTS = :events
-    TYPE_COMMIT = :commit
+  module Plugin
+    module GithubActivities
+      TYPE_EVENTS = :events
+      TYPE_COMMIT = :commit
+    end
   end
 end

data/lib/fluent/plugin/github-activities/crawler.rb

@@ -25,311 +25,322 @@ require "time"
 require "fluent/plugin/github-activities/users_manager"
 
 module Fluent
-  module GithubActivities
-    class Crawler
-      class EmptyRequestQueue < StandardError
-      end
+  module Plugin
+    module GithubActivities
+      class Crawler
+        class EmptyRequestQueue < StandardError
+        end
 
-      NO_INTERVAL = 0
-      DEFAULT_INTERVAL = 1
+        NO_INTERVAL = 0
+        DEFAULT_INTERVAL = 1
 
-      RELATED_USER_IMAGE_KEY = "$github-activities-related-avatar"
-      RELATED_ORGANIZATION_IMAGE_KEY = "$github-activities-related-organization-logo"
-      RELATED_EVENT = "$github-activities-related-event"
+        RELATED_USER_IMAGE_KEY = "$github-activities-related-avatar"
+        RELATED_ORGANIZATION_IMAGE_KEY = "$github-activities-related-organization-logo"
+        RELATED_EVENT = "$github-activities-related-event"
 
-      attr_writer :on_emit
-      attr_reader :request_queue, :interval_for_next_request
+        attr_writer :on_emit
+        attr_reader :request_queue, :interval_for_next_request, :log, :running
 
-      def initialize(options={})
-        @users_manager = UsersManager.new(:users    => options[:watching_users],
-                                          :pos_file => options[:pos_file])
+        def initialize(options={})
+          @users_manager = UsersManager.new(users: options[:watching_users],
+                                            pos_storage: options[:pos_storage])
 
-        @access_token = options[:access_token]
+          @access_token = options[:access_token]
 
-        @watching_users = options[:watching_users] || []
+          @watching_users = options[:watching_users] || []
 
-        @include_commits_from_pull_request = options[:include_commits_from_pull_request]
-        @include_foreign_commits = options[:include_foreign_commits]
+          @include_commits_from_pull_request = options[:include_commits_from_pull_request]
+          @include_foreign_commits = options[:include_foreign_commits]
 
-        @request_queue = options[:request_queue] || []
+          @request_queue = options[:request_queue] || []
 
-        @default_interval = options[:default_interval] || DEFAULT_INTERVAL
-      end
-
-      def process_request
-        request = @request_queue.shift
-        $log.debug("GithubActivities::Crawler: processing request: #{request.inspect}") if $log
-        if request[:process_after] and
-             Time.now.to_i < request[:process_after]
-          @request_queue.push(request)
-          @interval_for_next_request = NO_INTERVAL
-          return false
+          @default_interval = options[:default_interval] || DEFAULT_INTERVAL
+          @interval_for_next_request = @default_interval
+          # Fluent::PluginLogger instance
+          @log = options[:log]
+          @running = true
         end
 
-        uri = request_uri(request)
-        extra_headers = extra_request_headers(request)
-
-        $log.debug("GithubActivities::Crawler: requesting to #{uri.inspect}") if $log
-        response = http_get(uri, extra_headers)
-        $log.debug("GithubActivities::Crawler: response: #{response.inspect}") if $log
-
-        case response
-        when Net::HTTPSuccess
-          $log.trace("GithubActivities::Crawler: Net::HTTPSuccess / request type: #{request[:type]}") if $log
-          body = JSON.parse(response.body)
-          case request[:type]
-          when TYPE_EVENTS
-            events = body
-            $log.trace("GithubActivities::Crawler: events size: #{events.size}") if $log
-            process_user_events(request[:user], events)
-            reserve_user_events(request[:user], :previous_response => response)
-            @users_manager.save_position_for(request[:user], :entity_tag => response["ETag"])
-          when TYPE_COMMIT
-            process_commit(body, request[:push])
+        def process_request
+          request = @request_queue.shift
+          log.debug("GithubActivities::Crawler: processing request: #{request.inspect}")
+          if request[:process_after] and
+            Time.now.to_i < request[:process_after]
+            @request_queue.push(request)
+            @interval_for_next_request = NO_INTERVAL
+            return false
           end
-        when Net::HTTPNotModified
-          $log.trace("GithubActivities::Crawler: Net::HTTPNotModified / request type: #{request[:type]}") if $log
-          case request[:type]
-          when TYPE_EVENTS
-            reserve_user_events(request[:user],
-                                :previous_response => response,
-                                :previous_entity_tag => extra_headers["If-None-Match"])
+
+          uri = request_uri(request)
+          extra_headers = extra_request_headers(request)
+
+          log.debug("GithubActivities::Crawler: requesting to #{uri.inspect}")
+          response = http_get(uri, extra_headers)
+          log.debug("GithubActivities::Crawler: response: #{response.inspect}")
+
+          case response
+          when Net::HTTPSuccess
+            log.trace("GithubActivities::Crawler: Net::HTTPSuccess / request type: #{request[:type]}")
+            body = JSON.parse(response.body)
+            case request[:type]
+            when TYPE_EVENTS
+              events = body
+              log.trace("GithubActivities::Crawler: events size: #{events.size}")
+              process_user_events(request[:user], events)
+              reserve_user_events(request[:user], previous_response: response)
+              @users_manager.save_position_for(request[:user], entity_tag: response["ETag"])
+            when TYPE_COMMIT
+              process_commit(body, request[:push])
+            end
+          when Net::HTTPNotModified
+            log.trace("GithubActivities::Crawler: Net::HTTPNotModified / request type: #{request[:type]}")
+            case request[:type]
+            when TYPE_EVENTS
+              reserve_user_events(request[:user],
+                                  previous_response: response,
+                                  previous_entity_tag: extra_headers["If-None-Match"])
+            end
+            @interval_for_next_request = @default_interval
+            return true
+          else
+            log.trace("GithubActivities::Crawler: UnknownType / request type: #{request[:type]}")
+            case request[:type]
+            when TYPE_COMMIT
+              fake_body = {
+                "sha"    => request[:sha],
+                "author" => {},
+              }
+              process_commit(fake_body, request[:push])
+            end
           end
           @interval_for_next_request = @default_interval
           return true
-        else
-          $log.trace("GithubActivities::Crawler: UnknownType / request type: #{request[:type]}") if $log
-          case request[:type]
-          when TYPE_COMMIT
-            fake_body = {
-              "sha"    => request[:sha],
-              "author" => {},
-            }
-            process_commit(fake_body, request[:push])
-          end
+        rescue StandardError => error
+          log.error(error.inspect)
         end
-        @interval_for_next_request = @default_interval
-        return true
-      rescue StandardError => error
-        $log.error(error.inspect)
-      end
 
-      def request_uri(request)
-        uri = nil
-        case request[:type]
-        when TYPE_EVENTS
-          uri = user_activities(request[:user])
-        else
-          uri = request[:uri]
+        def request_uri(request)
+          uri = nil
+          case request[:type]
+          when TYPE_EVENTS
+            uri = user_activities(request[:user])
+          else
+            uri = request[:uri]
+          end
         end
-      end
 
-      def extra_request_headers(request)
-        headers = {}
-        if request[:previous_entity_tag]
-          headers["If-None-Match"] = request[:previous_entity_tag]
-        elsif request[:type] == TYPE_EVENTS
-          position = @users_manager.position_for(request[:user])
-          if position
-            entity_tag = position["entity_tag"]
-            headers["If-None-Match"] = entity_tag if entity_tag
+        def extra_request_headers(request)
+          headers = {}
+          if request[:previous_entity_tag]
+            headers["If-None-Match"] = request[:previous_entity_tag]
+          elsif request[:type] == TYPE_EVENTS
+            position = @users_manager.position_for(request[:user])
+            if position
+              entity_tag = position["entity_tag"]
+              headers["If-None-Match"] = entity_tag if entity_tag
+            end
           end
+          headers
         end
-        headers
-      end
-
-      def reserve_user_events(user, options={})
-        request = @users_manager.new_events_request(user, options)
-        @request_queue.push(request)
-      end
 
-      def process_user_events(user, events)
-        last_event_timestamp = UsersManager::DEFAULT_LAST_EVENT_TIMESTAMP
-        position = @users_manager.position_for(user)
-        if position and position["last_event_timestamp"]
-          last_event_timestamp = position["last_event_timestamp"]
+        def reserve_user_events(user, options={})
+          request = @users_manager.new_events_request(user, options)
+          @request_queue.push(request) if @running
         end
 
-        events = events.sort do |a, b|
-          b["created_at"] <=> a["created_at"]
-        end
-        events.each do |event|
-          timestamp = Time.parse(event["created_at"]).to_i
-          next if timestamp <= last_event_timestamp
-          process_user_event(user, event)
-          @users_manager.save_position_for(user, :last_event_timestamp => timestamp)
-        end
-      end
+        def process_user_events(user, events)
+          last_event_timestamp = UsersManager::DEFAULT_LAST_EVENT_TIMESTAMP
+          position = @users_manager.position_for(user)
+          if position and position["last_event_timestamp"]
+            last_event_timestamp = position["last_event_timestamp"]
+          end
 
-      def process_user_event(user, event)
-        # see also: https://developer.github.com/v3/activity/events/types/
-        event[RELATED_USER_IMAGE_KEY] = event["actor"]["avatar_url"]
-        if event["org"]
-          event[RELATED_ORGANIZATION_IMAGE_KEY] = event["org"]["avatar_url"]
+          events = events.sort do |a, b|
+            b["created_at"] <=> a["created_at"]
+          end
+          events.each do |event|
+            timestamp = Time.parse(event["created_at"]).to_i
+            next if timestamp <= last_event_timestamp
+            process_user_event(user, event)
+            @users_manager.save_position_for(user, last_event_timestamp: timestamp)
+          end
         end
-        case event["type"]
-        when "PushEvent"
-          process_push_event(event)
-        when "CommitCommentEvent"
-          emit("commit-comment", event)
-        when "IssuesEvent"
-          process_issue_event(event)
-        when "IssueCommentEvent"
-          process_issue_or_pull_request_comment_event(event)
-        when "ForkEvent"
-          emit("fork", event)
-        when "PullRequestEvent"
-          process_pull_request_event(event)
-        when "CreateEvent"
-          process_create_event(event)
-        else
-          emit(event["type"], event)
+
+        def process_user_event(user, event)
+          # see also: https://developer.github.com/v3/activity/events/types/
+          event[RELATED_USER_IMAGE_KEY] = event["actor"]["avatar_url"]
+          if event["org"]
+            event[RELATED_ORGANIZATION_IMAGE_KEY] = event["org"]["avatar_url"]
+          end
+          case event["type"]
+          when "PushEvent"
+            process_push_event(event)
+          when "CommitCommentEvent"
+            emit("commit-comment", event)
+          when "IssuesEvent"
+            process_issue_event(event)
+          when "IssueCommentEvent"
+            process_issue_or_pull_request_comment_event(event)
+          when "ForkEvent"
+            emit("fork", event)
+          when "PullRequestEvent"
+            process_pull_request_event(event)
+          when "CreateEvent"
+            process_create_event(event)
+          else
+            emit(event["type"], event)
+          end
+        rescue StandardError => error
+          log.fatal(error)
         end
-      rescue StandardError => error
-        $log.exception(error)
-      end
 
-      def process_push_event(event)
-        payload = event["payload"]
-        commit_refs = payload["commits"]
-        if !@include_commits_from_pull_request and
+        def process_push_event(event)
+          return unless @running
+          payload = event["payload"]
+          commit_refs = payload["commits"]
+          if !@include_commits_from_pull_request and
              push_event_from_merged_pull_request?(event)
-          return
-        end
-        commit_refs.reverse.each do |commit_ref|
-          @request_queue.push(:type => TYPE_COMMIT,
-                              :uri  => commit_ref["url"],
-                              :sha  => commit_ref["sha"],
-                              :push => event)
+            return
+          end
+          commit_refs.reverse.each do |commit_ref|
+            @request_queue.push(type: TYPE_COMMIT,
+                                uri: commit_ref["url"],
+                                sha: commit_ref["sha"],
+                                push: event)
+          end
+          # emit("push", event)
         end
-        # emit("push", event)
-      end
 
-      def process_commit(commit, push_event)
-        $log.debug("GithubActivities::Crawler: processing commit #{commit["sha"]}") if $log
-        user = commit["author"]["login"]
+        def process_commit(commit, push_event)
+          log.debug("GithubActivities::Crawler: processing commit #{commit["sha"]}")
+          user = commit["author"]["login"]
+
+          if user and (@include_foreign_commits or watching_user?(user))
+            commit[RELATED_USER_IMAGE_KEY] = push_event["actor"]["avatar_url"]
+            if push_event["org"]
+              commit[RELATED_ORGANIZATION_IMAGE_KEY] = push_event["org"]["avatar_url"]
+            end
+            commit[RELATED_EVENT] = push_event.reject {|k, _| k == "payload" }
+            emit("commit", commit)
+          end
 
-        if user and (@include_foreign_commits or watching_user?(user))
-          commit[RELATED_USER_IMAGE_KEY] = push_event["actor"]["avatar_url"]
-          if push_event["org"]
-            commit[RELATED_ORGANIZATION_IMAGE_KEY] = push_event["org"]["avatar_url"]
+          commit_refs = push_event["payload"]["commits"]
+          target_commit_ref = commit_refs.find do |commit_ref|
+            commit_ref["sha"] == commit["sha"]
           end
-          commit[RELATED_EVENT] = push_event.reject {|k, _| k == "payload" }
-          emit("commit", commit)
-        end
+          target_commit_ref["commit"] = commit if target_commit_ref
 
-        commit_refs = push_event["payload"]["commits"]
-        target_commit_ref = commit_refs.find do |commit_ref|
-          commit_ref["sha"] == commit["sha"]
+          completely_fetched = commit_refs.all? do |commit_ref|
+            commit_ref["commit"]
+          end
+          emit("push", push_event) if completely_fetched
         end
-        target_commit_ref["commit"] = commit if target_commit_ref
 
-        completely_fetched = commit_refs.all? do |commit_ref|
-          commit_ref["commit"]
+        def watching_user?(user)
+          @watching_users.include?(user)
         end
-        emit("push", push_event) if completely_fetched
-      end
-
-      def watching_user?(user)
-        @watching_users.include?(user)
-      end
 
-      def process_issue_event(event)
-        payload = event["payload"]
-        case payload["action"]
-        when "opened"
-          emit("issue-open", event)
-        when "closed"
-          emit("issue-close", event)
-        when "reopened"
-          emit("issue-reopen", event)
-        when "assigned"
-          emit("issue-assign", event)
-        when "unassigned"
-          emit("issue-unassign", event)
-        when "labeled"
-          emit("issue-label", event)
-        when "unlabeled"
-          emit("issue-unlabel", event)
+        def process_issue_event(event)
+          payload = event["payload"]
+          case payload["action"]
+          when "opened"
+            emit("issue-open", event)
+          when "closed"
+            emit("issue-close", event)
+          when "reopened"
+            emit("issue-reopen", event)
+          when "assigned"
+            emit("issue-assign", event)
+          when "unassigned"
+            emit("issue-unassign", event)
+          when "labeled"
+            emit("issue-label", event)
+          when "unlabeled"
+            emit("issue-unlabel", event)
+          end
         end
-      end
 
-      def process_pull_request_event(event)
-        payload = event["payload"]
-        case payload["action"]
-        when "opened"
-          emit("pull-request", event)
-        when "closed"
-          if payload["pull_request"]["merged"]
-            emit("pull-request-merged", event)
-          else
-            emit("pull-request-cancelled", event)
+        def process_pull_request_event(event)
+          payload = event["payload"]
+          case payload["action"]
+          when "opened"
+            emit("pull-request", event)
+          when "closed"
+            if payload["pull_request"]["merged"]
+              emit("pull-request-merged", event)
+            else
+              emit("pull-request-cancelled", event)
+            end
+          when "reopened"
+            emit("pull-request-reopen", event)
           end
-        when "reopened"
-          emit("pull-request-reopen", event)
         end
-      end
 
-      MERGE_COMMIT_MESSAGE_PATTERN = /\AMerge pull request #\d+ from [^\/]+\/[^\/]+\n\n/
+        MERGE_COMMIT_MESSAGE_PATTERN = /\AMerge pull request #\d+ from [^\/]+\/[^\/]+\n\n/
 
-      def push_event_from_merged_pull_request?(event)
-        payload = event["payload"]
-        inserted_requests = []
-        commit_refs = payload["commits"]
-        if MERGE_COMMIT_MESSAGE_PATTERN =~ commit_refs.last["message"]
-          true
-        else
-          false
+        def push_event_from_merged_pull_request?(event)
+          payload = event["payload"]
+          inserted_requests = []
+          commit_refs = payload["commits"]
+          if MERGE_COMMIT_MESSAGE_PATTERN =~ commit_refs.last["message"]
+            true
+          else
+            false
+          end
         end
-      end
 
-      def process_issue_or_pull_request_comment_event(event)
-        payload = event["payload"]
-        if payload["issue"]["pull_request"]
-          emit("pull-request-comment", event)
+        def process_issue_or_pull_request_comment_event(event)
+          payload = event["payload"]
+          if payload["issue"]["pull_request"]
+            emit("pull-request-comment", event)
           # emit("pull-request.cancel", event)
-        else
-          emit("issue-comment", event)
+          else
+            emit("issue-comment", event)
+          end
         end
-      end
 
-      def process_create_event(event)
-        payload = event["payload"]
-        case payload["ref_type"]
-        when "branch"
-          emit("branch", event)
-        when "tag"
-          emit("tag", event)
+        def process_create_event(event)
+          payload = event["payload"]
+          case payload["ref_type"]
+          when "branch"
+            emit("branch", event)
+          when "tag"
+            emit("tag", event)
+          end
         end
-      end
 
-      private
-      def user_activities(user)
-        "https://api.github.com/users/#{user}/events/public"
-      end
+        def stop
+          @running = false
+        end
 
-      def user_info(user)
-        "https://api.github.com/users/#{user}"
-      end
+        private
+        def user_activities(user)
+          "https://api.github.com/users/#{user}/events/public"
+        end
 
-      def emit(tag, record)
-        $log.trace("GithubActivities::Crawler: emit => #{tag}, #{record.inspect}") if $log
-        @on_emit.call(tag, record) if @on_emit
-      end
+        def user_info(user)
+          "https://api.github.com/users/#{user}"
+        end
 
-      def http_get(uri, extra_headers={})
-        parsed_uri = URI(uri)
-        if @access_token
-          extra_headers["Authorization"] = "token #{@access_token}"
+        def emit(tag, record)
+          log.trace("GithubActivities::Crawler: emit => #{tag}, #{record.inspect}")
+          @on_emit.call(tag, record) if @on_emit
         end
-        response = nil
-        http = Net::HTTP.new(parsed_uri.host, parsed_uri.port)
-        http.use_ssl = parsed_uri.is_a?(URI::HTTPS)
-        http.start do |http|
-          http_request = Net::HTTP::Get.new(parsed_uri.path, extra_headers)
-          response = http.request(http_request)
+
+        def http_get(uri, extra_headers={})
+          parsed_uri = URI(uri)
+          if @access_token
+            extra_headers["Authorization"] = "token #{@access_token}"
+          end
+          response = nil
+          http = Net::HTTP.new(parsed_uri.host, parsed_uri.port)
+          http.use_ssl = parsed_uri.is_a?(URI::HTTPS)
+          http.start do |http|
+            http_request = Net::HTTP::Get.new(parsed_uri.path, extra_headers)
+            response = http.request(http_request)
+          end
+          response
         end
-        response
       end
     end
   end