fluent-plugin-github-activities 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8133f90f59fd64a19ee93f670819992b65d3697c
-  data.tar.gz: e7dea9ef571e9c2065107e7bec3358f34c47460e
+  metadata.gz: db6ee8ed9280b6965e10bd6ebf3248aba5a945de
+  data.tar.gz: 58e9f85025bcab3c4094203eb8ebca1133faed30
 SHA512:
-  metadata.gz: 1f4921f84438dee4134f1685ce95da788eb731ed1fb2a67cc88dcde1a1be03c9a213b09dd74349e5068c1b0bb41719850a3896da82c039b4ff6a34f7bd18eb0e
-  data.tar.gz: da531444b368936dec4c321efc61c174bd2ccd9575ea239c3f2a9c35ad0bb42cf5f63526e53058bf8c61de5ea3fced393e51a1340c24b4c320f1bdbb4ff5f43e
+  metadata.gz: 3a357bc0e21a4eda4069a8eee4df00e7fe7508de55cd51f9251e529d3d54ef27d3e7613d87e6ce8069afb19d2200aee458914038a9a41dc845d32541758290b7
+  data.tar.gz: 0177ef399a2d19968e3628c6ad3686c2e2f1e99a581be729521c85977ca1e609136c8b5742c0bab43d985a8f984be122749fe05d31d55df6a7c9d45d1861b175

data/README.md

@@ -44,6 +44,7 @@ Notes:
 
 The configuration item `access_token` is optional but strongly recommended to be configured, because there is a rate limit: 60requests/hour by default.
 By an authenticated crawler, you can crawl 5000requests/hour (means about 80requests/minute).
+See also [the guide to create new access token](https://help.github.com/articles/creating-an-access-token-for-command-line-use/).
 
 A new access token for your instance can be generated by a simple BASIC authentication, like:
 
@@ -72,6 +73,7 @@ Enter host password for user 'your-account':
 }
 ~~~
 
+fluent-plugin-github-activities crawls only public activities, so you don't have to give any extra permission.
 Then the value of the `token` field is the access key to be written to the configuration file.
 
 ## Configurations
@@ -86,6 +88,9 @@ Then the value of the `token` field is the access key to be written to the confi
   # Interval seconds for requests. This is `1` by default.
   interval 1
 
+  # Number of clients. This is `4` by default.
+  clients 1
+
   # Path to a file to store timestamp of last crawled activity
   # for each user. If you don't specify this option, same records
   # can be forwarded after the fluentd is restarted.

data/fluent-plugin-github-activities.gemspec

@@ -19,11 +19,11 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-github-activities"
-  spec.version = "0.4.0"
+  spec.version = "0.5.0"
   spec.authors = ["YUKI Hiroshi"]
   spec.email = ["yuki@clear-code.com"]
   spec.summary = "Fluentd plugin to crawl public activities on the GitHub."
-  spec.description = "This provides ability you fluentd to crawl public " +
+  spec.description = "This provides ability to crawl public " +
                        "activities of users."
   spec.homepage = "https://github.com/groonga/fluent-plugin-groonga"
   spec.license = "LGPL-3.0"

data/lib/fluent/plugin/github-activities.rb

@@ -17,6 +17,7 @@
 # License along with fluent-plugin-github-activities.  If not, see
 # <http://www.gnu.org/licenses/>.
 
+require "fluent/plugin/github-activities/users_manager"
 require "fluent/plugin/github-activities/crawler"
 
 module Fluent

data/lib/fluent/plugin/github-activities/crawler.rb

@@ -20,10 +20,9 @@
 require "uri"
 require "net/https"
 require "json"
-require "pathname"
 require "time"
 
-require "fluent/plugin/github-activities/safe_file_writer"
+require "fluent/plugin/github-activities/users_manager"
 
 module Fluent
   module GithubActivities
@@ -34,15 +33,17 @@ module Fluent
       NO_INTERVAL = 0
       DEFAULT_INTERVAL = 1
 
-      DEFAULT_LAST_EVENT_TIMESTAMP = -1
-
       RELATED_USER_IMAGE_KEY = "$github-activities-related-avatar"
       RELATED_ORGANIZATION_IMAGE_KEY = "$github-activities-related-organization-logo"
+      RELATED_EVENT = "$github-activities-related-event"
 
       attr_writer :on_emit
       attr_reader :request_queue, :interval_for_next_request
 
       def initialize(options={})
+        @users_manager = UsersManager.new(:users    => options[:watching_users],
+                                          :pos_file => options[:pos_file])
+
         @access_token = options[:access_token]
 
         @watching_users = options[:watching_users] || []
@@ -50,25 +51,14 @@ module Fluent
         @include_commits_from_pull_request = options[:include_commits_from_pull_request]
         @include_foreign_commits = options[:include_foreign_commits]
 
-        @positions = {}
-        @pos_file = options[:pos_file]
-        @pos_file = Pathname(@pos_file) if @pos_file
-        load_positions
-
         @request_queue = options[:request_queue] || []
 
         @default_interval = options[:default_interval] || DEFAULT_INTERVAL
-
-        @watching_users.each do |user|
-          reserve_user_events(user)
-        end
       end
 
       def process_request
-        raise EmptyRequestQueue.new if @request_queue.empty?
-
         request = @request_queue.shift
-        $log.info("GithubActivities::Crawler: processing request: #{request.inspect}") if $log
+        $log.debug("GithubActivities::Crawler: processing request: #{request.inspect}") if $log
         if request[:process_after] and
              Time.now.to_i < request[:process_after]
           @request_queue.push(request)
@@ -79,34 +69,36 @@ module Fluent
         uri = request_uri(request)
         extra_headers = extra_request_headers(request)
 
-        $log.info("GithubActivities::Crawler: requesting to #{uri.inspect}") if $log
+        $log.debug("GithubActivities::Crawler: requesting to #{uri.inspect}") if $log
         response = http_get(uri, extra_headers)
-        $log.info("GithubActivities::Crawler: response: #{response.inspect}") if $log
+        $log.debug("GithubActivities::Crawler: response: #{response.inspect}") if $log
 
         case response
         when Net::HTTPSuccess
+          $log.trace("GithubActivities::Crawler: Net::HTTPSuccess / request type: #{request[:type]}") if $log
           body = JSON.parse(response.body)
-          $log.info("GithubActivities::Crawler: request type: #{request[:type]}") if $log
           case request[:type]
           when TYPE_EVENTS
             events = body
-            $log.info("GithubActivities::Crawler: events size: #{events.size}") if $log
+            $log.trace("GithubActivities::Crawler: events size: #{events.size}") if $log
             process_user_events(request[:user], events)
             reserve_user_events(request[:user], :previous_response => response)
-            save_user_position(request[:user], :entity_tag => response["ETag"])
+            @users_manager.save_position_for(request[:user], :entity_tag => response["ETag"])
           when TYPE_COMMIT
             process_commit(body, request[:push])
           end
         when Net::HTTPNotModified
+          $log.trace("GithubActivities::Crawler: Net::HTTPNotModified / request type: #{request[:type]}") if $log
           case request[:type]
           when TYPE_EVENTS
             reserve_user_events(request[:user],
                                 :previous_response => response,
                                 :previous_entity_tag => extra_headers["If-None-Match"])
           end
-          @interval_for_next_request = NO_INTERVAL
+          @interval_for_next_request = @default_interval
           return true
-        when Net::HTTPNotFound
+        else
+          $log.trace("GithubActivities::Crawler: UnknownType / request type: #{request[:type]}") if $log
           case request[:type]
           when TYPE_COMMIT
             fake_body = {
@@ -118,6 +110,8 @@ module Fluent
         end
         @interval_for_next_request = @default_interval
         return true
+      rescue StandardError => error
+        $log.error(error.inspect)
       end
 
       def request_uri(request)
@@ -134,35 +128,28 @@ module Fluent
         headers = {}
         if request[:previous_entity_tag]
           headers["If-None-Match"] = request[:previous_entity_tag]
-        elsif request[:type] == TYPE_EVENTS and @positions[request[:user]]
-          entity_tag = @positions[request[:user]]["entity_tag"]
-          headers["If-None-Match"] = entity_tag if entity_tag
+        elsif request[:type] == TYPE_EVENTS
+          position = @users_manager.position_for(request[:user])
+          if position
+            entity_tag = position["entity_tag"]
+            headers["If-None-Match"] = entity_tag if entity_tag
+          end
         end
         headers
       end
 
       def reserve_user_events(user, options={})
-        request = {
-          :type => TYPE_EVENTS,
-          :user => user,
-        }
-        response = options[:previous_response]
-        if response
-          now = options[:now] || Time.now
-          interval = response["X-Poll-Interval"].to_i
-          time_to_process = now.to_i + interval
-          request[:previous_entity_tag] = response["ETag"] ||
-                                            options[:previous_entity_tag]
-          request[:process_after] = time_to_process
-        end
+        request = @users_manager.new_events_request(user, options)
         @request_queue.push(request)
       end
 
       def process_user_events(user, events)
-        last_event_timestamp = DEFAULT_LAST_EVENT_TIMESTAMP
-        if @positions[user] and @positions[user]["last_event_timestamp"]
-          last_event_timestamp = @positions[user]["last_event_timestamp"]
+        last_event_timestamp = UsersManager::DEFAULT_LAST_EVENT_TIMESTAMP
+        position = @users_manager.position_for(user)
+        if position and position["last_event_timestamp"]
+          last_event_timestamp = position["last_event_timestamp"]
         end
+
         events = events.sort do |a, b|
           b["created_at"] <=> a["created_at"]
         end
@@ -170,7 +157,7 @@ module Fluent
           timestamp = Time.parse(event["created_at"]).to_i
           next if timestamp <= last_event_timestamp
           process_user_event(user, event)
-          save_user_position(user, :last_event_timestamp => timestamp)
+          @users_manager.save_position_for(user, :last_event_timestamp => timestamp)
         end
       end
 
@@ -198,6 +185,8 @@ module Fluent
         else
           emit(event["type"], event)
         end
+      rescue StandardError => error
+        $log.exception(error)
       end
 
       def process_push_event(event)
@@ -208,16 +197,16 @@ module Fluent
           return
         end
         commit_refs.reverse.each do |commit_ref|
-          @request_queue.unshift(:type => TYPE_COMMIT,
-                                 :uri  => commit_ref["url"],
-                                 :sha  => commit_ref["sha"],
-                                 :push => event)
+          @request_queue.push(:type => TYPE_COMMIT,
+                              :uri  => commit_ref["url"],
+                              :sha  => commit_ref["sha"],
+                              :push => event)
         end
         # emit("push", event)
       end
 
       def process_commit(commit, push_event)
-        $log.info("GithubActivities::Crawler: processing commit #{commit["sha"]}") if $log
+        $log.debug("GithubActivities::Crawler: processing commit #{commit["sha"]}") if $log
         user = commit["author"]["login"]
 
         if user and (@include_foreign_commits or watching_user?(user))
@@ -225,6 +214,7 @@ module Fluent
           if push_event["org"]
             commit[RELATED_ORGANIZATION_IMAGE_KEY] = push_event["org"]["avatar_url"]
           end
+          commit[RELATED_EVENT] = push_event
           emit("commit", commit)
         end
 
@@ -341,38 +331,6 @@ module Fluent
         end
         response
       end
-
-      def load_positions
-        return unless @pos_file
-        return unless @pos_file.exist?
-
-        @positions = JSON.parse(@pos_file.read)
-      rescue
-        @positions = {}
-      end
-
-      def save_positions
-        return unless @pos_file
-        SafeFileWriter.write(@pos_file, JSON.pretty_generate(@positions))
-      end
-
-      def save_user_position(user, params)
-        @positions[user] ||= {}
-
-        if params[:entity_tag]
-          @positions[user]["entity_tag"] = params[:entity_tag]
-        end
-
-        if params[:last_event_timestamp] and
-             params[:last_event_timestamp] != DEFAULT_LAST_EVENT_TIMESTAMP
-          old_timestamp = @positions[user]["last_event_timestamp"]
-          if old_timestamp.nil? or old_timestamp < params[:last_event_timestamp]
-            @positions[user]["last_event_timestamp"] = params[:last_event_timestamp]
-          end
-        end
-
-        save_positions
-      end
     end
   end
 end

data/lib/fluent/plugin/github-activities/users_manager.rb

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of fluent-plugin-github-activities.
+#
+# fluent-plugin-github-activities is free software: you can
+# redistribute it and/or modify it under the terms of the GNU Lesser
+# General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# fluent-plugin-github-activities is distributed in the hope that
+# it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with fluent-plugin-github-activities.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+require "pathname"
+require "json"
+
+require "fluent/plugin/github-activities/safe_file_writer"
+
+module Fluent
+  module GithubActivities
+    class UsersManager
+      DEFAULT_LAST_EVENT_TIMESTAMP = -1
+
+      def initialize(params={})
+        @users = params[:users]
+
+        @positions = {}
+        @pos_file = params[:pos_file]
+        @pos_file = Pathname(@pos_file) if @pos_file
+      end
+
+      def generate_initial_requests
+        @users.collect do |user|
+          new_events_request(user)
+        end
+      end
+
+      def new_events_request(user, options={})
+        request = {
+          :type => TYPE_EVENTS,
+          :user => user,
+        }
+        response = options[:previous_response]
+        if response
+          now = options[:now] || Time.now
+          interval = response["X-Poll-Interval"].to_i
+          time_to_process = now.to_i + interval
+          request[:previous_entity_tag] = response["ETag"] ||
+                                            options[:previous_entity_tag]
+          request[:process_after] = time_to_process
+        else
+          request[:previous_entity_tag] = options[:previous_entity_tag]
+        end
+        request
+      end
+
+      def position_for(user)
+        load_positions
+        @positions[user]
+      end
+
+      def save_position_for(user, params)
+        load_positions
+        @positions[user] ||= {}
+
+        if params[:entity_tag]
+          @positions[user]["entity_tag"] = params[:entity_tag]
+        end
+
+        if params[:last_event_timestamp] and
+             params[:last_event_timestamp] != DEFAULT_LAST_EVENT_TIMESTAMP
+          old_timestamp = @positions[user]["last_event_timestamp"]
+          if old_timestamp.nil? or old_timestamp < params[:last_event_timestamp]
+            @positions[user]["last_event_timestamp"] = params[:last_event_timestamp]
+          end
+        end
+
+        save_positions
+      end
+
+      private
+      def load_positions
+        return unless @pos_file
+        return unless @pos_file.exist?
+
+        @positions = JSON.parse(@pos_file.read)
+      rescue
+        @positions = {}
+      end
+
+      def save_positions
+        return unless @pos_file
+        SafeFileWriter.write(@pos_file, JSON.pretty_generate(@positions))
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_github-activities.rb

@@ -20,6 +20,7 @@
 module Fluent
   class GithubActivitiesInput < Input
     DEFAULT_BASE_TAG = "github-activity"
+    DEFAULT_CLIENTS = 4
 
     Plugin.register_input("github-activities", self)
 
@@ -30,6 +31,7 @@ module Fluent
     config_param :include_foreign_commits, :bool, :default => false
     config_param :base_tag, :string, :default => DEFAULT_BASE_TAG
     config_param :pos_file, :string, :default => nil
+    config_param :clients, :integer, :default => DEFAULT_CLIENTS
     config_param :interval, :integer, :default => 1
 
     def initialize
@@ -42,29 +44,49 @@ module Fluent
 
     def start
       @base_tag = @base_tag.sub(/\.\z/, "")
-      @thread = Thread.new do
-        crawler_options = {
-          :access_token => @access_token,
-          :watching_users => prepare_users_list,
-          :include_commits_from_pull_request => @include_commits_from_pull_request,
-          :include_foreign_commits => @include_foreign_commits,
-          :pos_file => @pos_file,
-          :default_interval => @interval,
-        }
-        @crawler = ::Fluent::GithubActivities::Crawler.new(crawler_options)
-        @crawler.on_emit = lambda do |tag, record|
-          Engine.emit("#{@base_tag}.#{tag}", Engine.now, record)
-        end
 
-        loop do
-          @crawler.process_request
-          sleep(@crawler.interval_for_next_request)
+      users = prepare_users_list
+      n_clients = [@clients, users.size].min
+      @interval = @interval * n_clients
+
+      @client_threads = []
+      @request_queue = Queue.new
+
+      users_manager_params = {
+        :users    => users,
+        :pos_file => @pos_file,
+      }
+      users_manager = ::Fluent::GithubActivities::UsersManager.new(users_manager_params)
+      users_manager.generate_initial_requests.each do |request|
+        @request_queue.push(request)
+      end
+
+      n_clients.times do
+        @client_threads << Thread.new do
+          crawler_options = {
+            :access_token => @access_token,
+            :watching_users => users,
+            :include_commits_from_pull_request => @include_commits_from_pull_request,
+            :include_foreign_commits => @include_foreign_commits,
+            :pos_file => @pos_file,
+            :request_queue => @request_queue,
+            :default_interval => @interval,
+          }
+          crawler = ::Fluent::GithubActivities::Crawler.new(crawler_options)
+          crawler.on_emit = lambda do |tag, record|
+            Engine.emit("#{@base_tag}.#{tag}", Engine.now, record)
+          end
+
+          loop do
+            crawler.process_request
+            sleep(crawler.interval_for_next_request)
+          end
         end
       end
     end
 
     def shutdown
-      @thread.exit
+      @client_threads.each(&:exit)
     end
 
     private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-github-activities
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - YUKI Hiroshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-02 00:00:00.000000000 Z
+date: 2015-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This provides ability you fluentd to crawl public activities of users.
+description: This provides ability to crawl public activities of users.
 email:
 - yuki@clear-code.com
 executables: []
@@ -107,6 +107,7 @@ files:
 - lib/fluent/plugin/github-activities.rb
 - lib/fluent/plugin/github-activities/crawler.rb
 - lib/fluent/plugin/github-activities/safe_file_writer.rb
+- lib/fluent/plugin/github-activities/users_manager.rb
 - lib/fluent/plugin/in_github-activities.rb
 - test/fixture.rb
 - test/fixture/accept-pull-request-event.json