fluent-plugin-github-activities 0.6.1 → 0.7.0

data/lib/fluent/plugin/github-activities/users_manager.rb

@@ -17,86 +17,66 @@
 # License along with fluent-plugin-github-activities.  If not, see
 # <http://www.gnu.org/licenses/>.
 
-require "pathname"
 require "json"
 
-require "fluent/plugin/github-activities/safe_file_writer"
-
 module Fluent
-  module GithubActivities
-    class UsersManager
-      DEFAULT_LAST_EVENT_TIMESTAMP = -1
-
-      def initialize(params={})
-        @users = params[:users]
+  module Plugin
+    module GithubActivities
+      class UsersManager
+        DEFAULT_LAST_EVENT_TIMESTAMP = -1
 
-        @positions = {}
-        @pos_file = params[:pos_file]
-        @pos_file = Pathname(@pos_file) if @pos_file
-      end
-
-      def generate_initial_requests
-        @users.collect do |user|
-          new_events_request(user)
+        def initialize(params={})
+          @users = params[:users]
+          @pos_storage = params[:pos_storage]
         end
-      end
 
-      def new_events_request(user, options={})
-        request = {
-          :type => TYPE_EVENTS,
-          :user => user,
-        }
-        response = options[:previous_response]
-        if response
-          now = options[:now] || Time.now
-          interval = response["X-Poll-Interval"].to_i
-          time_to_process = now.to_i + interval
-          request[:previous_entity_tag] = response["ETag"] ||
-                                            options[:previous_entity_tag]
-          request[:process_after] = time_to_process
-        else
-          request[:previous_entity_tag] = options[:previous_entity_tag] if options.key?(:previous_entity_tag)
+        def generate_initial_requests
+          @users.collect do |user|
+            new_events_request(user)
+          end
         end
-        request
-      end
-
-      def position_for(user)
-        load_positions
-        @positions[user]
-      end
 
-      def save_position_for(user, params)
-        load_positions
-        @positions[user] ||= {}
-
-        if params[:entity_tag]
-          @positions[user]["entity_tag"] = params[:entity_tag]
+        def new_events_request(user, options={})
+          request = {
+            type: TYPE_EVENTS,
+            user: user,
+          }
+          response = options[:previous_response]
+          if response
+            now = options[:now] || Time.now
+            interval = response["X-Poll-Interval"].to_i
+            time_to_process = now.to_i + interval
+            request[:previous_entity_tag] = response["ETag"] ||
+                                            options[:previous_entity_tag]
+            request[:process_after] = time_to_process
+          else
+            request[:previous_entity_tag] = options[:previous_entity_tag] if options.key?(:previous_entity_tag)
+          end
+          request
         end
 
-        if params[:last_event_timestamp] and
-             params[:last_event_timestamp] != DEFAULT_LAST_EVENT_TIMESTAMP
-          old_timestamp = @positions[user]["last_event_timestamp"]
-          if old_timestamp.nil? or old_timestamp < params[:last_event_timestamp]
-            @positions[user]["last_event_timestamp"] = params[:last_event_timestamp]
-          end
+        def position_for(user)
+          @pos_storage.get(user)
         end
 
-        save_positions
-      end
+        def save_position_for(user, params)
+          position = @pos_storage.get(user) || {}
 
-      private
-      def load_positions
-        return unless @pos_file
-        return unless @pos_file.exist?
+          if params[:entity_tag]
+            position["entity_tag"] = params[:entity_tag]
+          end
 
-        @positions = JSON.parse(@pos_file.read)
-      rescue
-        @positions = {}
-      end
+          if params[:last_event_timestamp] and
+            params[:last_event_timestamp] != DEFAULT_LAST_EVENT_TIMESTAMP
+            old_timestamp = position["last_event_timestamp"]
+            if old_timestamp.nil? or old_timestamp < params[:last_event_timestamp]
+              position["last_event_timestamp"] = params[:last_event_timestamp]
+            end
+          end
 
-      def save_positions
-        return unless @pos_file
-        SafeFileWriter.write(@pos_file, JSON.pretty_generate(@positions))
+          @pos_storage.put(user, position)
+          @pos_storage.save
+        end
       end
     end
   end

data/lib/fluent/plugin/in_github-activities.rb

@@ -17,98 +17,116 @@
 # License along with fluent-plugin-github-activities.  If not, see
 # <http://www.gnu.org/licenses/>.
 
-module Fluent
-  class GithubActivitiesInput < Input
-    DEFAULT_BASE_TAG = "github-activity"
-    DEFAULT_CLIENTS = 4
-
-    Plugin.register_input("github-activities", self)
-
-    config_param :access_token, :string, :default => nil, :secret => true
-    config_param :users, :string, :default => nil
-    config_param :users_list, :string, :default => nil
-    config_param :include_commits_from_pull_request, :bool, :default => false
-    config_param :include_foreign_commits, :bool, :default => false
-    config_param :base_tag, :string, :default => DEFAULT_BASE_TAG
-    config_param :pos_file, :string, :default => nil
-    config_param :clients, :integer, :default => DEFAULT_CLIENTS
-    config_param :interval, :integer, :default => 1
-
-    def initialize
-      super
-
-      require "thread"
-      require "pathname"
-      require "fluent/plugin/github-activities"
-    end
-
-    def start
-      @base_tag = @base_tag.sub(/\.\z/, "")
-
-      users = prepare_users_list
-      n_clients = [@clients, users.size].min
-      @interval = @interval * n_clients
+require "thread"
+require "pathname"
+require "fluent/plugin/input"
+require "fluent/plugin/github-activities"
 
-      @client_threads = []
-      @request_queue = Queue.new
+module Fluent
+  module Plugin
+    class GithubActivitiesInput < Fluent::Plugin::Input
+      DEFAULT_BASE_TAG = "github-activity"
+      DEFAULT_CLIENTS = 4
+      DEFAULT_STORAGE_TYPE = "local"
+
+      helpers :thread, :storage
+
+      Fluent::Plugin.register_input("github-activities", self)
+
+      config_param :access_token, :string, default: nil, secret: true
+      config_param :users, :array, default: []
+      config_param :users_list, :string, default: nil
+      config_param :include_commits_from_pull_request, :bool, default: false
+      config_param :include_foreign_commits, :bool, default: false
+      config_param :base_tag, :string, default: DEFAULT_BASE_TAG
+      config_param :pos_file, :string, default: nil, obsoleted: "Use storage instead."
+      config_param :clients, :integer, default: DEFAULT_CLIENTS
+      config_param :interval, :integer, default: 1
+
+      config_section :storage do
+        config_set_default :usage, "in-github-activities"
+        config_set_default :@type, DEFAULT_STORAGE_TYPE
+      end
 
-      users_manager_params = {
-        :users    => users,
-        :pos_file => @pos_file,
-      }
-      users_manager = ::Fluent::GithubActivities::UsersManager.new(users_manager_params)
-      users_manager.generate_initial_requests.each do |request|
-        @request_queue.push(request)
+      def configure(conf)
+        super
+
+        @base_tag = @base_tag.sub(/\.\z/, "")
+        @users += load_users_list
+        @n_clients = [@clients, @users.size].min
+        @interval = @interval * @n_clients
+        raise Fluent::ConfigError, "You can define <storage> section at once" unless @storage_configs.size == 1
+        storage_section = @storage_configs.first
+        storage_config = storage_section.corresponding_config_element
+        @pos_storage = storage_create(usage: storage_section.usage,
+                                      conf: storage_config,
+                                      default_type: DEFAULT_STORAGE_TYPE)
       end
 
-      n_clients.times do
-        @client_threads << Thread.new do
-          crawler_options = {
-            :access_token => @access_token,
-            :watching_users => users,
-            :include_commits_from_pull_request => @include_commits_from_pull_request,
-            :include_foreign_commits => @include_foreign_commits,
-            :pos_file => @pos_file,
-            :request_queue => @request_queue,
-            :default_interval => @interval,
-          }
-          crawler = ::Fluent::GithubActivities::Crawler.new(crawler_options)
-          crawler.on_emit = lambda do |tag, record|
-            router.emit("#{@base_tag}.#{tag}", Engine.now, record)
-          end
+      def start
+        super
+
+        @request_queue = Queue.new
+        @crawlers = []
 
-          loop do
-            crawler.process_request
-            sleep(crawler.interval_for_next_request)
+        users_manager_params = {
+          users: @users,
+          pos_storage: @pos_storage,
+        }
+        users_manager = ::Fluent::Plugin::GithubActivities::UsersManager.new(users_manager_params)
+        users_manager.generate_initial_requests.each do |request|
+          @request_queue.push(request)
+        end
+        @n_clients.times do |n|
+          thread_create("in_github_activity_#{n}".to_sym) do
+            crawler_options = {
+              access_token: @access_token,
+              watching_users: @users,
+              include_commits_from_pull_request: @include_commits_from_pull_request,
+              include_foreign_commits: @include_foreign_commits,
+              pos_storage: @pos_storage,
+              request_queue: @request_queue,
+              default_interval: @interval,
+              log: log
+            }
+            crawler = ::Fluent::Plugin::GithubActivities::Crawler.new(crawler_options)
+            @crawlers << crawler
+            crawler.on_emit = lambda do |tag, record|
+              router.emit("#{@base_tag}.#{tag}", Engine.now, record)
+            end
+
+            loop do
+              crawler.process_request
+              break unless crawler.running
+              sleep(crawler.interval_for_next_request)
+            end
           end
         end
       end
-    end
 
-    def shutdown
-      @client_threads.each(&:exit)
-    end
+      def shutdown
+        until @request_queue.empty?
+          log.debug(queue_size: @request_queue.size)
+          sleep(@interval)
+        end
+        @crawlers.each(&:stop)
+        super
+      end
 
-    private
-    def prepare_users_list
-      @users ||= ""
-      users = @users.split(",")
+      private
 
-      if @users_list
-        users_list = Pathname(@users_list)
-        if users_list.exist?
-          list = users_list.read
-          users += list.split("\n")
+      def load_users_list
+        users = []
+        if @users_list
+          users_list = Pathname(@users_list)
+          if users_list.exist?
+            list = users_list.read
+            users += list.split("\n")
+          end
         end
-      end
 
-      users = users.collect do |user|
-        user.strip
-      end.reject do |user|
-        user.empty?
+        users.collect(&:strip).reject(&:empty?)
       end
-
-      users
     end
   end
 end

data/test/fixture/piroor-events.json

@@ -0,0 +1,47 @@
+[
+  {
+    "id": "2823041920",
+    "type": "PushEvent",
+    "actor": {
+      "id": 70062,
+      "login": "piroor",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/piroor",
+      "avatar_url": "https://avatars.githubusercontent.com/u/70062?"
+    },
+    "repo": {
+      "id": 35922279,
+      "name": "clear-code/fluent-plugin-github-activities",
+      "url": "https://api.github.com/repos/clear-code/fluent-plugin-github-activities"
+    },
+    "payload": {
+      "push_id": 670478406,
+      "size": 1,
+      "distinct_size": 1,
+      "ref": "refs/heads/master",
+      "head": "c908f319c7b6d5c5a69c8b675bde40dd990ee364",
+      "before": "e77acc068b5568f8ea55605db2e3a9005805c898",
+      "commits": [
+        {
+          "sha": "8e90721ff5d89f52b5b3adf0b86db01f03dc5588",
+          "author": {
+            "email": "shimoda@clear-code.com",
+            "name": "YUKI Hiroshi"
+          },
+          "message": "Add missing constant",
+          "distinct": true,
+          "url": "https://api.github.com/repos/clear-code/fluent-plugin-github-activities/commits/8e90721ff5d89f52b5b3adf0b86db01f03dc5588"
+        }
+      ]
+    },
+    "public": true,
+    "created_at": "2015-05-21T05:37:34Z",
+    "org": {
+      "id": 176515,
+      "login": "clear-code",
+      "gravatar_id": "",
+      "url": "https://api.github.com/orgs/clear-code",
+      "avatar_url": "https://avatars.githubusercontent.com/u/176515?"
+    }
+  }
+]

data/test/fixture/users.txt

@@ -0,0 +1,2 @@
+okkez
+cosmo0920

data/test/plugin/test_in_github_activity.rb

@@ -0,0 +1,85 @@
+require "fluent/plugin/in_github-activities"
+require "fluent/test"
+require "fluent/test/helpers"
+require "fluent/test/driver/input"
+
+class TestGithubActivitiesInput < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  setup do
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::GithubActivitiesInput).configure(conf)
+  end
+
+  sub_test_case "configure" do
+    test "empty" do
+      d = create_driver(config_element)
+      plugin = d.instance
+      storage = plugin._storages["in-github-activities"].storage
+      assert { storage.path.nil? }
+      assert { !storage.persistent }
+    end
+
+    test "obsoleted pos_file" do
+      conf = config_element("ROOT", "", { "pos_file" => "/tmp/pos_file.json" })
+      assert_raise Fluent::ObsoletedParameterError do
+        create_driver(conf)
+      end
+    end
+
+    data("end with .json" => ["/tmp/test.json", "/tmp/test.json"],
+         "end with .pos" => ["/tmp/test.pos", "/tmp/test.pos/worker0/storage.json"])
+    test "persistent storage with path" do |(path, expected_path)|
+      storage_conf = config_element(
+        "storage",
+        "in-github-activities",
+        { "@type" => "local", "persistent" => true, "path" => path }
+      )
+      conf = config_element("ROOT", "", {}, [storage_conf])
+      d = create_driver(conf)
+      plugin = d.instance
+      storage = plugin._storages["in-github-activities"].storage
+      assert_equal(expected_path, storage.path)
+      assert { storage.persistent }
+    end
+
+    data(single: ["piroor", ["piroor"]],
+         multiple: ["okkez,cosmo0920", ["okkez", "cosmo0920"]])
+    test "users" do |(users, expected_users)|
+      config = config_element("ROOT", "", { "users" => users })
+      d = create_driver(config)
+      plugin = d.instance
+      assert_equal(expected_users, plugin.users)
+    end
+
+    test "users_list" do
+      config = config_element("ROOT", "", { "users_list" => fixture_path("users.txt") })
+      d = create_driver(config)
+      plugin = d.instance
+      assert_equal(["okkez", "cosmo0920"], plugin.users)
+    end
+  end
+
+  sub_test_case "emit" do
+    test "simple" do
+      user_events_template = Addressable::Template.new("https://api.github.com/users/{user}/events/public")
+      stub_request(:get, user_events_template)
+        .to_return(body: File.open(fixture_path("piroor-events.json")), status: 200)
+      commits_template = Addressable::Template.new("https://api.github.com/repos/{owner}/{project}/commits/{commit_hash}")
+      stub_request(:get, commits_template)
+        .to_return(body: File.open(fixture_path("commit.json")), status: 200)
+      config = config_element("ROOT", "", { "users" => "piroor", "@log_level" => "trace" })
+      d = create_driver(config)
+      d.run(timeout: 1, expect_emits: 2)
+      tag, _time, record = d.events[0]
+      assert_equal("github-activity.commit", tag)
+      assert_equal("8e90721ff5d89f52b5b3adf0b86db01f03dc5588", record["sha"])
+      tag, _time, record = d.events[1]
+      assert_equal("github-activity.push", tag)
+      assert_equal("2823041920", record["id"])
+    end
+  end
+end