fluent-plugin-droonga 1.0.0 → 1.0.1

data/lib/droonga/fluent_message_sender.rb

@@ -15,6 +15,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "thread"
+
 require "cool.io"
 
 require "droonga/loggable"
@@ -28,31 +30,40 @@ module Droonga
       @loop = loop
       @host = host
       @port = port
+      @socket = nil
+      @buffer = []
+      @write_mutex = Mutex.new
     end
 
     def start
       logger.trace("start: start")
-      connect
+      start_writer
       logger.trace("start: done")
     end
 
     def shutdown
       logger.trace("shutdown: start")
-      @socket.close unless @socket.closed?
+      shutdown_writer
+      shutdown_socket
       logger.trace("shutdown: done")
     end
 
     def send(tag, data)
       logger.trace("send: start")
-      connect if @socket.closed?
       fluent_message = [tag, Time.now.to_i, data]
       packed_fluent_message = MessagePackPacker.pack(fluent_message)
-      @socket.write(packed_fluent_message)
-      @loop.break_current_loop
+      @write_mutex.synchronize do
+        @buffer << packed_fluent_message
+        @writer.signal
+      end
       logger.trace("send: done")
     end
 
     private
+    def connected?
+      not @socket.nil?
+    end
+
     def connect
       logger.trace("connect: start")
 
@@ -64,6 +75,10 @@ module Droonga
       end
       log_failed = lambda do
         logger.error("failed to connect to #{@host}:#{@port}")
+        @socket = nil
+      end
+      on_close = lambda do
+        @socket = nil
       end
 
       @socket = Coolio::TCPSocket.connect(@host, @port)
@@ -76,11 +91,42 @@ module Droonga
       @socket.on_connect_failed do
         log_failed.call
       end
+      @socket.on_close do
+        on_close.call
+      end
       @loop.attach(@socket)
 
       logger.trace("connect: done")
     end
 
+    def shutdown_socket
+      return unless connected?
+      @socket.close unless @socket.closed?
+    end
+
+    def start_writer
+      @writer = Coolio::AsyncWatcher.new
+
+      on_signal = lambda do
+        @write_mutex.synchronize do
+          connect unless connected?
+          @buffer.each do |data|
+            @socket.write(data)
+          end
+          @buffer.clear
+        end
+      end
+      @writer.on_signal do
+        on_signal.call
+      end
+
+      @loop.attach(@writer)
+    end
+
+    def shutdown_writer
+      @writer.detach
+    end
+
     def log_tag
       "[#{Process.ppid}][#{Process.pid}] fluent-message-sender"
     end

data/lib/droonga/handler_runner.rb

@@ -80,7 +80,7 @@ module Droonga
       end
       logger.debug("#{self.class.name}: activating plugins for the dataset \"#{@dataset_name}\": " +
                      "#{@options[:plugins].join(", ")}")
-      @step_runner = StepRunner.new(@options[:plugins] || [])
+      @step_runner = StepRunner.new(nil, @options[:plugins] || [])
       @forwarder = Forwarder.new(@loop)
     end
 

data/lib/droonga/job_protocol.rb

@@ -0,0 +1,20 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module JobProtocol
+    READY_SIGNAL = "R"
+  end
+end

data/lib/droonga/job_pusher.rb

@@ -0,0 +1,178 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "msgpack"
+
+require "droonga/logger"
+require "droonga/job_protocol"
+
+module Droonga
+  class JobPusher
+    include Loggable
+
+    attr_reader :socket_path
+    def initialize(loop, base_path)
+      @loop = loop
+      @socket_path = "#{base_path}.sock"
+      @job_queue = JobQueue.new(@loop)
+    end
+
+    def start
+      FileUtils.rm_f(@socket_path)
+      @server = Coolio::UNIXServer.new(@socket_path) do |connection|
+        @job_queue.add_worker(WorkerConnection.new(connection))
+      end
+      FileUtils.chmod(0600, @socket_path)
+      @loop.attach(@server)
+    end
+
+    def close
+      @server.close
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      @server.close
+      @job_queue.close
+      FileUtils.rm_f(@socket_path)
+      logger.trace("shutdown: done")
+    end
+
+    def push(message)
+      logger.trace("push: start")
+      @job_queue.push(message)
+      logger.trace("push: done")
+    end
+
+    private
+    def log_tag
+      "job_pusher"
+    end
+
+    class JobQueue
+      include Loggable
+
+      def initialize(loop)
+        @loop = loop
+        @buffers = []
+        @ready_workers = []
+        @workers = []
+        @many_jobs_report_interval = 100
+        update_many_jobs_threshold
+      end
+
+      def close
+        @workers.each do |worker|
+          worker.close
+        end
+      end
+
+      def add_worker(worker)
+        @workers << worker
+        update_many_jobs_threshold
+        worker.on_ready = lambda do |ready_worker|
+          supply_job(ready_worker)
+        end
+      end
+
+      def push(message)
+        job = message.to_msgpack
+        if @ready_workers.empty?
+          @buffers << job
+          report_statistics_on_push
+        else
+          worker = @ready_workers.shift
+          if @buffers.empty?
+            worker.write(job)
+          else
+            @buffers << job
+            worker.write(@buffers.shift)
+          end
+        end
+      end
+
+      private
+      def supply_job(worker)
+        if @buffers.empty?
+          @ready_workers << worker
+        else
+          worker.write(@buffers.shift)
+          report_statistics_on_pull
+        end
+      end
+
+      def update_many_jobs_threshold
+        @many_jobs_threshold = @workers.size * 100
+      end
+
+      def report_statistics_on_push
+        if @buffers.size >= @many_jobs_threshold
+          if (@buffers.size % @many_jobs_report_interval).zero?
+            logger.warn("push: many jobs in queue: #{@buffers.size}")
+          end
+        end
+      end
+
+      def report_statistics_on_pull
+        if @buffers.size >= @many_jobs_threshold
+          if (@buffers.size % @many_jobs_report_interval).zero?
+            logger.info("pull: many jobs in queue: #{@buffers.size}")
+          end
+        elsif @buffers.size == (@many_jobs_threshold - 1)
+          logger.info("pull: reducing jobs in queue: #{@buffers.size}")
+        end
+      end
+
+      def log_tag
+        "job_queue"
+      end
+    end
+
+    class WorkerConnection
+      attr_writer :on_ready
+
+      def initialize(connection)
+        @connection = connection
+        @ready = false
+        @on_ready = nil
+        setup_connection
+      end
+
+      def ready?
+        @ready
+      end
+
+      def write(job)
+        @connection.write(job)
+        @ready = false
+      end
+
+      def close
+        @connection.close
+      end
+
+      private
+      def setup_connection
+        on_read = lambda do |data|
+          @ready = (data == JobProtocol::READY_SIGNAL)
+          @on_ready.call(self) if @on_ready
+        end
+        @connection.on_read do |data|
+          on_read.call(data)
+        end
+      end
+    end
+  end
+end

data/lib/droonga/{message_receiver.rb → job_receiver.rb}

@@ -16,21 +16,22 @@
 require "msgpack"
 
 require "droonga/loggable"
+require "droonga/job_protocol"
 
 module Droonga
-  class MessageReceiver
+  class JobReceiver
     include Loggable
 
-    def initialize(loop, receiver, &callback)
+    def initialize(loop, socket_path, &callback)
       @loop = loop
-      @receiver = Coolio::Server.new(receiver, Coolio::Socket) do |connection|
-        setup_receive_handler(connection)
-      end
+      @socket_path = socket_path
       @callback = callback
     end
 
     def start
       logger.trace("start: start")
+      @receiver = Coolio::UNIXSocket.connect(@socket_path)
+      setup_receive_handler(@receiver)
       @loop.attach(@receiver)
       logger.trace("start: done")
     end
@@ -50,14 +51,20 @@ module Droonga
           @callback.call(message)
         end
         logger.trace("on_read: done")
+        send_ready(connection)
       end
       connection.on_read do |data|
         on_read.call(data)
       end
+      send_ready(connection)
+    end
+
+    def send_ready(connection)
+      connection.write(JobProtocol::READY_SIGNAL)
     end
 
     def log_tag
-      "message_receiver"
+      "job_receiver"
     end
   end
 end

data/lib/droonga/message_matcher.rb

@@ -59,47 +59,50 @@ module Droonga
   class MessageMatcher
     # @param [Array] pattern The pattern to be matched against a message.
     def initialize(pattern)
-      @pattern = pattern
+      path, operator, *arguments = pattern
+      @path_components = path.split(".")
+      @operator = operator
+      @arguments = arguments
     end
 
     def match?(message)
-      return false if @pattern.nil?
-      path, operator, *arguments = @pattern
-      target = resolve_path(path, message)
-      apply_operator(operator, target, arguments)
+      target = extract_target(message)
+      apply_operator(target)
     end
 
     private
     NONEXISTENT_PATH = Object.new
-    def resolve_path(path, message)
-      path.split(".").inject(message) do |result, component|
+    def extract_target(message)
+      result = message
+      @path_components.each do |component|
         return NONEXISTENT_PATH unless result.is_a?(Hash)
-        result[component]
+        result = result[component]
       end
+      result
     end
 
-    def apply_operator(operator, target, arguments)
-      case operator
+    def apply_operator(target)
+      case @operator
       when :equal
-        [target] == arguments
+        [target] == @arguments
       when :in
-        arguments.any? do |argument|
+        @arguments.any? do |argument|
           argument.include?(target)
         end
       when :include
         return false unless target.respond_to?(:include?)
-        arguments.any? do |argument|
+        @arguments.any? do |argument|
           target.include?(argument)
         end
       when :exist
         target != NONEXISTENT_PATH
       when :start_with
         return false unless target.respond_to?(:start_with?)
-        arguments.any? do |argument|
+        @arguments.any? do |argument|
           target.start_with?(argument)
         end
       else
-        raise ArgumentError, "Unknown operator: <#{operator}>"
+        raise ArgumentError, "Unknown operator: <#{@operator}>"
       end
     end
   end

data/lib/droonga/planner.rb

@@ -30,10 +30,9 @@ module Droonga
     end
 
     private
-    def scatter(message, options={})
+    def scatter(message, record, options={})
       planner = DistributedCommandPlanner.new(message)
-      planner.scatter
-      planner.key = options[:key]
+      planner.scatter(record)
       planner.reduce(options[:reduce])
       planner.plan
     end

data/lib/droonga/plugins/crud.rb

@@ -32,7 +32,7 @@ module Droonga
           request = input_message.body
           key = request["key"] || rand.to_s
           values = request["values"] || {}
-          request["filter"] = values.merge("key" => key)
+          request["filter"] = values.merge("_key" => key)
         end
 
         def adapt_output(output_message)