fluent-plugin-droonga 0.7.0 → 0.8.0

data/lib/droonga/distribution_planner.rb

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "tsort"
+
+module Droonga
+  class DistributionPlanner
+    class UndefinedInputError < StandardError
+      attr_reader :input
+      def initialize(input)
+        @input = input
+        super("undefined input assigned: <#{input}>")
+      end
+    end
+
+    class CyclicComponentsError < StandardError
+      attr_reader :components
+      def initialize(components)
+        @components = components
+        super("cyclic components found: <#{components}>")
+      end
+    end
+
+    include TSort
+
+    def initialize(dispatcher, components)
+      @dispatcher = dispatcher
+      @components = components
+    end
+
+    def plan
+      @dependency = {}
+      @components.each do |component|
+        @dependency[component] = component["inputs"]
+        next unless component["outputs"]
+        component["outputs"].each do |output|
+          @dependency[output] = [component]
+        end
+      end
+      components = []
+      each_strongly_connected_component do |cs|
+        raise CyclicComponentsError.new(cs) if cs.size > 1
+        components.concat(cs) unless cs.first.is_a? String
+      end
+      components
+    end
+
+    private
+    def tsort_each_node(&block)
+      @dependency.each_key(&block)
+    end
+
+    def tsort_each_child(node, &block)
+      if node.is_a? String and @dependency[node].nil?
+        raise UndefinedInputError.new(node)
+      end
+      if @dependency[node]
+        @dependency[node].each(&block)
+      end
+    end
+  end
+end

data/lib/droonga/distributor.rb

@@ -17,6 +17,7 @@
 
 require "droonga/pluggable"
 require "droonga/distributor_plugin"
+require "droonga/distribution_planner"
 
 module Droonga
   class Distributor
@@ -30,13 +31,10 @@ module Droonga
       load_plugins(options[:distributors] || ["search", "crud", "groonga", "watch"])
     end
 
-    def distribute(envelope)
-      command = envelope["type"]
-      process(command, envelope)
-    end
-
-    def post(message)
-      @dispatcher.handle(message, [])
+    def distribute(components)
+      planner = DistributionPlanner.new(@dispatcher, components)
+      planned_components = planner.plan
+      @dispatcher.dispatch_components(planned_components)
     end
 
     private

data/lib/droonga/distributor_plugin.rb

@@ -26,34 +26,42 @@ module Droonga
       @distributor = distributor
     end
 
-    # TODO: consider better name
-    def post(message)
-      @distributor.post(message)
+    def distribute(message)
+      @distributor.distribute(message)
     end
 
-    def scatter_all(envelope, key)
-      message = [{
-        "command"=> envelope["type"],
-        "dataset"=> envelope["dataset"],
-        "body"=> envelope["body"],
+    def scatter_all(message, key)
+      distribute_message = [{
+        "command"=> message["type"],
+        "dataset"=> message["dataset"],
+        "body"=> message["body"],
         "key"=> key,
         "type"=> "scatter",
         "replica"=> "all",
         "post"=> true
       }]
-      post(message)
+      distribute(distribute_message)
     end
 
-    def broadcast_all(envelope)
-      distirubte_message = [{
-        "command"=> envelope["type"],
-        "dataset"=> envelope["dataset"],
-        "body"=> envelope["body"],
+    def broadcast_all(message)
+      distribute_message = [{
+        "command"=> message["type"],
+        "dataset"=> message["dataset"],
+        "body"=> message["body"],
         "type"=> "broadcast",
         "replica"=> "all",
         "post"=> true
       }]
-      post(distirubte_message)
+      distribute(distribute_message)
+    end
+
+    private
+    def process_error(command, error, arguments)
+      if error.is_a?(MessageProcessingError)
+        raise error
+      else
+        super
+      end
     end
   end
 end

data/lib/droonga/engine.rb

@@ -35,8 +35,8 @@ module Droonga
       $log.trace("engine: shutdown: done")
     end
 
-    def process(envelope)
-      @dispatcher.handle_envelope(envelope)
+    def process(message)
+      @dispatcher.process_message(message)
     end
   end
 end

data/lib/droonga/event_loop.rb

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "coolio"
+
+module Droonga
+  class EventLoop
+    def initialize
+      @loop = Coolio::Loop.new
+      @loop_breaker = Coolio::AsyncWatcher.new
+      @loop_breaker.attach(@loop)
+    end
+
+    def run
+      @loop.run
+    end
+
+    def stop
+      @loop.stop
+      break_current_loop
+    end
+
+    def attach(watcher)
+      @loop.attach(watcher)
+      break_current_loop
+    end
+
+    def break_current_loop
+      @loop_breaker.signal
+    end
+  end
+end

data/lib/droonga/farm.rb

@@ -19,11 +19,15 @@ require "droonga/partition"
 
 module Droonga
   class Farm
-    def initialize(name)
+    def initialize(name, loop, options={})
       @name = name
+      @loop = loop
+      @options = options
       @partitions = {}
-      Droonga.catalog.get_partitions(name).each do |partition_name, options|
-        partition = Droonga::Partition.new(options)
+      partitions = Droonga.catalog.get_partitions(name)
+      partitions.each do |partition_name, partition_options|
+        partition = Droonga::Partition.new(@loop,
+                                           @options.merge(partition_options))
         @partitions[partition_name] = partition
       end
     end
@@ -40,8 +44,8 @@ module Droonga
       end
     end
 
-    def process(partition_name, envelope, synchronous)
-      @partitions[partition_name].process(envelope, synchronous)
+    def process(partition_name, message)
+      @partitions[partition_name].process(message)
     end
   end
 end

data/lib/droonga/fluent_message_sender.rb

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "cool.io"
+require "droonga/message_pack_packer"
+
+module Droonga
+  class FluentMessageSender
+    def initialize(loop, host, port)
+      @loop = loop
+      @host = host
+      @port = port
+    end
+
+    def start
+      $log.trace("#{log_tag}: start: start")
+      connect
+      $log.trace("#{log_tag}: start: done")
+    end
+
+    def shutdown
+      $log.trace("#{log_tag}: shutdown: start")
+      @socket.close unless @socket.closed?
+      $log.trace("#{log_tag}: shutdown: done")
+    end
+
+    def send(tag, data)
+      $log.trace("#{log_tag}: send: start")
+      connect if @socket.closed?
+      fluent_message = [tag, Time.now.to_i, data]
+      packed_fluent_message = MessagePackPacker.pack(fluent_message)
+      @socket.write(packed_fluent_message)
+      @loop.break_current_loop
+      $log.trace("#{log_tag}: send: done")
+    end
+
+    private
+    def connect
+      $log.trace("#{log_tag}: connect: start")
+
+      log_write_complete = lambda do
+        $log.trace("#{log_tag}: write completed")
+      end
+      log_connect = lambda do
+        $log.trace("#{log_tag}: connected to #{@host}:#{@port}")
+      end
+      log_failed = lambda do
+        $log.error("#{log_tag}: failed to connect to #{@host}:#{@port}")
+      end
+
+      @socket = Coolio::TCPSocket.connect(@host, @port)
+      @socket.on_write_complete do
+        log_write_complete.call
+      end
+      @socket.on_connect do
+        log_connect.call
+      end
+      @socket.on_connect_failed do
+        log_failed.call
+      end
+      @loop.attach(@socket)
+
+      $log.trace("#{log_tag}: connect: done")
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] fluent-message-sender"
+    end
+  end
+end

data/lib/droonga/forwarder.rb

@@ -15,34 +15,40 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "fluent-logger"
-require "fluent/logger/fluent_logger"
+require "droonga/event_loop"
+require "droonga/fluent_message_sender"
 
 module Droonga
   class Forwarder
-    def initialize
-      @outputs = {}
+    def initialize(loop)
+      @loop = loop
+      @senders = {}
+    end
+
+    def start
+      $log.trace("#{log_tag}: start: start")
+      $log.trace("#{log_tag}: start: done")
     end
 
     def shutdown
       $log.trace("#{log_tag}: shutdown: start")
-      @outputs.each do |dest, output|
-        output[:logger].close if output[:logger]
+      @senders.each_value do |sender|
+        sender.shutdown
       end
       $log.trace("#{log_tag}: shutdown: done")
     end
 
-    def forward(envelope, body, destination)
-      $log.trace("#{log_tag}: post: start")
+    def forward(message, destination)
+      $log.trace("#{log_tag}: forward: start")
       command = destination["type"]
       receiver = destination["to"]
       arguments = destination["arguments"]
-      output(receiver, envelope, body, command, arguments)
-      $log.trace("#{log_tag}: post: done")
+      output(receiver, message, command, arguments)
+      $log.trace("#{log_tag}: forward: done")
     end
 
     private
-    def output(receiver, envelope, body, command, arguments)
+    def output(receiver, message, command, arguments)
       $log.trace("#{log_tag}: output: start")
       unless receiver.is_a?(String) && command.is_a?(String)
         $log.trace("#{log_tag}: output: abort: invalid argument",
@@ -57,65 +63,49 @@ module Droonga
       port = $2
       tag  = $3
       params = $4
-      output = get_output(host, port, params)
-      unless output
-        $log.trace("#{log_tag}: output: abort: no output",
+      sender = find_sender(host, port, params)
+      unless sender
+        $log.trace("#{log_tag}: output: abort: no sender",
                    :host   => host,
                    :port   => port,
                    :params => params)
         return
       end
-      if command =~ /\.result$/
-        message = {
-          inReplyTo: envelope["id"],
-          statusCode: 200,
-          type: command,
-          body: body
-        }
-      else
-        message = envelope.merge(
-          body: body,
-          type: command,
-          arguments: arguments
-        )
-      end
+      override_message = {
+        "type" => command,
+      }
+      override_message["arguments"] = arguments if arguments
+      message = message.merge(override_message)
       output_tag = "#{tag}.message"
       log_info = "<#{receiver}>:<#{output_tag}>"
       $log.trace("#{log_tag}: output: post: start: #{log_info}")
-      output.post(output_tag, message)
+      sender.send(output_tag, message)
       $log.trace("#{log_tag}: output: post: done: #{log_info}")
       $log.trace("#{log_tag}: output: done")
     end
 
-    def get_output(host, port, params)
-      host_port = "#{host}:#{port}"
-      @outputs[host_port] ||= {}
-      output = @outputs[host_port]
+    def find_sender(host, port, params)
+      connection_id = extract_connection_id(params)
+      destination = "#{host}:#{port}"
+      destination << "?#{connection_id}" if connection_id
 
-      has_connection_id = (not params.nil? \
-                           and params =~ /[\?&;]connection_id=([^&;]+)/)
-      if output[:logger].nil? or has_connection_id
-        connection_id = $1
-        if not has_connection_id or output[:connection_id] != connection_id
-          output[:connection_id] = connection_id
-          logger = create_logger(:host => host, :port => port.to_i)
-          # output[:logger] should be closed if it exists beforehand?
-          output[:logger] = logger
-        end
-      end
+      @senders[destination] ||= create_sender(host, port)
+    end
 
-      has_client_session_id = (not params.nil? \
-                               and params =~ /[\?&;]client_session_id=([^&;]+)/)
-      if has_client_session_id
-        client_session_id = $1
-        # some generic way to handle client_session_id is expected
-      end
+    def extract_connection_id(params)
+      return nil unless params
 
-      output[:logger]
+      if /[\?&;]connection_id=([^&;]+)/ =~ params
+        $1
+      else
+        nil
+      end
     end
 
-    def create_logger(options)
-      Fluent::Logger::FluentLogger.new(nil, options)
+    def create_sender(host, port)
+      sender = FluentMessageSender.new(@loop, host, port)
+      sender.start
+      sender
     end
 
     def log_tag