fluent-plugin-droonga 0.7.0 → 0.8.0

data/lib/droonga/logger.rb

@@ -24,10 +24,11 @@ module Droonga
       end
     end
 
-    def error(message, additional_information={})
+    def error(message, exception, additional_information={})
       if $log
-        $log.error(message, additional_information)
-        $log.error_backtrace
+        $log.error("#{message}: #{exception.message}(#{exception.class})",
+                   additional_information)
+        $log.error_backtrace(exception.backtrace)
       end
     end
   end

data/lib/droonga/message_pack_packer.rb

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "msgpack"
+
+require "droonga/time_formatter"
+
+module Droonga
+  class MessagePackPacker
+    class << self
+      def pack(object)
+        packer = new
+        packer.pack(object)
+        packer.to_s
+      end
+    end
+
+    MICRO_SECONDS_DECIMAL_PLACE = 6
+
+    def initialize
+      @packer = MessagePack::Packer.new
+    end
+
+    def pack(object)
+      case object
+      when Array
+        @packer.write_array_header(object.size)
+        object.each do |element|
+          pack(element)
+        end
+      when Hash
+        @packer.write_map_header(object.size)
+        object.each do |key, value|
+          pack(key)
+          pack(value)
+        end
+      when Time
+        @packer.write(TimeFormatter.format(object))
+      else
+        @packer.write(object)
+      end
+    end
+
+    def to_s
+      @packer.to_s
+    end
+  end
+end

data/lib/droonga/message_processing_error.rb

@@ -0,0 +1,54 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class MessageProcessingError < StandardError
+    attr_reader :message, :detail
+ 
+    def initialize(message, detail=nil)
+      @message = message
+      @detail = detail
+    end
+
+    def name
+      self.class.name.split("::").last
+    end
+
+    def status_code
+      500
+    end
+
+    def response_body
+      body = {
+        "name"    => name,
+        "message" => @message,
+      }
+      body["detail"] = @detail unless @detail.nil?
+      body
+    end
+  end
+
+  class BadRequest < MessageProcessingError
+    def status_code
+      400
+    end
+  end
+
+  class NotFound < MessageProcessingError
+    def status_code
+      404
+    end
+  end
+end

data/lib/droonga/message_pusher.rb

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "msgpack"
+
+module Droonga
+  class MessagePusher
+    attr_reader :raw_receiver
+    def initialize(loop)
+      @loop = loop
+    end
+
+    def start(base_path)
+      socket_path = "#{base_path}.sock"
+      FileUtils.rm_f(socket_path)
+      @raw_receiver = UNIXServer.new(socket_path)
+      FileUtils.chmod(0600, socket_path)
+    end
+
+    def shutdown
+      $log.trace("#{log_tag}: shutdown: start")
+      socket_path = @raw_receiver.path
+      @raw_receiver.close
+      FileUtils.rm_f(socket_path)
+      $log.trace("#{log_tag}: shutdown: done")
+    end
+
+    def push(message)
+      $log.trace("#{log_tag}: push: start")
+      packed_message = message.to_msgpack
+      path = @raw_receiver.path
+      sender = Coolio::UNIXSocket.connect(path)
+      sender.write(message.to_msgpack)
+      sender.on_write_complete do
+        close
+      end
+      @loop.attach(sender)
+      $log.trace("#{log_tag}: push: done")
+    end
+
+    private
+    def log_tag
+      "message_pusher"
+    end
+  end
+end

data/lib/droonga/message_receiver.rb

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "msgpack"
+
+module Droonga
+  class MessageReceiver
+    def initialize(loop, receiver, &callback)
+      @loop = loop
+      @receiver = Coolio::Server.new(receiver, Coolio::Socket) do |connection|
+        setup_receive_handler(connection)
+      end
+      @callback = callback
+    end
+
+    def start
+      $log.trace("#{log_tag}: start: start")
+      @loop.attach(@receiver)
+      $log.trace("#{log_tag}: start: done")
+    end
+
+    def shutdown
+      $log.trace("#{log_tag}: shutdown: start")
+      @receiver.close
+      $log.trace("#{log_tag}: shutdown: done")
+    end
+
+    private
+    def setup_receive_handler(connection)
+      unpacker = MessagePack::Unpacker.new
+      on_read = lambda do |data|
+        $log.trace("#{log_tag}: on_read: start")
+        unpacker.feed_each(data) do |message|
+          @callback.call(message)
+        end
+        $log.trace("#{log_tag}: on_read: done")
+      end
+      connection.on_read do |data|
+        on_read.call(data)
+      end
+    end
+
+    def log_tag
+      "message_receiver"
+    end
+  end
+end

data/lib/droonga/output_adapter.rb

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/pluggable"
+require "droonga/output_adapter_plugin"
+require "droonga/output_message"
+
+module Droonga
+  class OutputAdapter
+    include Pluggable
+
+    def initialize(dispatcher, options={})
+      @dispatcher = dispatcher
+      load_plugins(options[:plugins] || [])
+    end
+
+    def adapt(message)
+      adapted_message = message
+      message["via"].reverse_each do |command|
+        @plugins.each do |plugin|
+          next unless plugin.processable?(command)
+          output_message = OutputMessage.new(adapted_message)
+          process(command, output_message)
+          adapted_message = output_message.adapted_message
+        end
+      end
+      adapted_message
+    end
+
+    private
+    def instantiate_plugin(name)
+      OutputAdapterPlugin.repository.instantiate(name, @dispatcher)
+    end
+
+    def log_tag
+      "output-adapter"
+    end
+  end
+end

data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb}

@@ -18,7 +18,7 @@
 require "droonga/plugin"
 
 module Droonga
-  class AdapterPlugin < Plugin
+  class OutputAdapterPlugin < Plugin
     extend PluginRegisterable
 
     def initialize(dispatcher)
@@ -26,26 +26,8 @@ module Droonga
       @dispatcher = dispatcher
     end
 
-    def add_route(route)
-      @dispatcher.add_route(route)
-    end
-
-    def post(body, destination=nil)
-      @dispatcher.post(body, destination)
-    end
-
-    def emit(value, name=nil)
-      if name
-        @output_values[name] = value
-      else
-        @output_values = value
-      end
-    end
-
-    def process(command, message)
-      @output_values = {}
-      super(command, message)
-      post(@output_values) unless @output_values.empty?
+    def forward(message, destination)
+      @dispatcher.forward(message, destination)
     end
   end
 end

data/lib/droonga/output_message.rb

@@ -0,0 +1,37 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class OutputMessage
+    def initialize(raw_message)
+      @raw_message = raw_message
+    end
+
+    def adapted_message
+      # TODO: We can create adapted message non-destructively.
+      # If it is not performance issue, it is better that we don't
+      # change message destructively. Consider about it later.
+      @raw_message
+    end
+
+    def body
+      @raw_message["body"]
+    end
+
+    def body=(body)
+      @raw_message["body"] = body
+    end
+  end
+end

data/lib/droonga/partition.rb

@@ -19,36 +19,57 @@ require "serverengine"
 
 require "droonga/server"
 require "droonga/worker"
+require "droonga/event_loop"
+require "droonga/message_pusher"
 require "droonga/processor"
 
 module Droonga
   class Partition
-    def initialize(options={})
+    def initialize(loop, options={})
       @options = options
       @n_workers = @options[:n_workers] || 0
-      @processor = Processor.new(@options)
+      @loop = loop
+      @message_pusher = MessagePusher.new(@loop)
+      @processor = Processor.new(@loop, @message_pusher, @options)
       @supervisor = nil
     end
 
     def start
+      ensure_database
       @processor.start
+      base_path = @options[:database]
+      @message_pusher.start(base_path)
       start_supervisor if @n_workers > 0
     end
 
     def shutdown
       $log.trace("partition: shutdown: start")
       shutdown_supervisor if @supervisor
+      @message_pusher.shutdown
       @processor.shutdown
       $log.trace("partition: shutdown: done")
     end
 
-    def process(envelope, synchronous=nil)
+    def process(message)
       $log.trace("partition: process: start")
-      @processor.process(envelope, synchronous)
+      @processor.process(message)
       $log.trace("partition: process: done")
     end
 
     private
+    def ensure_database
+      database_path = @options[:database]
+      return if File.exist?(database_path)
+      FileUtils.mkdir_p(File.dirname(database_path))
+      context = Groonga::Context.new
+      begin
+        context.create_database(database_path) do
+        end
+      ensure
+        context.close
+      end
+    end
+
     def start_supervisor
       @supervisor = ServerEngine::Supervisor.new(Server, Worker) do
         force_options = {
@@ -56,7 +77,8 @@ module Droonga
           :workers       => @options[:n_workers],
           :log_level     => $log.level,
           :server_process_name => "Server[#{@options[:database]}] #$0",
-          :worker_process_name => "Worker[#{@options[:database]}] #$0"
+          :worker_process_name => "Worker[#{@options[:database]}] #$0",
+          :message_receiver => @message_pusher.raw_receiver,
         }
         @options.merge(force_options)
       end