fluent-plugin-droonga 0.0.2 → 0.7.0

data/lib/droonga/distributor.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/pluggable"
+require "droonga/distributor_plugin"
+
+module Droonga
+  class Distributor
+    include Pluggable
+
+    def initialize(dispatcher, options={})
+      @dispatcher = dispatcher
+      @plugins = []
+      @options = options
+      # TODO: don't put the default distributions
+      load_plugins(options[:distributors] || ["search", "crud", "groonga", "watch"])
+    end
+
+    def distribute(envelope)
+      command = envelope["type"]
+      process(command, envelope)
+    end
+
+    def post(message)
+      @dispatcher.handle(message, [])
+    end
+
+    private
+    def instantiate_plugin(name)
+      DistributorPlugin.repository.instantiate(name, self)
+    end
+
+    def log_tag
+      "[#{Process.pid}] distributor"
+    end
+  end
+end

data/lib/droonga/distributor_plugin.rb

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+
+module Droonga
+  class DistributorPlugin < Plugin
+    extend PluginRegisterable
+
+    def initialize(distributor)
+      super()
+      @distributor = distributor
+    end
+
+    # TODO: consider better name
+    def post(message)
+      @distributor.post(message)
+    end
+
+    def scatter_all(envelope, key)
+      message = [{
+        "command"=> envelope["type"],
+        "dataset"=> envelope["dataset"],
+        "body"=> envelope["body"],
+        "key"=> key,
+        "type"=> "scatter",
+        "replica"=> "all",
+        "post"=> true
+      }]
+      post(message)
+    end
+
+    def broadcast_all(envelope)
+      distirubte_message = [{
+        "command"=> envelope["type"],
+        "dataset"=> envelope["dataset"],
+        "body"=> envelope["body"],
+        "type"=> "broadcast",
+        "replica"=> "all",
+        "post"=> true
+      }]
+      post(distirubte_message)
+    end
+  end
+end

data/lib/droonga/engine.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,69 +15,28 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "serverengine"
-require "msgpack"
-require "cool.io"
-
-require "droonga/server"
-require "droonga/worker"
-require "droonga/executor"
+require "droonga/logger"
+require "droonga/dispatcher"
 
 module Droonga
   class Engine
-    DEFAULT_OPTIONS = {
-      :queue_name => "DroongaQueue",
-      :n_workers  => 0,
-    }
-
     def initialize(options={})
-      @options = DEFAULT_OPTIONS.merge(options)
+      @options = options
+      @dispatcher = Dispatcher.new(@options)
     end
 
     def start
-      if @options[:database] && !@options[:database].empty?
-        Droonga::JobQueue.ensure_schema(@options[:database],
-                                        @options[:queue_name])
-      end
-      start_supervisor if @options[:n_workers] > 0
-      @executor = Executor.new(@options)
+      @dispatcher.start
     end
 
     def shutdown
       $log.trace("engine: shutdown: start")
-      @executor.shutdown if @executor
-      shutdown_supervisor if @supervisor
+      @dispatcher.shutdown
       $log.trace("engine: shutdown: done")
     end
 
-    def emit(tag, time, record, synchronous=nil)
-      $log.trace("[#{Process.pid}] tag: <#{tag}> caller: <#{caller.first}>")
-      @executor.dispatch(tag, time, record, synchronous)
-    end
-
-    private
-    def start_supervisor
-      @supervisor = ServerEngine::Supervisor.new(Server, Worker) do
-        force_options = {
-          :worker_type   => "process",
-          :workers       => @options[:n_workers],
-          :log_level     => $log.level,
-          :server_process_name => "Server[#{@options[:database]}] #$0",
-          :worker_process_name => "Worker[#{@options[:database]}] #$0"
-        }
-        @options.merge(force_options)
-      end
-      @supervisor_thread = Thread.new do
-        @supervisor.main
-      end
-    end
-
-    def shutdown_supervisor
-      $log.trace("supervisor: shutdown: start")
-      @supervisor.stop(true)
-      $log.trace("supervisor: shutdown: stopped")
-      @supervisor_thread.join
-      $log.trace("supervisor: shutdown: done")
+    def process(envelope)
+      @dispatcher.handle_envelope(envelope)
     end
   end
 end

data/lib/droonga/farm.rb

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/partition"
+
+module Droonga
+  class Farm
+    def initialize(name)
+      @name = name
+      @partitions = {}
+      Droonga.catalog.get_partitions(name).each do |partition_name, options|
+        partition = Droonga::Partition.new(options)
+        @partitions[partition_name] = partition
+      end
+    end
+
+    def start
+      @partitions.each_value do |partition|
+        partition.start
+      end
+    end
+
+    def shutdown
+      @partitions.each_value do |partition|
+        partition.shutdown
+      end
+    end
+
+    def process(partition_name, envelope, synchronous)
+      @partitions[partition_name].process(envelope, synchronous)
+    end
+  end
+end

data/lib/droonga/forwarder.rb

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "fluent-logger"
+require "fluent/logger/fluent_logger"
+
+module Droonga
+  class Forwarder
+    def initialize
+      @outputs = {}
+    end
+
+    def shutdown
+      $log.trace("#{log_tag}: shutdown: start")
+      @outputs.each do |dest, output|
+        output[:logger].close if output[:logger]
+      end
+      $log.trace("#{log_tag}: shutdown: done")
+    end
+
+    def forward(envelope, body, destination)
+      $log.trace("#{log_tag}: post: start")
+      command = destination["type"]
+      receiver = destination["to"]
+      arguments = destination["arguments"]
+      output(receiver, envelope, body, command, arguments)
+      $log.trace("#{log_tag}: post: done")
+    end
+
+    private
+    def output(receiver, envelope, body, command, arguments)
+      $log.trace("#{log_tag}: output: start")
+      unless receiver.is_a?(String) && command.is_a?(String)
+        $log.trace("#{log_tag}: output: abort: invalid argument",
+                   :receiver => receiver,
+                   :command  => command)
+        return
+      end
+      unless receiver =~ /\A(.*):(\d+)\/(.*?)(\?.+)?\z/
+        raise "format: hostname:port/tag(?params)"
+      end
+      host = $1
+      port = $2
+      tag  = $3
+      params = $4
+      output = get_output(host, port, params)
+      unless output
+        $log.trace("#{log_tag}: output: abort: no output",
+                   :host   => host,
+                   :port   => port,
+                   :params => params)
+        return
+      end
+      if command =~ /\.result$/
+        message = {
+          inReplyTo: envelope["id"],
+          statusCode: 200,
+          type: command,
+          body: body
+        }
+      else
+        message = envelope.merge(
+          body: body,
+          type: command,
+          arguments: arguments
+        )
+      end
+      output_tag = "#{tag}.message"
+      log_info = "<#{receiver}>:<#{output_tag}>"
+      $log.trace("#{log_tag}: output: post: start: #{log_info}")
+      output.post(output_tag, message)
+      $log.trace("#{log_tag}: output: post: done: #{log_info}")
+      $log.trace("#{log_tag}: output: done")
+    end
+
+    def get_output(host, port, params)
+      host_port = "#{host}:#{port}"
+      @outputs[host_port] ||= {}
+      output = @outputs[host_port]
+
+      has_connection_id = (not params.nil? \
+                           and params =~ /[\?&;]connection_id=([^&;]+)/)
+      if output[:logger].nil? or has_connection_id
+        connection_id = $1
+        if not has_connection_id or output[:connection_id] != connection_id
+          output[:connection_id] = connection_id
+          logger = create_logger(:host => host, :port => port.to_i)
+          # output[:logger] should be closed if it exists beforehand?
+          output[:logger] = logger
+        end
+      end
+
+      has_client_session_id = (not params.nil? \
+                               and params =~ /[\?&;]client_session_id=([^&;]+)/)
+      if has_client_session_id
+        client_session_id = $1
+        # some generic way to handle client_session_id is expected
+      end
+
+      output[:logger]
+    end
+
+    def create_logger(options)
+      Fluent::Logger::FluentLogger.new(nil, options)
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] forwarder"
+    end
+  end
+end

data/lib/droonga/handler.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,75 +15,52 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "groonga"
+
+require "droonga/forwarder"
+require "droonga/pluggable"
 require "droonga/handler_plugin"
-require "droonga/command_mapper"
-require "droonga/logger"
 
 module Droonga
   class Handler
-    class << self
-      def inherited(sub_class)
-        super
-        sub_class.instance_variable_set(:@command_mapper, CommandMapper.new)
-      end
+    include Pluggable
 
-      def command(name_or_map)
-        @command_mapper.register(name_or_map)
-      end
-
-      def method_name(command)
-        @command_mapper[command]
-      end
+    attr_reader :context, :envelope, :name
 
-      def handlable?(command)
-        not method_name(command).nil?
-      end
-    end
-
-    def initialize(worker)
-      @worker = worker
-      @context = @worker.context
-    end
-
-    def post(body, destination=nil)
-      @worker.post(body, destination)
-    end
-
-    def envelope
-      @worker.envelope
-    end
-
-    def add_route(route)
-      @worker.add_route(route)
+    def initialize(options={})
+      @options = options
+      @name = options[:name]
+      @database_name = options[:database]
+      prepare
     end
 
     def shutdown
+      $log.trace("#{log_tag}: shutdown: start")
+      super
+      @forwarder.shutdown
+      if @database
+        @database.close
+        @context.close
+        @database = @context = nil
+      end
+      $log.trace("#{log_tag}: shutdown: done")
     end
 
-    def handlable?(command)
-      self.class.handlable?(command)
-    end
-
-    def invoke(command, request, *arguments)
-      __send__(self.class.method_name(command), request, *arguments)
-    rescue => exception
-      Logger.error("error while handling #{command}",
-                   request: request,
-                   arguments: arguments,
-                   exception: exception)
+    def prefer_synchronous?(command)
+      find_plugin(command).prefer_synchronous?(command)
     end
 
-    def handle(command, request, *arguments)
-      unless try_handle_as_internal_message(command, request, arguments)
-        @task = {}
-        @output_values = {}
-        invoke(command, request, *arguments)
-        post(@output_values) unless @output_values.empty?
+    def process(envelope)
+      $log.trace("#{log_tag}: process: start")
+      body, command, arguments = parse_envelope(envelope)
+      plugin = find_plugin(command)
+      if plugin.nil?
+        $log.trace("#{log_tag}: process: done: no plugin: <#{command}>")
+        return
       end
-    end
-
-    def prefer_synchronous?(command)
-      return false
+      process_command(plugin, command, body, arguments)
+      $log.trace("#{log_tag}: process: done: <#{command}>",
+                 :plugin => plugin.class)
     end
 
     def emit(value, name = nil)
@@ -98,7 +75,31 @@ module Droonga
       @output_values[name] = value
     end
 
-    def try_handle_as_internal_message(command, request, arguments)
+    def post(message, destination)
+      @forwarder.forward(envelope, message, destination)
+    end
+
+    private
+    def parse_envelope(envelope)
+      @envelope = envelope
+      envelope["via"] ||= []
+      [envelope["body"], envelope["type"], envelope["arguments"]]
+    end
+
+    def prepare
+      if @database_name && !@database_name.empty?
+        @context = Groonga::Context.new
+        @database = @context.open_database(@database_name)
+      end
+      load_plugins(@options[:handlers] || [])
+      @forwarder = Forwarder.new
+    end
+
+    def instantiate_plugin(name)
+      HandlerPlugin.repository.instantiate(name, self)
+    end
+
+    def process_command(plugin, command, request, arguments)
       return false unless request.is_a? Hash
 
       @task = request["task"]
@@ -115,14 +116,18 @@ module Droonga
       @input_name = request["name"]
       @descendants = request["descendants"]
 
-      invoke(command, @body, *arguments)
+      plugin.process(command, @body, *arguments)
       output if @descendants
       true
     end
 
     def output
       result = @task["values"]
-      post(result, @component["post"]) if @component["post"]
+      if @component["post"]
+        destination = @component["post"]
+        destination = envelope["replyTo"] if destination == true
+        post(result, destination)
+      end
       @descendants.each do |name, dests|
         message = {
           "id" => @id,
@@ -131,10 +136,14 @@ module Droonga
         }
         dests.each do |routes|
           routes.each do |route|
-            post(message, "to"=>route, "type"=>"proxy")
+            post(message, "to"=>route, "type"=>"dispatcher")
           end
         end
       end
     end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] handler"
+    end
   end
 end