fluent-plugin-droonga 0.0.2 → 0.7.0

data/lib/droonga/handler_plugin.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,21 +15,32 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "droonga/plugin"
+
 module Droonga
-  class HandlerPlugin
-    @@plugins = {}
-    class << self
-      def register(name, handler_class)
-        @@plugins[name] = handler_class
-      end
+  class HandlerPlugin < Plugin
+    extend PluginRegisterable
+
+    def initialize(handler)
+      super()
+      @handler = handler
+      @context = @handler.context
+    end
+
+    def envelope
+      @handler.envelope
+    end
+
+    def emit(value, name=nil)
+      @handler.emit(value, name)
     end
 
-    def initialize(name)
-      @name = name
+    def post(body, destination=nil)
+      @handler.post(body, destination)
     end
 
-    def instantiate(*args)
-      @@plugins[@name].new(*args)
+    def prefer_synchronous?(command)
+      false
     end
   end
 end

data/lib/droonga/input_message.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class InputMessage
+    def initialize(envelope)
+      @envelope = envelope
+    end
+
+    def adapted_envelope
+      # TODO: We can create adapted envelope non-destructively.
+      # If it is not performance issue, it is better that we don't
+      # change envelope destructively. Consider about it later.
+      @envelope
+    end
+
+    def add_route(route)
+      @envelope["via"].push(route)
+    end
+
+    def body
+      @envelope["body"]
+    end
+
+    def body=(body)
+      @envelope["body"] = body
+    end
+
+    def command
+      @envelope["type"]
+    end
+
+    def command=(command)
+      @envelope["type"] = command
+    end
+  end
+end

data/lib/droonga/job_queue.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -67,6 +67,10 @@ module Droonga
       MessagePack.unpack(packed_message)
     end
 
+    def unblock
+      @queue.unblock
+    end
+
     def close
       @queue = nil
       if @database

data/lib/droonga/job_queue_schema.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/lib/droonga/logger.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/lib/droonga/partition.rb

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "serverengine"
+
+require "droonga/server"
+require "droonga/worker"
+require "droonga/processor"
+
+module Droonga
+  class Partition
+    def initialize(options={})
+      @options = options
+      @n_workers = @options[:n_workers] || 0
+      @processor = Processor.new(@options)
+      @supervisor = nil
+    end
+
+    def start
+      @processor.start
+      start_supervisor if @n_workers > 0
+    end
+
+    def shutdown
+      $log.trace("partition: shutdown: start")
+      shutdown_supervisor if @supervisor
+      @processor.shutdown
+      $log.trace("partition: shutdown: done")
+    end
+
+    def process(envelope, synchronous=nil)
+      $log.trace("partition: process: start")
+      @processor.process(envelope, synchronous)
+      $log.trace("partition: process: done")
+    end
+
+    private
+    def start_supervisor
+      @supervisor = ServerEngine::Supervisor.new(Server, Worker) do
+        force_options = {
+          :worker_type   => "process",
+          :workers       => @options[:n_workers],
+          :log_level     => $log.level,
+          :server_process_name => "Server[#{@options[:database]}] #$0",
+          :worker_process_name => "Worker[#{@options[:database]}] #$0"
+        }
+        @options.merge(force_options)
+      end
+      @supervisor_thread = Thread.new do
+        @supervisor.main
+      end
+    end
+
+    def shutdown_supervisor
+      $log.trace("supervisor: shutdown: start")
+      @supervisor.stop(true)
+      $log.trace("supervisor: shutdown: stopped")
+      @supervisor_thread.join
+      $log.trace("supervisor: shutdown: done")
+    end
+  end
+end

data/lib/droonga/pluggable.rb

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Pluggable
+    def shutdown
+      $log.trace("#{log_tag}: shutdown: plugin: start")
+      @plugins.each do |plugin|
+        plugin.shutdown
+      end
+      $log.trace("#{log_tag}: shutdown: plugin: done")
+    end
+
+    def processable?(command)
+      not find_plugin(command).nil?
+    end
+
+    def process(command, *arguments)
+      plugin = find_plugin(command)
+      $log.trace("#{log_tag}: process: start: <#{command}>",
+                 :plugin => plugin.class)
+      if plugin.nil?
+        raise "unknown plugin: <#{command}>: " +
+                "TODO: improve error handling"
+      end
+      plugin.process(command, *arguments)
+      $log.trace("#{log_tag}: process: done: <#{command}>",
+                 :plugin => plugin.class)
+    end
+
+    private
+    def load_plugins(names)
+      @plugins = names.collect do |name|
+        plugin = instantiate_plugin(name)
+        if plugin.nil?
+          raise "unknown plugin: <#{name}>: TODO: improve error handling"
+        end
+        plugin
+      end
+    end
+
+    def find_plugin(command)
+      @plugins.find do |plugin|
+        plugin.processable?(command)
+      end
+    end
+  end
+end

data/lib/droonga/plugin.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,27 +15,29 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "droonga/plugin_registerable"
+
 module Droonga
   class Plugin
-    class << self
-      def load_all
-        $LOAD_PATH.each do |load_path|
-          Dir.glob("#{load_path}/droonga/plugin/*_*.rb") do |path|
-            type, name = File.basename(path, ".rb").split(/_/, 2)
-            plugin = new(type, name)
-            plugin.load
-          end
-        end
-      end
+    def initialize
+    end
+
+    def start
+    end
+
+    def shutdown
     end
 
-    def initialize(type, name)
-      @type = type
-      @name = name
+    def processable?(command)
+      self.class.processable?(command)
     end
 
-    def load
-      require "droonga/plugin/#{@type}_#{@name}"
+    def process(command, *arguments)
+      __send__(self.class.method_name(command), *arguments)
+    rescue => exception
+      Logger.error("error while processing #{command}",
+                   arguments: arguments,
+                   exception: exception)
     end
   end
 end

data/lib/droonga/plugin/{adapter_groonga.rb → adapter/groonga.rb}

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,20 +13,20 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "groonga"
-
-require "droonga/adapter"
+require "droonga/adapter_plugin"
 
 module Droonga
-  class GroongaAdapter < Droonga::Adapter
-    # TODO: AdapterPlugin or something should be defined to avoid conflicts.
-    Droonga::HandlerPlugin.register("select", self)
+  class GroongaAdapter < Droonga::AdapterPlugin
+    repository.register("select", self)
+
     command :select
-    def select(select_request)
+    def select(input_message)
       command = Select.new
+      select_request = input_message.body
       search_request = command.convert_request(select_request)
-      add_route("select_response")
-      post(search_request, "search")
+      input_message.add_route("select_response")
+      input_message.command = "search"
+      input_message.body = search_request
     end
 
     command :select_response

data/lib/droonga/plugin/adapter/groonga/select.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,10 +20,12 @@ module Droonga
         table = select_request["table"]
         result_name = table + "_result"
         match_columns = select_request["match_columns"]
-        match_to = match_columns
+        match_to = match_columns ? match_columns.split(/ *\|\| */) : []
         query = select_request["query"]
-        output_columns = select_request["output_columns"]
+        output_columns = select_request["output_columns"] || ""
         attributes = output_columns.split(/, */)
+        offset = (select_request["offset"] || "0").to_i
+        limit = (select_request["limit"] || "10").to_i
 
         search_request = {
           "queries" => {
@@ -38,6 +40,8 @@ module Droonga
                   "records",
                 ],
                 "attributes" => attributes,
+                "offset" => offset,
+                "limit" => limit,
               },
             }
           }
@@ -76,7 +80,12 @@ module Droonga
           end
 
           header = [status_code, start_time_in_unix_time, elapsed_time]
-          results = [[count], converted_attributes, value["records"]]
+          records = value["records"]
+          if records.empty?
+            results = [[count], converted_attributes]
+          else
+            results = [[count], converted_attributes, records]
+          end
           body = [results]
 
           [header, body]

data/lib/droonga/plugin/collector/basic.rb

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/collector_plugin"
+
+module Droonga
+  class BasicCollector < Droonga::CollectorPlugin
+    repository.register("basic", self)
+
+    UNLIMITED = -1
+
+    command :collector_gather
+    def collector_gather(result)
+      output = body ? body[input_name] : input_name
+      if output.is_a?(Hash)
+        element = output["element"]
+        if element
+          result[element] = apply_output_range(result[element], output)
+          result[element] = apply_output_attributes_and_format(result[element], output)
+        end
+        output = output["output"]
+      end
+      emit(result, output)
+    end
+
+    def apply_output_range(items, output)
+      if items && items.is_a?(Array)
+        offset = output["offset"] || 0
+        unless offset.zero?
+          items = items[offset..-1]
+        end
+
+        limit = output["limit"] || 0
+        unless limit == UNLIMITED
+          items = items[0...limit]
+        end
+      end
+      items
+    end
+
+    def apply_output_attributes_and_format(items, output)
+      attributes = output["attributes"]
+      if attributes
+        format = output["format"]
+        if format == "complex"
+          items.collect! do |item|
+            complex_item = {}
+            attributes.each_with_index do |label, index|
+              complex_item[label] = item[index]
+            end
+            complex_item
+          end
+        else
+          items.collect! do |item|
+            item[0...attributes.size]
+          end
+        end
+      end
+      items
+    end
+
+    command :collector_reduce
+    def collector_reduce(request)
+      return unless request
+      body[input_name].each do |output, elements|
+        value = request
+        old_value = output_values[output]
+        value = reduce(elements, old_value, request) if old_value
+        emit(value, output)
+      end
+    end
+
+    def reduce(elements, *values)
+      result = {}
+      elements.each do |key, deal|
+        reduced_values = nil
+
+        case deal["type"]
+        when "sum"
+          reduced_values = values[0][key] + values[1][key]
+        when "sort"
+          reduced_values = merge(values[0][key], values[1][key], deal["operators"])
+        end
+
+        reduced_values = apply_output_range(reduced_values, "limit" => deal["limit"])
+
+        result[key] = reduced_values
+      end
+      return result
+    end
+
+    def merge(x, y, operators)
+      # Normalize operators at first for optimization.
+      operators ||= []
+      operators = operators.collect do |operator|
+        if operator.is_a?(String)
+          { "operator" => operator }
+        else
+          operator
+        end
+      end
+
+      index = 0
+      y.each do |_y|
+        loop do
+          _x = x[index]
+          break unless _x
+          break if compare(_y, _x, operators)
+          index += 1
+        end
+        x.insert(index, _y)
+        index += 1
+      end
+      return x
+    end
+
+    def compare(x, y, operators)
+      operators.each_with_index do |operator, index|
+        column = operator["column"] || index
+        operator = operator["operator"]
+        _x = x[column]
+        _y = y[column]
+        return true if _x.__send__(operator, _y)
+      end
+      return false
+    end
+  end
+end