droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/planner.rb

@@ -22,26 +22,92 @@ module Droonga
     include Loggable
     include ErrorMessages
 
+    attr_writer :write, :single_operation, :use_all_replicas, :collector_class
+
     def initialize(dataset)
       @dataset = dataset
+      @write = false
+      @single_operation = false
+      @use_all_replicas = false
+      @collector_class = nil
     end
 
-    def plan(message)
-      raise NotImplemented, "#{self.class.name}\##{__method__} must implement."
+    def plan(message, params={})
+      options = {
+        :record => params[:record],
+      }
+
+      #TODO: We don't have to reduce results of the message when
+      #      the message doesn't have "replyTo" information, because:
+      #
+      #       * Currently the "super step" mecahnism is not
+      #         implemented yet.
+      #       * So, reduced results won't be forwarded to other
+      #         handlers directly. Results will be forwarded to
+      #         the sender as the "response".
+      #       * So, if "replyTo" information is not given, the
+      #         reduced result will have no receiver.
+      #
+      #      However, in the future after the "super step" mechanism
+      #      is introduced, reduced results can be required even if
+      #      the request message have no "replyTo" information.
+      #      Then we must update this logic.
+      if @collector_class and message["replyTo"]
+        reduce_key = "result"
+        options[:reduce] = {
+          reduce_key => @collector_class.operator,
+        }
+      end
+
+      if options[:record] or single_operation?
+        scatter(message, options)
+      else
+        broadcast(message, options)
+      end
     end
 
     private
-    def scatter(message, record, options={})
+    def write?
+      @write
+    end
+
+    def single_operation?
+      return false if write?
+      @single_operation
+    end
+
+    def use_all_replicas?
+      write? or @use_all_replicas
+    end
+
+    def scatter(message, options={})
       planner = DistributedCommandPlanner.new(@dataset, message)
-      planner.scatter(record)
-      planner.reduce(options[:reduce])
+      scatter_options = {
+        :write => write?,
+        :record => options[:record],
+      }
+      if single_operation?
+        scatter_options[:slice]   = "random"
+        scatter_options[:replica] = "random"
+      end
+      planner.scatter(scatter_options)
+      planner.reduce(options[:reduce]) if options[:reduce]
       planner.plan
     end
 
     def broadcast(message, options={})
       planner = DistributedCommandPlanner.new(@dataset, message)
-      planner.broadcast(:write => options[:write])
-      planner.reduce(options[:reduce])
+      broadcast_options = {
+        :write => write?,
+      }
+      if use_all_replicas?
+        broadcast_options[:replica] = "all"
+      elsif single_operation?
+        broadcast_options[:slice]   = "random"
+        broadcast_options[:replica] = "random"
+      end
+      planner.broadcast(broadcast_options)
+      planner.reduce(options[:reduce]) if options[:reduce]
       planner.plan
     end
   end

data/lib/droonga/plugin/async_command.rb

@@ -0,0 +1,154 @@
+# Copyright (C) 2014-2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "coolio"
+
+require "droonga/loggable"
+require "droonga/handler"
+require "droonga/error_messages"
+
+module Droonga
+  module Plugins
+    module AsyncCommand
+      class Request
+        def initialize(message)
+          @message = message
+        end
+
+        def need_start?
+          reply_to
+        end
+
+        def id
+          @message["id"]
+        end
+
+        def dataset
+          @message.raw["dataset"]
+        end
+
+        def reply_to
+          (@message.raw["replyTo"] || {})["to"]
+        end
+
+        def request
+          @message.request
+        end
+      end
+
+      class Handler < Droonga::Handler
+        def handle(message)
+          request = request_class.new(message)
+          if request.need_start?
+            start(request)
+            {
+              "started" => true,
+            }
+          else
+            {
+              "started" => false,
+            }
+          end
+        end
+
+        private
+        def request_class
+          #XXX override me!
+          Request
+        end
+
+        def start(request)
+          #XXX override me!
+          # handler = MyAsyncHandler.new(loop, messenger, request)
+          # handler.start
+        end
+      end
+
+      class AsyncHandler
+        include Loggable
+
+        def initialize(loop, messenger, request)
+          @loop = loop
+          @messenger = messenger
+          @request = request
+        end
+
+        def start
+          #XXX override me!!
+          on_start
+          on_finish
+        end
+
+        private
+        def prefix
+          "" #XXX override me!!
+        end
+
+        def on_start
+          setup_forward_data
+          forward("#{prefix}.start")
+        end
+
+        def on_finish
+          forward("#{prefix}.end")
+        end
+
+        def setup_forward_data
+          @base_forward_message = {
+            "inReplyTo" => @request.id,
+            "dataset"   => @request.dataset,
+          }
+          @forward_to = @request.reply_to
+        end
+
+        def error_name
+          "Failure" #XXX override me!!
+        end
+
+        def error_message
+          "failed to do" #XXX override me!!
+        end
+
+        def error(name, message)
+          message = {
+            "statusCode" => ErrorMessages::InternalServerError::STATUS_CODE,
+            "body" => {
+              "name"    => name,
+              "message" => message,
+            },
+          }
+          error_message = @base_forward_message.merge(message)
+          @messenger.forward(error_message,
+                             "to"   => @forward_to,
+                             "type" => "#{prefix}.error")
+        end
+
+        def forward(type, body=nil)
+          forward_message = @base_forward_message
+          if body
+            forward_message = forward_message.merge("body" => body)
+          end
+          @messenger.forward(forward_message,
+                             "to"   => @forward_to,
+                             "type" => type)
+        end
+
+        def log_tag
+          "[#{Process.ppid}] async-handler"
+        end
+      end
+    end
+  end
+end

data/lib/droonga/plugins/catalog.rb

@@ -32,6 +32,7 @@ module Droonga
 
       define_single_step do |step|
         step.name = "catalog.fetch"
+        step.single_operation = true
         step.handler = FetchHandler
         step.collector = Collectors::Or
       end

data/lib/droonga/plugins/crud.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -70,8 +70,8 @@ module Droonga
         end
 
         class InvalidValue < BadRequest
-          def initialize(column, value, request)
-            super("The column #{column.inspect} cannot store the value #{value.inspect}.",
+          def initialize(column, value, reason, request)
+            super("The column #{column.inspect} cannot store the value #{value.inspect}: #{reason}",
                   request)
           end
         end
@@ -107,7 +107,8 @@ module Droonga
         def add_record(table, request)
           record = nil
           if table.support_key?
-            record = table.add(request["key"])
+            key = normalize_record_key(request["key"], table)
+            record = table.add(key)
           else
             record = table.add
           end
@@ -116,16 +117,31 @@ module Droonga
               record[column] = value
             rescue ::Groonga::InvalidArgument => error
               record.delete if record.added?
-              raise InvalidValue.new(column, value, request)
+              raise InvalidValue.new(column, value, error.message, request)
             rescue ArgumentError => error
               record.delete if record.added?
-              raise InvalidValue.new(column, value, request)
+              raise InvalidValue.new(column, value, error.message, request)
+            rescue TypeError => error
+              record.delete if record.added?
+              raise InvalidValue.new(column, value, error.message, request)
             rescue ::Groonga::NoSuchColumn => error
               record.delete if record.added?
               raise UnknownColumn.new(column, request["table"], request)
             end
           end
         end
+
+        def normalize_record_key(key, table)
+          case table.domain.name
+          when "Int8",  "UInt8",
+               "Int16", "UInt16",
+               "Int32", "UInt32",
+               "Int64", "UInt64"
+            key.to_i
+          else
+            key.to_s
+          end
+        end
       end
 
       define_single_step do |step|

data/lib/droonga/plugins/dump.rb

@@ -13,10 +13,13 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "fiber"
 require "groonga"
 
 require "droonga/plugin"
+require "droonga/plugin/async_command"
 require "droonga/error_messages"
+require "droonga/database_scanner"
 
 module Droonga
   module Plugins
@@ -35,149 +38,99 @@ module Droonga
       extend Plugin
       register("dump")
 
-      class Handler < Droonga::Handler
-        def handle(message)
-          request = Request.new(message)
-          if request.need_dump?
-            dumper = Dumper.new(@context, loop, messenger, request)
-            dumper.start_dump
-            true
-          else
-            false
-          end
-        end
-      end
-
-      class Request
-        def initialize(message)
-          @message = message
-        end
-
-        def need_dump?
-          reply_to
-        end
+      class Request < AsyncCommand::Request
+        DEFAULT_MESSAGES_PER_SECOND = 10000
 
-        def id
-          @message["id"]
+        def messages_per_second
+          request = (@message.request || {})
+          minimum_messages_per_second = 10
+          [
+            minimum_messages_per_second,
+            (request["messagesPerSecond"] || DEFAULT_MESSAGES_PER_SECOND).to_i,
+          ].max
         end
+      end
 
-        def dataset
-          @message.raw["dataset"]
+      class Handler < AsyncCommand::Handler
+        private
+        def request_class
+          Request
         end
 
-        def reply_to
-          (@message.raw["replyTo"] || {})["to"]
-        end
-
-        def messages_per_seconds
-          request = (@message.request || {})
-          minimum_messages_per_seconds = 10
-          [
-            minimum_messages_per_seconds,
-            (request["messagesPerSecond"] || 10000).to_i,
-          ].max
+        def start(request)
+          dumper = Dumper.new(@context, loop, messenger, request)
+          dumper.start
         end
       end
 
-      class Dumper
-        include Loggable
+      class Dumper < AsyncCommand::AsyncHandler
+        include DatabaseScanner
 
         def initialize(context, loop, messenger, request)
           @context = context
-          @loop = loop
-          @messenger = messenger
-          @request = request
+          super(loop, messenger, request)
         end
 
-        def start_dump
-          setup_forward_data
-
-          forward("dump.start")
+        def start
+          on_start
 
-          dumper = Fiber.new do
+          runner = Fiber.new do
+            forecast
             dump_schema
             dump_records
             dump_indexes
-            forward("dump.end")
-          end
-
-          on_error = lambda do |exception|
-            message = "failed to dump"
-            logger.exception(message, $!)
-            error("DumpFailure", message)
+            on_finish
           end
 
           timer = Coolio::TimerWatcher.new(0.1, true)
           timer.on_timer do
-            begin
-              dumper.resume
-            rescue FiberError
-              timer.detach
-            rescue
+            if runner.alive?
+              begin
+                runner.resume
+              rescue
+                timer.detach
+                logger.trace("start: timer detached by unexpected exception",
+                             :watcher => timer)
+                logger.exception(error_message, $!)
+                error(error_name, error_message)
+              end
+            else
               timer.detach
-              on_error.call($!)
+              logger.trace("start: timer detached by unexpected exception",
+                           :watcher => timer)
             end
           end
 
           @loop.attach(timer)
+          logger.trace("start: timer attached",
+                       :watcher => timer)
         end
 
         private
-        def setup_forward_data
-          @base_forward_message = {
-            "inReplyTo" => @request.id,
-            "dataset"   => @request.dataset,
-          }
-          @forward_to = @request.reply_to
-          @n_forwarded_messages = 0
-          @messages_per_100msec = @request.messages_per_seconds / 10
+        def prefix
+          "dump"
         end
 
-        def error(name, message)
-          message = {
-            "statusCode" => ErrorMessages::InternalServerError::STATUS_CODE,
-            "body" => {
-              "name"    => name,
-              "message" => message,
-            },
-          }
-          error_message = @base_forward_message.merge(message)
-          @messenger.forward(error_message,
-                             "to"   => @forward_to,
-                             "type" => "dump.error")
+        def error_name
+          "DumpFailure"
         end
 
-        def forward(type, body=nil)
-          forward_message = @base_forward_message
-          if body
-            forward_message = forward_message.merge("body" => body)
-          end
-          @messenger.forward(forward_message,
-                             "to"   => @forward_to,
-                             "type" => type)
+        def error_message
+          "failed to dump"
+        end
 
-          @n_forwarded_messages += 1
-          @n_forwarded_messages %= @messages_per_100msec
-          Fiber.yield if @n_forwarded_messages.zero?
+        def forecast
+          forward("#{prefix}.forecast", "nMessages" => total_n_objects)
         end
 
         def dump_schema
-          reference_tables = []
           each_table do |table|
-            if reference_table?(table)
-              reference_tables << table
-              next
-            end
-            dump_table(table)
-          end
-
-          reference_tables.each do |table|
             dump_table(table)
           end
         end
 
         def dump_table(table)
-          forward("dump.table", table_body(table))
+          forward("#{prefix}.table", table_body(table))
 
           columns = table.columns.sort_by(&:name)
           columns.each do |column|
@@ -210,7 +163,7 @@ module Droonga
         end
 
         def dump_column(column)
-          forward("dump.column", column_body(column))
+          forward("#{prefix}.column", column_body(column))
         end
 
         def column_body(column)
@@ -270,7 +223,7 @@ module Droonga
                 "key"    => record.key,
                 "values" => values,
               }
-              forward("dump.record", body)
+              forward("#{prefix}.record", body)
             end
           end
         end
@@ -286,6 +239,8 @@ module Droonga
                 element
               end
             end
+          when Hash
+            value["_key"]
           else
             value
           end
@@ -297,41 +252,17 @@ module Droonga
           end
         end
 
-        def each_table
-          options = {
-            :ignore_missing_object => true,
-            :order_by => :key,
-          }
-          @context.database.each(options) do |object|
-            next unless table?(object)
-            yield(object)
-          end
-        end
-
-        def table?(object)
-          object.is_a?(::Groonga::Table)
-        end
-
-        def index_only_table?(table)
-          table.columns.all? do |column|
-            index_column?(column)
-          end
-        end
-
-        def reference_table?(table)
-          table.support_key? and table?(table.domain)
-        end
-
-        def index_column?(column)
-          column.is_a?(::Groonga::IndexColumn)
+        def setup_forward_data
+          super
+          @n_forwarded_messages = 0
+          @messages_per_100msec = @request.messages_per_second / 10
         end
 
-        def each_index_columns
-          each_table do |table|
-            table.columns.each do |column|
-              yield(column) if index_column?(column)
-            end
-          end
+        def forward(type, body=nil)
+          super
+          @n_forwarded_messages += 1
+          @n_forwarded_messages %= @messages_per_100msec
+          Fiber.yield if @n_forwarded_messages.zero?
         end
 
         def log_tag