fluent-plugin-droonga 0.7.0 → 0.8.0

data/lib/droonga/plugin/distributor/groonga.rb

@@ -22,16 +22,16 @@ module Droonga
     repository.register("groonga", self)
 
     command :table_create
-    def table_create(envelope)
-      unless envelope["dataset"]
+    def table_create(message)
+      unless message["dataset"]
         raise "dataset must be set. FIXME: This error should return client."
       end
-      broadcast_all(envelope)
+      broadcast_all(message)
     end
 
     command :column_create
-    def column_create(envelope)
-      broadcast_all(envelope)
+    def column_create(message)
+      broadcast_all(message)
     end
   end
 end

data/lib/droonga/plugin/distributor/search.rb

@@ -16,258 +16,16 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 require "droonga/distributor_plugin"
+require "droonga/plugin/distributor/distributed_search_planner"
 
 module Droonga
   class SearchDistributor < Droonga::DistributorPlugin
     repository.register("search", self)
 
     command :search
-    def search(envelope)
-      message = []
-      input_names = []
-      output_names = []
-      output_mapper = {}
-
-      request = envelope["body"]
-      request["queries"].each do |input_name, query|
-        output = query["output"]
-        # Skip reducing phase for a result with no output.
-        next unless output
-
-        input_names << input_name
-        output_name = input_name + "_reduced"
-        output_names << output_name
-        output_mapper[output_name] = {
-          "output" => input_name,
-        }
-
-        # The collector module supports only "simple" format search results.
-        # So we have to override the format and restore it on the gathering
-        # phase.
-        final_format = output["format"] || "simple"
-        output["format"] = "simple"
-
-        final_offset, final_limit = calculate_offset_and_limit!(query)
-
-        elements = {}
-        output["elements"].each do |element|
-          case element
-          when "count"
-            elements[element] = {
-              "type" => "sum",
-            }
-          when "records"
-            # Skip reducing phase for a result with no record output.
-            next if final_limit.zero?
-
-            # Append sort key attributes to the list of output attributes
-            # temporarily, for the reducing phase. After all extra columns
-            # are removed on the gathering phase.
-            final_attributes = collect_output_attributes(output["attributes"])
-            output["attributes"] = format_attributes_to_array_style(output["attributes"])
-            output["attributes"] += collect_sort_attributes(output["attributes"], query["sortBy"])
- 
-            elements[element] = sort_reducer(output["attributes"], query["sortBy"])
-            # On the reducing phase, we apply only "limit". We cannot apply
-            # "offset" on this phase because the collecter merges a pair of
-            # results step by step even if there are three or more results.
-            # Instead, we apply "offset" on the gethering phase.
-            elements[element]["limit"] = output["limit"]
-
-            output_mapper[output_name]["element"] = element
-            output_mapper[output_name]["offset"] = final_offset
-            output_mapper[output_name]["limit"] = final_limit
-            output_mapper[output_name]["format"] = final_format
-            output_mapper[output_name]["attributes"] = final_attributes
-          end
-        end
-
-        reducer = {
-          "type" => "reduce",
-          "body" => {
-            input_name => {
-              output_name => elements,
-            },
-          },
-          "inputs" => [input_name], # XXX should be placed in the "body"?
-          "outputs" => [output_name], # XXX should be placed in the "body"?
-        }
-        message << reducer
-      end
-      gatherer = {
-        "type" => "gather",
-        "body" => output_mapper,
-        "inputs" => output_names, # XXX should be placed in the "body"?
-        "post" => true, # XXX should be placed in the "body"?
-      }
-      message << gatherer
-      searcher = {
-        "type" => "broadcast",
-        "command" => "search", # XXX should be placed in the "body"?
-        "dataset" => envelope["dataset"] || request["dataset"],
-        "body" => request,
-        "outputs" => input_names, # XXX should be placed in the "body"?
-        "replica" => "random", # XXX should be placed in the "body"?
-      }
-      message.push(searcher)
-      post(message)
-    end
-
-    private
-    UNLIMITED = -1
-
-    def calculate_offset_and_limit!(query)
-      rich_sort = query["sortBy"].is_a?(Hash)
-
-      have_records = false
-      if query["output"] &&
-           query["output"]["elements"].is_a?(Array) &&
-           query["output"]["elements"].include?("records")
-        have_records = true
-      end
-
-      # Offset for workers must be zero, because we have to apply "limit" and
-      # "offset" on the last gapthering phase instaed of each reducing phase.
-      sort_offset = 0
-      if rich_sort
-        sort_offset = query["sortBy"]["offset"] || 0
-        query["sortBy"]["offset"] = 0
-      end
-
-      output_offset = query["output"]["offset"] || 0
-      query["output"]["offset"] = 0 if have_records
-
-      final_offset = sort_offset + output_offset
-
-      # We have to calculate limit based on offset.
-      # <A, B = limited integer (0...MAXINT)>
-      # | sort limit | output limit | => | worker's sort limit      | worker's output limit   | final limit |
-      # =============================    ====================================================================
-      # | UNLIMITED  | UNLIMITED    | => | UNLIMITED                | UNLIMITED               | UNLIMITED   |
-      # | UNLIMITED  | B            | => | final_offset + B         | final_offset + B        | B           |
-      # | A          | UNLIMITED    | => | final_offset + A         | final_offset + A        | A           |
-      # | A          | B            | => | final_offset + min(A, B) | final_offset + min(A, B)| min(A, B)   |
-      sort_limit = UNLIMITED
-      if rich_sort
-        sort_limit = query["sortBy"]["limit"] || UNLIMITED
-      end
-      output_limit = query["output"]["limit"] || 0
-
-      final_limit = 0
-      if sort_limit == UNLIMITED && output_limit == UNLIMITED
-        final_limit = UNLIMITED
-        query["output"]["limit"] = UNLIMITED
-      else
-        if sort_limit == UNLIMITED
-          final_limit = output_limit
-        elsif output_limit == UNLIMITED
-          final_limit = sort_limit
-        else
-          final_limit = [sort_limit, output_limit].min
-        end
-        query["sortBy"]["limit"] = final_offset + final_limit if rich_sort
-        query["output"]["limit"] = final_offset + final_limit
-      end
-
-      [final_offset, final_limit]
-    end
-
-    def format_attributes_to_array_style(attributes)
-      attributes ||= []
-      if attributes.is_a?(Hash)
-        attributes.keys.collect do |key|
-          attribute = attributes[key]
-          case attribute
-          when String
-            {
-              "label" => key,
-              "source" => attribute,
-            }
-          when Hash
-            attribute["label"] = key
-            attribute
-          end
-        end
-      else
-        attributes
-      end
-    end
-
-    def collect_output_attributes(attributes)
-      attributes ||= []
-      if attributes.is_a?(Hash)
-        attributes.keys
-      else
-        attributes.collect do |attribute|
-          if attribute.is_a?(Hash)
-            attribute["label"] || attribute["source"]
-          else
-            attribute
-          end
-        end
-      end
-    end
-
-    def collect_source_column_names(attributes)
-      attributes ||= []
-      if attributes.is_a?(Hash)
-        attributes_hash = attributes
-        attributes = []
-        attributes_hash.each do |key, attribute|
-          attributes << attribute["source"] || key
-        end
-        attributes
-      else
-        attributes.collect do |attribute|
-          if attribute.is_a?(Hash)
-            attribute["source"] || attribute["label"]
-          else
-            attribute
-          end
-        end
-      end
-    end
-
-    def collect_sort_attributes(attributes, sort_keys)
-      sort_keys ||= []
-      sort_keys = sort_keys["keys"] || [] if sort_keys.is_a?(Hash)
-
-      attributes = collect_source_column_names(attributes)
-
-      sort_attributes = sort_keys.collect do |key|
-        key = key[1..-1] if key[0] == "-"
-        key
-      end
-      sort_attributes.reject! do |attribute|
-        attributes.include?(attribute)
-      end
-      sort_attributes      
-    end
-
-    ASCENDING_OPERATOR = "<".freeze
-    DESCENDING_OPERATOR = ">".freeze
-
-    def sort_reducer(attributes, sort_keys)
-      attributes ||= []
-      sort_keys ||= []
-      sort_keys = sort_keys["keys"] || [] if sort_keys.is_a?(Hash)
-
-      operators = sort_keys.collect do |sort_key|
-        operator = ASCENDING_OPERATOR
-        if sort_key[0] == "-"
-          operator = DESCENDING_OPERATOR
-          sort_key = sort_key[1..-1]
-        end
-        {
-          "operator" => operator,
-          "column" => attributes.index(sort_key),
-        }
-      end
-
-      {
-        "type" => "sort",
-        "operators" => operators,
-      }
+    def search(message)
+      planner = DistributedSearchPlanner.new(message)
+      distribute(planner.messages)
     end
   end
 end

data/lib/droonga/plugin/distributor/watch.rb

@@ -22,18 +22,23 @@ module Droonga
     repository.register("watch", self)
 
     command "watch.feed" => :feed
-    def feed(envelope)
-      broadcast_all(envelope)
+    def feed(message)
+      broadcast_all(message)
     end
 
     command "watch.subscribe" => :subscribe
-    def subscribe(envelope)
-      broadcast_all(envelope)
+    def subscribe(message)
+      broadcast_all(message)
     end
 
     command "watch.unsubscribe" => :unsubscribe
-    def unsubscribe(envelope)
-      broadcast_all(envelope)
+    def unsubscribe(message)
+      broadcast_all(message)
+    end
+
+    command "watch.sweep" => :sweep
+    def sweep(message)
+      broadcast_all(message)
     end
   end
 end

data/lib/droonga/plugin/handler/add.rb

@@ -18,27 +18,89 @@
 require "groonga"
 
 require "droonga/handler_plugin"
+require "droonga/message_processing_error"
 
 module Droonga
   class AddHandler < Droonga::HandlerPlugin
     repository.register("add", self)
 
+    class MissingTableParameter < BadRequest
+      def initialize
+        super("\"table\" must be specified.")
+      end
+    end
+
+    class MissingPrimaryKeyParameter < BadRequest
+      def initialize(table_name)
+        super("\"key\" must be specified. " +
+                "The table #{table_name.inspect} requires a primary key for a new record.")
+      end
+    end
+
+    class UnknownTable < NotFound
+      def initialize(table_name)
+        super("The table #{table_name.inspect} does not exist in the dataset.")
+      end
+    end
+
+    class InvalidValue < BadRequest
+      def initialize(column, value, request)
+        super("The column #{column.inspect} cannot store the value #{value.inspect}.",
+              request)
+      end
+    end
+
+    class UnknownColumn < NotFound
+      def initialize(column, table, request)
+        super("The column #{column.inspect} does not exist in the table #{table.inspect}.",
+              request)
+      end
+    end
+
     command :add
-    def add(request)
-      outputs = process_add(request)
-      emit(outputs)
+    def add(message, messenger)
+      outputs = process_add(message.request)
+      messenger.emit(outputs)
     end
 
     private
     def process_add(request)
+      raise MissingTableParameter.new unless request.include?("table")
+
       table = @context[request["table"]]
-      return [false] unless table
+      raise UnknownTable.new(request["table"]) unless table
+
       if table.support_key?
-        table.add(request["key"], request["values"])
+        unless request.include?("key")
+          raise MissingPrimaryKeyParameter.new(request["table"])
+        end
+      end
+
+      add_record(table, request)
+      true
+    end
+
+    def add_record(table, request)
+      record = nil
+      if table.support_key?
+        record = table.add(request["key"])
       else
-        table.add(request["values"])
+        record = table.add
+      end
+      (request["values"] || []).each do |column, value|
+        begin
+          record[column] = value
+        rescue Groonga::InvalidArgument => error
+          record.delete if record.added?
+          raise InvalidValue.new(column, value, request)
+        rescue ArgumentError => error
+          record.delete if record.added?
+          raise InvalidValue.new(column, value, request)
+        rescue Groonga::NoSuchColumn => error
+          record.delete if record.added?
+          raise UnknownColumn.new(column, request["table"], request)
+        end
       end
-      [true]
     end
   end
 end

data/lib/droonga/plugin/handler/groonga.rb

@@ -24,17 +24,17 @@ module Droonga
     repository.register("groonga", self)
 
     command :table_create
-    def table_create(request)
+    def table_create(message, messenger)
       command = TableCreate.new(@context)
-      outputs = command.execute(request)
-      emit(outputs)
+      outputs = command.execute(message.request)
+      messenger.emit(outputs)
     end
 
     command :column_create
-    def column_create(request)
+    def column_create(message, messenger)
       command = ColumnCreate.new(@context)
-      outputs = command.execute(request)
-      emit(outputs)
+      outputs = command.execute(message.request)
+      messenger.emit(outputs)
     end
 
     def prefer_synchronous?(command)

data/lib/droonga/plugin/handler/search.rb

@@ -23,11 +23,13 @@ module Droonga
     repository.register("search", self)
 
     command :search
-    def search(request)
+    def search(message, messenger)
       searcher = Droonga::Searcher.new(@context)
-      searcher.search(request["queries"]).each do |output, value|
-        emit(value, output)
+      values = {}
+      searcher.search(message.request["queries"]).each do |output, value|
+        values[output] = value
       end
+      messenger.emit(values)
     end
   end
 end