fluent-plugin-droonga 1.0.0 → 1.0.1

data/lib/droonga/plugins/groonga/column_create.rb

@@ -69,6 +69,9 @@ module Droonga
               options[:type] = :scalar
             elsif @command.column_vector?
               options[:type] = :vector
+              if @command.with_weight?
+                options[:with_weight] = true
+              end
             end
             options
           end
@@ -120,7 +123,7 @@ module Droonga
           step.name = "column_create"
           step.write = true
           step.handler = Handler
-          step.collector = Collectors::Add
+          step.collector = Collectors::Or
         end
       end
     end

data/lib/droonga/plugins/groonga/table_create.rb

@@ -103,7 +103,7 @@ module Droonga
           step.name = "table_create"
           step.write = true
           step.handler = Handler
-          step.collector = Collectors::Add
+          step.collector = Collectors::Or
         end
       end
     end

data/lib/droonga/plugins/groonga/table_remove.rb

@@ -54,7 +54,7 @@ module Droonga
           step.name = "table_remove"
           step.write = true
           step.handler = Handler
-          step.collector = Collectors::Add
+          step.collector = Collectors::Or
         end
       end
     end

data/lib/droonga/plugins/search/distributed_search_planner.rb

@@ -122,6 +122,7 @@ module Droonga
 
             calculate_offset_and_limit!
             build_count_mapper_and_reducer!
+            build_elapsed_time_mapper_and_reducer!
             build_records_mapper_and_reducer!
           end
 
@@ -256,6 +257,14 @@ module Droonga
             end
           end
 
+          def build_elapsed_time_mapper_and_reducer!
+            return unless @output["elements"].include?("elapsedTime")
+
+            @reducers["elapsedTime"] = {
+              "type" => "sum",
+            }
+          end
+
           def build_records_mapper_and_reducer!
             # Skip reducing phase for a result with no record output.
             return if !@output["elements"].include?("records") || @records_limit.zero?

data/lib/droonga/processor.rb

@@ -20,9 +20,9 @@ module Droonga
   class Processor
     include Loggable
 
-    def initialize(loop, message_pusher, options={})
+    def initialize(loop, job_pusher, options={})
       @loop = loop
-      @message_pusher = message_pusher
+      @job_pusher = job_pusher
       @options = options
       @n_workers = @options[:n_workers] || 0
     end
@@ -47,7 +47,7 @@ module Droonga
         if @n_workers.zero? or synchronous
           @handler_runner.process(message)
         else
-          @message_pusher.push(message)
+          @job_pusher.push(message)
         end
       else
         logger.trace("process: ignore #{type}")

data/lib/droonga/reducer.rb

@@ -134,25 +134,28 @@ module Droonga
         base_items, unified_items = unified_items, base_items
       end
 
-      rest_unified_items = unified_items.dup
+      unified_key_map = {}
+      unified_items.each do |unified_item|
+        unified_key_map[unified_item[key_column_index]] = unified_item
+      end
 
+      unified = false
       base_items.reject! do |base_item|
         key = base_item[key_column_index]
-        rest_unified_items.any? do |unified_item|
-          if unified_item[key_column_index] == key
-            base_item.each_with_index do |value, column|
-              next if column == key_column_index
-              unified_item[column] += value
-            end
-            rest_unified_items -= [unified_item]
-            true
-          else
-            false
+        unified_item = unified_key_map[key]
+        if unified_item
+          base_item.each_with_index do |value, column_index|
+            next if column_index == key_column_index
+            unified_item[column_index] += value
           end
+          unified = true
+          true
+        else
+          false
         end
       end
 
-      unless rest_unified_items.size == unified_items.size
+      if unified
         unified_items.sort! do |a, b|
           if compare(a, b, options[:operators])
             -1

data/lib/droonga/searcher.rb

@@ -61,14 +61,26 @@ module Droonga
     def search(queries)
       outputs = nil
       logger.trace("search: start", :queries => queries)
-      @context.push_memory_pool do
-        outputs = process_queries(queries)
+      # TODO: THIS IS JUST A WORKAROUND! We should remove it ASAP!
+      disable_gc do
+        @context.push_memory_pool do
+          outputs = process_queries(queries)
+        end
       end
       logger.trace("search: done")
       return outputs
     end
 
     private
+    def disable_gc
+      GC.disable
+      begin
+        yield
+      ensure
+        GC.enable
+      end
+    end
+
     def process_queries(queries)
       logger.trace("process_queries: start")
       if queries.nil? or queries.empty?
@@ -302,7 +314,12 @@ module Droonga
         @records = @request.source
 
         condition = @request.query["condition"]
-        apply_condition!(condition) if condition
+        if condition
+          apply_condition!(condition)
+
+          adjusters = @request.query["adjusters"]
+          apply_adjusters!(adjusters) if adjusters
+        end
 
         group_by = @request.query["groupBy"]
         apply_group_by!(group_by) if group_by
@@ -328,6 +345,29 @@ module Droonga
         @result.condition = expression
       end
 
+      def apply_adjusters!(adjusters)
+        logger.trace("search_query: adjusters: start")
+        adjusters.each do |adjuster|
+          column_name = adjuster["column"]
+          value = adjuster["value"]
+          factor = adjuster["factor"] || 1
+          logger.trace("search_query: adjusters: adjuster: start",
+                       :column_name => column_name,
+                       :value => value,
+                       :factor => factor)
+          column = @request.source.column(column_name)
+          index, = column.indexes(:match)
+          # TODO: add index.nil? check
+          # TODO: add value.nil? check
+          index.search(value,
+                       :result => @records,
+                       :operator => :adjust,
+                       :weight => factor)
+          logger.trace("search_query: adjusters: adjuster: done")
+        end
+        logger.trace("search_query: adjusters: done")
+      end
+
       def apply_group_by!(group_by)
         logger.trace("search_query: group: start",
                      :by => group_by)
@@ -449,7 +489,12 @@ module Droonga
     module RecordsFormattable
       def record_value(record, attribute)
         if attribute[:source] == "_subrecs"
-          record.sub_records.collect do |sub_record|
+          if record.table.is_a?(Groonga::Array)
+            target_record = record.value
+          else
+            target_record = record
+          end
+          target_record.sub_records.collect do |sub_record|
             sub_attributes = attribute[:attributes]
             format_record(sub_attributes, sub_record)
           end

data/lib/droonga/server.rb

@@ -21,6 +21,8 @@ module Droonga
   module Server
     def before_run
       Droonga.logger.trace("#{log_tag}: before_run: start")
+      config[:job_pusher].close
+      config.delete(:job_pusher)
       Droonga.logger.trace("#{log_tag}: before_run: done")
     end
 

data/lib/droonga/single_step.rb

@@ -18,7 +18,8 @@ require "droonga/collectors"
 
 module Droonga
   class SingleStep
-    def initialize(definition)
+    def initialize(dataset, definition)
+      @dataset = dataset
       @definition = definition
     end
 
@@ -40,14 +41,28 @@ module Droonga
           reduce_key => collector_class.operator,
         }
       end
-      inputs = @definition.inputs
-      if inputs.empty?
-        planner.send(:broadcast, message, options)
+
+      body = message["body"]
+      fact_input = find_fact_input(@definition.inputs, @dataset.fact, body)
+      if fact_input
+        record = body[fact_input[:filter]]
+        planner.send(:scatter, message, record, options)
       else
-        input = inputs.values.first
-        options[:key] = message["body"][input[:filter]]["key"]
-        planner.send(:scatter, message, options)
+        planner.send(:broadcast, message, options)
+      end
+    end
+
+    def find_fact_input(inputs, fact, body)
+      inputs.each do |key, input|
+        if input[:type] == :table
+          # for backward compatibility. We can remove the following code
+          # when all our catalog.json specify "fact" parameter.
+          return input if fact.nil?
+
+          return input if body[key] == fact
+        end
       end
+      nil
     end
   end
 end

data/lib/droonga/slice.rb

@@ -19,7 +19,7 @@ require "droonga/loggable"
 require "droonga/server"
 require "droonga/worker"
 require "droonga/event_loop"
-require "droonga/message_pusher"
+require "droonga/job_pusher"
 require "droonga/processor"
 
 module Droonga
@@ -30,23 +30,22 @@ module Droonga
       @options = options
       @n_workers = @options[:n_workers] || 0
       @loop = loop
-      @message_pusher = MessagePusher.new(@loop)
-      @processor = Processor.new(@loop, @message_pusher, @options)
+      @job_pusher = JobPusher.new(@loop, @options[:database])
+      @processor = Processor.new(@loop, @job_pusher, @options)
       @supervisor = nil
     end
 
     def start
       ensure_database
       @processor.start
-      base_path = @options[:database]
-      @message_pusher.start(base_path)
+      @job_pusher.start
       start_supervisor if @n_workers > 0
     end
 
     def shutdown
       logger.trace("shutdown: start")
       shutdown_supervisor if @supervisor
-      @message_pusher.shutdown
+      @job_pusher.shutdown
       @processor.shutdown
       logger.trace("shutdown: done")
     end
@@ -84,7 +83,8 @@ module Droonga
           :log_level     => logger.level,
           :server_process_name => "Server[#{@options[:database]}] #$0",
           :worker_process_name => "Worker[#{@options[:database]}] #$0",
-          :message_receiver => @message_pusher.raw_receiver,
+          :job_receive_socket_path => @job_pusher.socket_path,
+          :job_pusher => @job_pusher,
         }
         @options.merge(force_options)
       end

data/lib/droonga/step_runner.rb

@@ -21,7 +21,8 @@ module Droonga
   class StepRunner
     include Loggable
 
-    def initialize(plugins)
+    def initialize(dataset, plugins)
+      @dataset = dataset
       @definitions = {}
       plugins.each do |name|
         plugin = Plugin.registry[name]
@@ -43,7 +44,7 @@ module Droonga
       if definition.nil?
         raise UnsupportedMessageError.new(:planner, message)
       end
-      step = SingleStep.new(definition)
+      step = SingleStep.new(@dataset, definition)
       plan = step.plan(message)
       logger.trace("plan: done",
                    :dataset => message["dataset"],

data/lib/droonga/worker.rb

@@ -15,16 +15,17 @@
 
 require "droonga/event_loop"
 require "droonga/handler_runner"
-require "droonga/message_receiver"
+require "droonga/job_receiver"
 
 module Droonga
   module Worker
     def initialize
-      @loop = EventLoop.new
+      @raw_loop = Coolio::Loop.new
+      @loop = EventLoop.new(@raw_loop)
       @handler_runner = HandlerRunner.new(@loop,
                                           config.merge(:dispatcher => nil))
-      receiver_socket = config[:message_receiver]
-      @message_receiver = MessageReceiver.new(@loop, receiver_socket) do |message|
+      receive_socket_path = config[:job_receive_socket_path]
+      @job_receiver = JobReceiver.new(@loop, receive_socket_path) do |message|
         process(message)
       end
     end
@@ -32,16 +33,17 @@ module Droonga
     def run
       Droonga.logger.trace("#{log_tag}: run: start")
       @handler_runner.start
-      @message_receiver.start
-      @loop.run
+      @job_receiver.start
+      @raw_loop.run
       @handler_runner.shutdown
       Droonga.logger.trace("#{log_tag}: run: done")
     end
 
     def stop
       Droonga.logger.trace("#{log_tag}: stop: start")
-      @message_receiver.shutdown
-      @loop.stop
+      @job_receiver.shutdown
+      @raw_loop.stop
+      @loop.break_current_loop
       Droonga.logger.trace("#{log_tag}: stop: done")
     end
 

data/test/command/suite/add/dimension/column.catalog.json

@@ -0,0 +1,27 @@
+{
+  "datasets": {
+    "Droonga": {
+      "schema": {
+        "Products": {
+          "type": "Hash",
+          "keyType": "ShortText",
+          "columns": {
+            "category": {
+              "type": "Scalar",
+              "valueType": "ShortText"
+            }
+          }
+        }
+      },
+      "replicas": [
+        {
+          "dimension": "category"
+        },
+        {
+          "dimension": "category"
+        }
+      ]
+    }
+  }
+}
+

data/test/command/suite/add/dimension/column.expected

@@ -0,0 +1,57 @@
+[
+  "droonga.message",
+  0,
+  {
+    "inReplyTo": "request-id",
+    "statusCode": 200,
+    "type": "add.result",
+    "body": true
+  }
+]
+[
+  "droonga.message",
+  0,
+  {
+    "inReplyTo": "request-id",
+    "statusCode": 200,
+    "type": "add.result",
+    "body": true
+  }
+]
+[
+  "droonga.message",
+  0,
+  {
+    "inReplyTo": "request-id",
+    "statusCode": 200,
+    "type": "add.result",
+    "body": true
+  }
+]
+[
+  "droonga.message",
+  0,
+  {
+    "inReplyTo": "request-id",
+    "statusCode": 200,
+    "type": "search.result",
+    "body": {
+      "products": {
+        "records": [
+          [
+            "Groonga",
+            "groonga"
+          ],
+          [
+            "Rroonga",
+            "groonga"
+          ],
+          [
+            "Ruby",
+            "ruby"
+          ]
+        ]
+      }
+    }
+  }
+]