droonga-engine 1.0.2 → 1.0.3

data/lib/droonga/plugins/groonga/select.rb

@@ -26,8 +26,8 @@ module Droonga
             @table = select_request["table"]
             @result_name = @table + "_result"
 
-            output_columns = select_request["output_columns"] || ""
-            attributes = output_columns.split(/, */)
+            output_columns = select_request["output_columns"] || "_id, _key, *"
+            attributes = output_columns.split(/\s*,\s*/)
             offset = (select_request["offset"] || "0").to_i
             limit = (select_request["limit"] || "10").to_i
 
@@ -121,7 +121,7 @@ module Droonga
             drilldown_keys = drilldown_keys.split(",")
 
             sort_keys = (select_request["drilldown_sortby"] || "").split(",")
-            columns   = (select_request["drilldown_output_columns"] || "").split(",")
+            columns   = (select_request["drilldown_output_columns"] || "_key,_nsubrecs").split(",")
             offset    = (select_request["drilldown_offset"] || "0").to_i
             limit     = (select_request["drilldown_limit"] || "10").to_i
 
@@ -177,16 +177,18 @@ module Droonga
           def convert_main_result(result)
             status_code = 0
             start_time = result["startTime"]
-            start_time_in_unix_time = if start_time
-                                        Time.parse(start_time).to_f
-                                      else
-                                        Time.now.to_f
-                                      end
+            start_time_in_unix_time = normalize_time(start_time).to_f
             elapsed_time = result["elapsedTime"] || 0
             @header = [status_code, start_time_in_unix_time, elapsed_time]
             @body = convert_search_result(result)
           end
 
+          def normalize_time(time)
+            time ||= Time.now
+            time = Time.parse(time) if time.is_a?(String)
+            time
+          end
+
           def convert_drilldown_result(key, result)
             @drilldown_results << convert_search_result(result)
           end
@@ -194,12 +196,8 @@ module Droonga
           def convert_search_result(result)
             count      = result["count"]
             attributes = convert_attributes(result["attributes"])
-            records    = result["records"]
-            if records.nil? or records.empty?
-              [[count], attributes]
-            else
-              [[count], attributes, records]
-            end
+            records    = convert_records(attributes, result["records"] || [])
+            [[count], attributes, *records]
           end
 
           def convert_attributes(attributes)
@@ -210,6 +208,20 @@ module Droonga
               [name, type]
             end
           end
+
+          def convert_records(attributes, records)
+            records.collect do |record|
+              record.collect.each_with_index do |value, i|
+                name, type = attributes[i]
+                case type
+                when "Time"
+                  normalize_time(value).to_f
+                else
+                  value
+                end
+              end
+            end
+          end
         end
 
         class Adapter < Droonga::Adapter

data/lib/droonga/plugins/search.rb

@@ -76,6 +76,28 @@ module Droonga
             end
           end
 
+          attributes_mapper = elements["attributes"]
+          if attributes_mapper and value["attributes"]
+            attributes = value["attributes"]
+            output_attributes = []
+            attributes_mapper["names"].each do |name|
+              if name == "*"
+                attributes.each do |attribute|
+                  next if attribute["name"].start_with?("_")
+                  output_attributes << attribute
+                end
+              else
+                attributes.each do |attribute|
+                  if attribute["name"] == name
+                    output_attributes << attribute
+                    break
+                  end
+                end
+              end
+            end
+            value["attributes"] = output_attributes
+          end
+
           records_mapper = elements["records"]
           if records_mapper and value["records"]
             if records_mapper["no_output"]
@@ -99,8 +121,14 @@ module Droonga
               complex_item
             end
           else
-            items.collect do |item|
-              item[0...attributes.size]
+            # FIXME: Compare with "attributes" value from "search" not
+            # gather parameter like the following.
+            if attributes.include?("*")
+              items
+            else
+              items.collect do |item|
+                item[0...attributes.size]
+              end
             end
           end
         end

data/lib/droonga/plugins/search/distributed_search_planner.rb

@@ -77,18 +77,9 @@ module Droonga
         end
 
         def transform_query(input_name, query)
-          output = query["output"]
-
-          # Skip reducing phase for a result with no output.
-          if output.nil? or
-              output["elements"].nil? or
-              (!output["elements"].include?("count") and
-              !output["elements"].include?("records"))
-            return
-          end
+          return unless need_reduce?(query)
 
           transformer = QueryTransformer.new(query)
-
           elements = transformer.mappers
           mapper = {}
           mapper["elements"] = elements unless elements.empty?
@@ -96,6 +87,19 @@ module Droonga
                                  :gather => mapper })
         end
 
+        def need_reduce?(query)
+          output = query["output"]
+          return false if output.nil?
+
+          output_elements = output["elements"]
+          return false if output_elements.nil?
+
+          need_reduce_elements = ["count", "attributes", "records"]
+          output_elements.any? do |element|
+            need_reduce_elements.include?(element)
+          end
+        end
+
         class QueryTransformer
           attr_reader :reducers, :mappers
 
@@ -123,6 +127,7 @@ module Droonga
             calculate_offset_and_limit!
             build_count_mapper_and_reducer!
             build_elapsed_time_mapper_and_reducer!
+            build_attributes_mapper_and_reducer!
             build_records_mapper_and_reducer!
           end
 
@@ -265,6 +270,18 @@ module Droonga
             }
           end
 
+          def build_attributes_mapper_and_reducer!
+            return unless @output["elements"].include?("attributes")
+
+            @reducers["attributes"] = {
+              "type" => "or",
+            }
+
+            @mappers["attributes"] = {
+              "names" => output_attribute_names,
+            }
+          end
+
           def build_records_mapper_and_reducer!
             # Skip reducing phase for a result with no record output.
             return if !@output["elements"].include?("records") || @records_limit.zero?
@@ -325,7 +342,7 @@ module Droonga
                   }
                 when Hash
                   attribute["label"] = key
-              attribute
+                  attribute
                 end
               end
             else

data/lib/droonga/processor.rb

@@ -15,6 +15,7 @@
 
 require "droonga/loggable"
 require "droonga/handler_runner"
+require "fileutils"
 
 module Droonga
   class Processor
@@ -46,6 +47,9 @@ module Droonga
         synchronous = @handler_runner.prefer_synchronous?(type)
         if @n_workers.zero? or synchronous
           @handler_runner.process(message)
+          #XXX Workaround to restart system by any schema change.
+          #    This should be done more smartly...
+          FileUtils.touch(Path.catalog.to_s) if synchronous
         else
           @job_pusher.push(message)
         end

data/lib/droonga/searcher.rb

@@ -663,6 +663,7 @@ module Droonga
 
       def output_target_attributes
         attributes = @request.output["attributes"]
+        attributes = expand_attributes(attributes)
         normalize_target_attributes(attributes)
       end
 
@@ -675,6 +676,31 @@ module Droonga
         formatter.format(output_target_attributes, @result.records, output_limit, output_offset)
       end
 
+      def expand_attributes(attributes, domain = @result.records)
+        expanded_attributes = []
+        attributes.each do |attribute|
+          if attribute.is_a?(String)
+            source = attribute
+          else
+            source = attribute["source"]
+          end
+          if source == "*"
+            real_table = domain
+            loop do
+              next_domain = real_table.domain
+              break unless next_domain.is_a?(Groonga::Table)
+              real_table = next_domain
+            end
+            real_table.columns.each do |column|
+              expanded_attributes << column.local_name
+            end
+          else
+            expanded_attributes << attribute
+          end
+        end
+        expanded_attributes
+      end
+
       def normalize_target_attributes(attributes, domain = @result.records)
         attributes.collect do |attribute|
           if attribute.is_a?(String)

data/lib/droonga/serf.rb

@@ -0,0 +1,119 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/path"
+require "droonga/loggable"
+require "droonga/catalog_loader"
+require "droonga/serf_downloader"
+
+module Droonga
+  class Serf
+    class << self
+      def path
+        Droonga::Path.base + "serf"
+      end
+    end
+
+    include Loggable
+
+    def initialize(loop, name)
+      @loop = loop
+      @name = name
+      @pid = nil
+    end
+
+    def start
+      logger.trace("start: start")
+      ensure_serf
+      ENV["SERF"] = @serf
+      ENV["SERF_RPC_ADDRESS"] = rpc_address
+      retry_joins = []
+      detect_other_hosts.each do |other_host|
+        retry_joins.push("-retry-join", other_host)
+      end
+      @pid = run("agent",
+                 "-node", @name,
+                 "-bind", extract_host(@name),
+                 "-event-handler", "droonga-engine-serf-event-handler",
+                 *retry_joins)
+      logger.trace("start: done")
+    end
+
+    def running?
+      not @pid.nil?
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      Process.waitpid(run("leave"))
+      Process.waitpid(@pid)
+      @pid = nil
+      logger.trace("shutdown: done")
+    end
+
+    def restart
+      shutdown
+      start
+    end
+
+    private
+    def ensure_serf
+      @serf = find_system_serf
+      return if @serf
+
+      serf_path = self.class.path
+      @serf = serf_path.to_s
+      return if serf_path.executable?
+      downloader = SerfDownloader.new(serf_path)
+      downloader.download
+    end
+
+    def find_system_serf
+      paths = (ENV["PATH"] || "").split(File::PATH_SEPARATOR)
+      paths.each do |path|
+        serf = File.join(path, "serf")
+        return serf if File.executable?(serf)
+      end
+      nil
+    end
+
+    def run(command, *options)
+      spawn(@serf, command, "-rpc-addr", rpc_address, *options)
+    end
+
+    def extract_host(node_name)
+      node_name.split(":").first
+    end
+
+    def rpc_address
+      "#{extract_host(@name)}:7373"
+    end
+
+    def detect_other_hosts
+      loader = CatalogLoader.new(Path.catalog.to_s)
+      catalog = loader.load
+      other_nodes = catalog.all_nodes.reject do |node|
+        node == @name
+      end
+      other_nodes.collect do |node|
+        extract_host(node)
+      end
+    end
+
+    def log_tag
+      "serf"
+    end
+  end
+end

data/lib/droonga/serf_downloader.rb

@@ -0,0 +1,90 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "stringio"
+require "tmpdir"
+require "fileutils"
+
+require "faraday"
+require "faraday_middleware"
+require "archive/zip"
+
+require "droonga/loggable"
+
+module Droonga
+  class SerfDownloader
+    include Loggable
+
+    def initialize(output_path)
+      @output_path = output_path
+    end
+
+    def download
+      detect_platform
+      version = "0.6.0"
+      url_base = "https://dl.bintray.com/mitchellh/serf"
+      base_name = "#{version}_#{@os}_#{@architecture}.zip"
+      connection = Faraday.new(url_base) do |builder|
+        builder.response(:follow_redirects)
+        builder.adapter(Faraday.default_adapter)
+      end
+      response = connection.get(base_name)
+      absolete_output_path = @output_path.expand_path
+      Dir.mktmpdir do |dir|
+        Archive::Zip.extract(StringIO.new(response.body),
+                             dir,
+                             :directories => false)
+        FileUtils.mv("#{dir}/serf", absolete_output_path.to_s)
+        FileUtils.chmod(0755, absolete_output_path.to_s)
+      end
+    end
+
+    private
+    def detect_platform
+      detect_os
+      detect_architecture
+    end
+
+    def detect_os
+      case RUBY_PLATFORM
+      when /linux/
+        @os = "linux"
+      when /freebsd/
+        @os = "freebsd"
+      when /darwin/
+        @os = "darwin"
+      when /mswin|mingw/
+        @os = "windows"
+      else
+        raise "Unsupported OS: #{RUBY_PLATFORM}"
+      end
+    end
+
+    def detect_architecture
+      case RUBY_PLATFORM
+      when /x86_64|x64/
+        @architecture = "amd64"
+      when /i\d86/
+        @architecture = "i386"
+      else
+        raise "Unsupported architecture: #{RUBY_PLATFORM}"
+      end
+    end
+
+    def log_tag
+      "serf-downloader"
+    end
+  end
+end