droonga-engine 1.0.7 → 1.0.8

data/lib/droonga/distributed_command_planner.rb

@@ -17,17 +17,17 @@
 
 module Droonga
   class DistributedCommandPlanner
-    attr_accessor :key, :dataset
+    attr_accessor :key
 
     REDUCE_SUM = "sum"
 
     DEFAULT_LIMIT = -1
 
-    def initialize(source_message)
+    def initialize(dataset, source_message)
+      @dataset = dataset
       @source_message = source_message
 
       @key = nil
-      @dataset = nil
       @outputs = []
 
       @reducers = []
@@ -58,7 +58,7 @@ module Droonga
     def scatter(record, options={})
       @processor = {
         "command" => @source_message["type"],
-        "dataset" => @dataset || @source_message["dataset"],
+        "dataset" => @dataset.name,
         "body"    => options[:body] || @source_message["body"],
         "record"  => record,
         "type"    => "scatter",
@@ -71,7 +71,7 @@ module Droonga
     def broadcast(options={})
       processor = {
         "command" => @source_message["type"],
-        "dataset" => @dataset || @source_message["dataset"],
+        "dataset" => @dataset.name,
         "body"    => options[:body] || @source_message["body"],
         "type"    => "broadcast",
         "outputs" => [],

data/lib/droonga/engine.rb

@@ -24,6 +24,7 @@ require "droonga/catalog_loader"
 require "droonga/dispatcher"
 require "droonga/file_observer"
 require "droonga/live_nodes_list_loader"
+require "droonga/node_status"
 
 module Droonga
   class Engine
@@ -39,6 +40,12 @@ module Droonga
       @live_nodes_list_observer.on_change = lambda do
         @state.live_nodes = load_live_nodes
       end
+      @node_status_observer = FileObserver.new(loop, Path.node_status)
+      @node_status_observer.on_change = lambda do
+        logger.trace("reloading node_status: start")
+        node_status.reload
+        logger.trace("reloading node_status: done")
+      end
       @on_ready = nil
     end
 
@@ -49,6 +56,7 @@ module Droonga
       end
       @state.start
       @live_nodes_list_observer.start
+      @node_status_observer.start
       @dispatcher.start
       logger.trace("start: done")
     end
@@ -56,9 +64,10 @@ module Droonga
     def stop_gracefully
       logger.trace("stop_gracefully: start")
       @live_nodes_list_observer.stop
+      @node_status_observer.stop
       on_finish = lambda do
         logger.trace("stop_gracefully/on_finish: start")
-        output_last_processed_timestamp
+        save_last_processed_message_timestamp
         @dispatcher.stop_gracefully do
           @state.shutdown
           yield
@@ -78,8 +87,9 @@ module Droonga
     # It may be called after stop_gracefully.
     def stop_immediately
       logger.trace("stop_immediately: start")
-      output_last_processed_timestamp
+      save_last_processed_message_timestamp
       @live_nodes_list_observer.stop
+      @node_status_observer.stop
       @dispatcher.stop_immediately
       @state.shutdown
       logger.trace("stop_immediately: done")
@@ -87,10 +97,14 @@ module Droonga
 
     def process(message)
       return unless effective_message?(message)
-      @last_processed_timestamp = message["date"]
+      @last_processed_message_timestamp = message["date"]
       @dispatcher.process_message(message)
     end
 
+    def node_status
+      @node_status ||= NodeStatus.new
+    end
+
     private
     def load_catalog
       catalog_path = Path.catalog
@@ -116,14 +130,10 @@ module Droonga
       Dispatcher.new(@state, @catalog)
     end
 
-    def output_last_processed_timestamp
-      logger.trace("output_last_processed_timestamp: start")
-      path = Path.last_processed_timestamp
-      FileUtils.mkdir_p(path.dirname.to_s)
-      path.open("w") do |file|
-        file.write(@last_processed_timestamp)
-      end
-      logger.trace("output_last_processed_timestamp: done")
+    def save_last_processed_message_timestamp
+      logger.trace("output_last_processed_message_timestamp: start")
+      node_status.set(:last_processed_message_timestamp, @last_processed_message_timestamp.to_s)
+      logger.trace("output_last_processed_message_timestamp: done")
     end
 
     def effective_message?(message)
@@ -131,17 +141,19 @@ module Droonga
       return true if effective_timestamp.nil?
 
       message_timestamp = Time.parse(message["date"])
+      logger.trace("checking effective_message_timestamp (#{effective_timestamp}) vs message_timestamp(message_timestamp)")
       return false if effective_timestamp >= message_timestamp
 
-      FileUtils.rm(Path.effective_timestamp.to_s)
+      logger.trace("deleting obsolete effective_message_timestamp: start")
+      node_status.delete(:effective_message_timestamp)
+      logger.trace("deleting obsolete effective_message_timestamp: done")
       true
     end
 
     def effective_message_timestamp
-      path = Path.effective_message_timestamp
-      return nil unless path.exist?
+      timestamp = node_status.get(:effective_message_timestamp)
+      return nil unless timestamp
 
-      timestamp = path.read
       begin
         Time.parse(timestamp)
       rescue ArgumentError

data/lib/droonga/engine/version.rb

@@ -15,6 +15,6 @@
 
 module Droonga
   class Engine
-    VERSION = "1.0.7"
+    VERSION = "1.0.8"
   end
 end

data/lib/droonga/handler_runner.rb

@@ -44,6 +44,10 @@ module Droonga
       logger.trace("shutdown: done")
     end
 
+    def change_schema?(type)
+      find_handler_class(type).action.change_schema?
+    end
+
     def prefer_synchronous?(type)
       find_handler_class(type).action.synchronous?
     end

data/lib/droonga/node_status.rb

@@ -20,7 +20,7 @@ require "droonga/safe_file_writer"
 module Droonga
   class NodeStatus
     def initialize
-      @status = load
+      reload
     end
 
     def have?(key)
@@ -45,13 +45,17 @@ module Droonga
       SafeFileWriter.write(status_file, JSON.pretty_generate(@status))
     end
 
+    def reload
+      @status = load
+    end
+
     private
     def normalize_key(key)
       key.to_sym
     end
 
     def status_file
-      @status_file ||= Path.state + "status_file"
+      @status_file ||= Path.node_status
     end
 
     def load

data/lib/droonga/path.rb

@@ -34,34 +34,28 @@ module Droonga
         ENV[BASE_DIR_ENV_NAME] = new_base
       end
 
-      def databases
-        base + "database"
+      def databases(base_path=nil)
+        base_path ||= base
+        path = Pathname(base_path) + "databases"
+        path.expand_path
       end
 
       def state
         base + "state"
       end
 
-      def live_nodes
-        state + "live-nodes.json"
-      end
-
-      def last_processed_timestamp
-        state + "last-processed.timestamp"
+      def node_status
+        state + "status_file"
       end
 
-      def effective_message_timestamp
-        state + "effective-message.timestamp"
+      def live_nodes
+        state + "live-nodes.json"
       end
 
       def config
         base + "droonga-engine.yaml"
       end
 
-      def default_pid_file
-        base + "droonga-engine.pid"
-      end
-
       def default_log_file
         base + "droonga-engine.log"
       end

data/lib/droonga/planner.rb

@@ -22,7 +22,8 @@ module Droonga
     include Loggable
     include ErrorMessages
 
-    def initialize
+    def initialize(dataset)
+      @dataset = dataset
     end
 
     def plan(message)
@@ -31,14 +32,14 @@ module Droonga
 
     private
     def scatter(message, record, options={})
-      planner = DistributedCommandPlanner.new(message)
+      planner = DistributedCommandPlanner.new(@dataset, message)
       planner.scatter(record)
       planner.reduce(options[:reduce])
       planner.plan
     end
 
     def broadcast(message, options={})
-      planner = DistributedCommandPlanner.new(message)
+      planner = DistributedCommandPlanner.new(@dataset, message)
       planner.broadcast(:write => options[:write])
       planner.reduce(options[:reduce])
       planner.plan

data/lib/droonga/plugin/metadata/handler_action.rb

@@ -21,6 +21,14 @@ module Droonga
           @handler_class = handler_class
         end
 
+        def change_schema?
+          configuration[:change_schema]
+        end
+
+        def change_schema=(boolean)
+          configuration[:change_schema] = boolean
+        end
+
         def synchronous?
           configuration[:synchronous]
         end

data/lib/droonga/plugins/groonga/column_create.rb

@@ -103,7 +103,7 @@ module Droonga
         end
 
         class Handler < Droonga::Handler
-          action.synchronous = true
+          action.change_schema = true
 
           def handle(message)
             command = Command.new(@context)

data/lib/droonga/plugins/groonga/column_list.rb

@@ -47,6 +47,28 @@ module Droonga
           private
           def list_columns(table_name)
             table = @context[table_name]
+            virtual_columns(table) + real_columns(table)
+          end
+
+          def virtual_columns(table)
+            if not table.support_key? or table.domain.nil?
+              return []
+            end
+            [
+              [
+                table.id,          # id
+                "_key",            # name
+                "",                # path
+                "",                # type
+                "COLUMN_SCALAR",   # flags
+                table.name,        # domain
+                table.domain.name, # range
+                [],                # source
+              ]
+            ]
+          end
+
+          def real_columns(table)
             table.columns.collect do |column|
               format_column(column)
             end
@@ -98,7 +120,7 @@ module Droonga
             return [] unless column.is_a?(::Groonga::IndexColumn)
             column.sources.collect do |source|
               if source.is_a?(::Groonga::Table)
-                "_key"
+                source.name
               else
                 source.local_name
               end

data/lib/droonga/plugins/groonga/column_remove.rb

@@ -46,7 +46,7 @@ module Droonga
         end
 
         class Handler < Droonga::Handler
-          action.synchronous = true
+          action.change_schema = true
 
           def handle(message)
             command = Command.new(@context)

data/lib/droonga/plugins/groonga/column_rename.rb

@@ -48,7 +48,7 @@ module Droonga
         end
 
         class Handler < Droonga::Handler
-          action.synchronous = true
+          action.change_schema = true
 
           def handle(message)
             command = Command.new(@context)

data/lib/droonga/plugins/groonga/delete.rb

@@ -97,7 +97,7 @@ module Droonga
         end
 
         class Handler < Droonga::Handler
-          action.synchronous = true
+          action.change_schema = true
 
           def handle(message)
             command = Command.new(@context)

data/lib/droonga/plugins/groonga/select.rb

@@ -28,7 +28,7 @@ module Droonga
             @result_name = @table + "_result"
 
             output_columns = select_request["output_columns"] || "_id, _key, *"
-            attributes = output_columns.split(/\s*,\s*/)
+            attributes = convert_output_columns(output_columns)
             offset = (select_request["offset"] || "0").to_i
             limit = (select_request["limit"] || "10").to_i
 
@@ -131,7 +131,7 @@ module Droonga
             drilldown_keys = drilldown_keys.split(",")
 
             sort_keys = (select_request["drilldown_sortby"] || "").split(",")
-            columns   = (select_request["drilldown_output_columns"] || "_key,_nsubrecs").split(",")
+            columns   = convert_output_columns(select_request["drilldown_output_columns"] || "_key,_nsubrecs")
             offset    = (select_request["drilldown_offset"] || "0").to_i
             limit     = (select_request["drilldown_limit"] || "10").to_i
 
@@ -165,6 +165,20 @@ module Droonga
             end
             queries
           end
+
+          # for a backward compatibility for command_version=1,
+          # whitespace-separeted case (without functions) should be accepted.
+          COMMAND_VERSION_1_ONLY_PATTERN = /\A[^\s,()]+(\s+[^\s,()]+)+\z/
+
+          def convert_output_columns(output_columns)
+            output_columns = output_columns.strip
+            command_version_is_1 = output_columns =~ COMMAND_VERSION_1_ONLY_PATTERN
+            if command_version_is_1
+              output_columns.split(/\s+/)
+            else
+              output_columns.split(/\s*,\s*/)
+            end
+          end
         end
 
         class ResponseConverter
@@ -223,6 +237,7 @@ module Droonga
             records.collect do |record|
               record.collect.each_with_index do |value, i|
                 name, type = attributes[i]
+                _ = name # suppress a warning
                 case type
                 when "Time"
                   normalize_time(value).to_f

data/lib/droonga/plugins/groonga/table_create.rb

@@ -35,6 +35,8 @@ module Droonga
                                      :result => false)
             end
 
+            validate_key_type
+
             options = parse_command
             ::Groonga::Schema.define(:context => @context) do |schema|
               schema.create_table(name, options)
@@ -88,10 +90,33 @@ module Droonga
             return unless @command["normalizer"]
             options[:normalizer] = @command["normalizer"]
           end
+
+          def validate_key_type
+            if @command.table_hash_key? and @command["key_type"].nil?
+              message = "key_type is required for TABLE_HASH_KEY table"
+              raise CommandError.new(:status => Status::INVALID_ARGUMENT,
+                                     :message => message,
+                                     :result => false)
+            end
+
+            if @command.table_pat_key? and @command["key_type"].nil?
+              message = "key_type is required for TABLE_PAT_KEY table"
+              raise CommandError.new(:status => Status::INVALID_ARGUMENT,
+                                     :message => message,
+                                     :result => false)
+            end
+
+            if @command.table_dat_key? and @command["key_type"].nil?
+              message = "key_type is required for TABLE_DAT_KEY table"
+              raise CommandError.new(:status => Status::INVALID_ARGUMENT,
+                                     :message => message,
+                                     :result => false)
+            end
+          end
         end
 
         class Handler < Droonga::Handler
-          action.synchronous = true
+          action.change_schema = true
 
           def handle(message)
             command = Command.new(@context)