droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/serf/tag.rb

@@ -0,0 +1,56 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Droonga
+  class Serf
+    module Tag
+      class << self
+        def node_type
+          "type"
+        end
+
+        def node_role
+          "role"
+        end
+
+        def internal_node_name
+          "internal-name"
+        end
+
+        def cluster_id
+          "cluster_id"
+        end
+
+        def accept_messages_newer_than
+          "accept-newer-than"
+        end
+
+        HAVE_UNPROCESSED_MESSAGES_TAG_PREFIX = "buffered-for-"
+
+        def have_unprocessed_messages_tag_for(node_name)
+         "#{HAVE_UNPROCESSED_MESSAGES_TAG_PREFIX}#{node_name}"
+        end
+
+        def have_unprocessed_messages_tag?(tag)
+          tag.start_with?(HAVE_UNPROCESSED_MESSAGES_TAG_PREFIX)
+        end
+
+        def extract_node_name_from_have_unprocessed_messages_tag(tag)
+          tag.sub(HAVE_UNPROCESSED_MESSAGES_TAG_PREFIX, "")
+        end
+      end
+    end
+  end
+end

data/lib/droonga/service_installation.rb

@@ -47,7 +47,7 @@ module Droonga
       test_file = Path.config
       begin
         test_file.read
-      rescue Errno::EACCES => error
+      rescue Errno::EACCES
         return false
       end
       true
@@ -58,7 +58,7 @@ module Droonga
       begin
         FileUtils.mkdir_p(Path.base)
         FileUtils.touch(test_file.to_s)
-      rescue Errno::EACCES => error
+      rescue Errno::EACCES
       end
       unless test_file.exist?
         return false

data/lib/droonga/session.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,6 +13,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "coolio"
+
 require "droonga/loggable"
 
 module Droonga
@@ -26,26 +28,57 @@ module Droonga
       @tasks = tasks
       @n_dones = 0
       @inputs = inputs
+      logger.trace("initialized", :tasks => tasks, :inputs => inputs)
     end
 
     def done?
       @n_dones == @tasks.size
     end
 
+    #TODO: We don't have to wait results if no collection step is
+    #      in the list of tasks, because:
+    #
+    #       * Currently the "super step" mecahnism is not
+    #         implemented yet.
+    #       * So, results won't be used by other handlers directly.
+    #         Results will be used only for the "response" for the
+    #         sender.
+    #       * So, if there is no collection step, no-one requires
+    #         results and there is no need to wait for results.
+    #
+    #      However, in the future after the "super step" mechanism
+    #      is introduced, results can be used by other handlers
+    #      even if there is no collection step.
+    #      Then we must update this logic.
+    def need_result?
+      @tasks.any? do |task|
+        @collector_runner.collectable?("task" => task)
+      end
+    end
+
     def start
       tasks = @inputs[nil] || []
+      logger.trace("start: no task!") if tasks.empty?
       tasks.each do |task|
         local_message = {
           "id"   => @id,
           "task" => task,
         }
+        logger.trace("start: dispatching local message", :message => local_message)
         @dispatcher.process_local_message(local_message)
         @n_dones += 1
       end
     end
 
+    def finish
+      @timeout_timer.detach if @timeout_timer
+      @timeout_timer = nil
+    end
+
     def receive(name, value)
       tasks = @inputs[name]
+      logger.trace("receive: process response",
+                   :name => name, :value => value, :task => tasks)
       unless tasks
         #TODO: result arrived before its query
         return
@@ -82,6 +115,17 @@ module Droonga
       end
     end
 
+    def set_timeout(loop, timeout, &block)
+      @timeout_timer = Coolio::TimerWatcher.new(timeout)
+      @timeout_timer.on_timer do
+        @timeout_timer.detach
+        @timeout_timer = nil
+        report_timeout_error
+        yield
+      end
+      loop.attach(@timeout_timer)
+    end
+
     private
     def send_to_descendantas(descendantas, result)
       descendantas.each do |name, routes|
@@ -96,6 +140,10 @@ module Droonga
       end
     end
 
+    def report_timeout_error
+      #TODO: implement me!
+    end
+
     def log_tag
       "session"
     end

data/lib/droonga/single_step.rb

@@ -32,23 +32,18 @@ module Droonga
 
       # XXX: Re-implement me.
       planner = Planner.new(@dataset)
-      options = {}
-      options[:write] = @definition.write?
-      collector_class = @definition.collector_class
-      if collector_class
-        reduce_key = "result"
-        options[:reduce] = {
-          reduce_key => collector_class.operator,
-        }
-      end
+      planner.write = @definition.write?
+      planner.single_operation = @definition.single_operation?
+      planner.use_all_replicas = @definition.use_all_replicas?
+      planner.collector_class = @definition.collector_class
 
       body = message["body"]
       fact_input = find_fact_input(@definition.inputs, @dataset.fact, body)
       if fact_input
         record = body[fact_input[:filter]]
-        planner.send(:scatter, message, record, options)
+        planner.plan(message, :record => record)
       else
-        planner.send(:broadcast, message, options)
+        planner.plan(message)
       end
     end
 

data/lib/droonga/single_step_definition.rb

@@ -18,7 +18,9 @@ module Droonga
     attr_accessor :name
     attr_accessor :handler
     attr_accessor :collector
-    attr_writer :write
+    attr_writer :write, :single_operation
+    attr_writer :use_all_replicas
+    attr_writer :timeout_calculator
     attr_accessor :inputs
     attr_accessor :output
     def initialize(plugin_module)
@@ -27,6 +29,17 @@ module Droonga
       @handler = nil
       @collector = nil
       @write = false
+      @single_operation = false
+      @use_all_replicas = false
+      @timeout_calculator = lambda do |step|
+        if step["timeout"]
+          return step["timeout"]
+        elsif step["body"]
+          return step["body"]["timeout"] if step["body"]["timeout"]
+        end
+        nil
+      end
+
       @inputs = []
       @output = {}
       yield(self)
@@ -36,6 +49,24 @@ module Droonga
       @write
     end
 
+    def single_operation?
+      return false if @write
+      @single_operation
+    end
+
+    def use_all_replicas?
+      return true if @write
+      @use_all_replicas
+    end
+
+    def timeout_for_step(step)
+      if @timeout_calculator
+        @timeout_calculator.call(step)
+      else
+        nil
+      end
+    end
+
     def handler_class
       resolve_class(@handler)
     end

data/lib/droonga/slice.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,6 +14,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/loggable"
+require "droonga/deferrable"
 require "droonga/supervisor"
 require "droonga/event_loop"
 require "droonga/job_pusher"
@@ -23,9 +24,10 @@ require "droonga/schema_applier"
 module Droonga
   class Slice
     include Loggable
+    include Deferrable
 
-    attr_writer :on_ready
-    def initialize(dataset, loop, options={})
+    def initialize(label, dataset, loop, options={})
+      @label = label
       @dataset = dataset
       @loop = loop
       @options = options
@@ -34,7 +36,6 @@ module Droonga
       @job_pusher = JobPusher.new(@loop, @database_path)
       @processor = Processor.new(@loop, @job_pusher, @options)
       @supervisor = nil
-      @on_ready = nil
     end
 
     def start
@@ -50,6 +51,7 @@ module Droonga
         @job_pusher.shutdown
         @processor.shutdown
         yield if block_given?
+        logger.trace("stop_gracefully: done")
       end
       if @supervisor
         @supervisor.stop_gracefully do
@@ -58,7 +60,6 @@ module Droonga
       else
         on_stop.call
       end
-      logger.trace("stop_gracefully: done")
     end
 
     def stop_immediately
@@ -69,6 +70,12 @@ module Droonga
       logger.trace("stop_immediately: done")
     end
 
+    def refresh_node_reference
+      logger.trace("refresh_node_reference: start")
+      @supervisor.refresh_node_reference if @supervisor
+      logger.trace("refresh_node_reference: done")
+    end
+
     def process(message)
       logger.trace("process: start")
       @processor.process(message)
@@ -112,10 +119,12 @@ module Droonga
 
       config = Supervisor::WorkerConfiguration.new
       config.name = @options[:name]
+      config.label = @label
       config.dataset = @dataset
       config.database_path = @database_path
       config.plugins = @options[:plugins]
       config.job_pusher = @job_pusher
+      config.internal_connection_lifetime = @options[:internal_connection_lifetime]
       @supervisor = Supervisor.new(@loop, @n_workers, config)
       @supervisor.on_ready = lambda do
         on_ready
@@ -123,10 +132,6 @@ module Droonga
       @supervisor.start
     end
 
-    def on_ready
-      @on_ready.call if @on_ready
-    end
-
     def log_tag
       "slice"
     end

data/lib/droonga/supervisor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,18 +14,18 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/loggable"
+require "droonga/deferrable"
 require "droonga/process_supervisor"
 
 module Droonga
   class Supervisor
     include Loggable
+    include Deferrable
 
-    attr_writer :on_ready
     def initialize(loop, n_workers, config)
       @loop = loop
       @n_workers = n_workers
       @config = config
-      @on_ready = nil
     end
 
     def start
@@ -35,7 +35,7 @@ module Droonga
         worker_runner.on_ready = lambda do
           n_ready_workers += 1
           if n_ready_workers == @n_workers
-            @on_ready.call if @on_ready
+            on_ready
           end
         end
         worker_runner.start
@@ -53,10 +53,10 @@ module Droonga
           n_done_worker_runners += 1
           if n_done_worker_runners == n_worker_runners
             yield if block_given?
+            logger.trace("stop_gracefully: done")
           end
         end
       end
-      logger.trace("stop_gracefully: done")
     end
 
     def stop_immediately
@@ -65,6 +65,12 @@ module Droonga
       end
     end
 
+    def refresh_node_reference
+      @worker_runners.each do |worker_runner|
+        worker_runner.refresh_node_reference
+      end
+    end
+
     private
     def log_tag
       "supervisor"
@@ -72,30 +78,31 @@ module Droonga
 
     class WorkerConfiguration
       attr_accessor :name
+      attr_accessor :label
       attr_accessor :dataset
       attr_accessor :database_path
       attr_accessor :plugins
       attr_accessor :job_pusher
+      attr_accessor :internal_connection_lifetime
       def initialize
         @name = nil
+        @label = nil
         @dataset = nil
         @database_path = nil
         @plugins = []
         @job_pusher = nil
+        @internal_connection_lifetime = nil
       end
     end
 
     class WorkerRunner
       include Loggable
+      include Deferrable
 
-      attr_writer :on_ready
-      attr_writer :on_failure
       def initialize(loop, id, config)
         @loop = loop
         @id = id
         @config = config
-        @on_ready = nil
-        @on_failure = nil
         @stop_gracefully_callback = nil
       end
 
@@ -111,9 +118,12 @@ module Droonga
           "--control-write-fd", control_read_out.fileno.to_s,
           "--job-queue-socket-path", @config.job_pusher.socket_path.to_s,
           "--pid-file", pid_path.to_s,
+          "--label", @config.label,
           "--dataset", @config.dataset.name,
           "--database-path", @config.database_path.to_s,
           "--plugins", @config.plugins.join(","),
+          "--internal-connection-lifetime",
+            @config.internal_connection_lifetime.to_s,
         ]
         options = {
           control_write_in => control_write_in,
@@ -130,14 +140,20 @@ module Droonga
       def stop_gracefully(&block)
         logger.trace("stop_gracefully: start")
         @supervisor.stop_gracefully
-        @stop_gracefully_callback = block
-        logger.trace("stop_gracefully: done")
+        @stop_gracefully_callback = lambda do
+          yield if block_given?
+          logger.trace("stop_gracefully: done")
+        end
       end
 
       def stop_immediately
         @supervisor.stop_immediately
       end
 
+      def refresh_node_reference
+        @supervisor.refresh_node_reference
+      end
+
       def success?
         @success
       end
@@ -158,15 +174,6 @@ module Droonga
         supervisor
       end
 
-      def on_ready
-        @on_ready.call if @on_ready
-      end
-
-      def on_failure
-        # TODO: log
-        @on_failure.call if @on_failure
-      end
-
       def on_finish
         _, status = Process.waitpid2(@pid)
         @success = status.success?