droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/engine_state.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,9 +15,8 @@
 
 require "English"
 
-require "coolio"
-
 require "droonga/loggable"
+require "droonga/deferrable"
 require "droonga/event_loop"
 require "droonga/forwarder"
 require "droonga/replier"
@@ -25,28 +24,32 @@ require "droonga/replier"
 module Droonga
   class EngineState
     include Loggable
+    include Deferrable
+
+    DEFAULT_SESSION_TIMEOUT_SECONDS = 60
 
     attr_reader :loop
     attr_reader :name
     attr_reader :internal_name
+    attr_reader :internal_connection_lifetime
     attr_reader :forwarder
     attr_reader :replier
-    attr_writer :on_ready
-    attr_accessor :on_finish
     attr_accessor :catalog
-    def initialize(loop, name, internal_name)
+    attr_accessor :on_finish
+
+    def initialize(loop, name, internal_name, params)
       @loop = loop
       @name = name
       @internal_name = internal_name
+      @internal_connection_lifetime = params[:internal_connection_lifetime]
       @sessions = {}
       @current_id = 0
-      @forwarder = Forwarder.new(@loop, :buffering => true)
+      @forwarder = Forwarder.new(@loop,
+                                 :auto_close_timeout =>
+                                   @internal_connection_lifetime)
       @replier = Replier.new(@forwarder)
-      @on_ready = nil
       @on_finish = nil
-      @catalog = nil
-      @live_nodes = nil
-      @dead_nodes = []
+      @catalog = params[:catalog]
     end
 
     def start
@@ -65,8 +68,43 @@ module Droonga
       route.start_with?(@name) or route.start_with?(@internal_name)
     end
 
-    def farm_path(route)
-      if /\A[^:]+:\d+\/[^.]+/ =~ route
+    FARM_PATH_MATCHER = /\A[^:]+:\d+\/[^.]+/
+
+    def internal_route(route)
+      if FARM_PATH_MATCHER =~ route
+        name = $MATCH
+        if name == @name or name == @internal_name
+          return route.sub(name, @internal_name)
+        end
+      end
+      route
+    end
+
+    def public_route(route)
+      if FARM_PATH_MATCHER =~ route
+        name = $MATCH
+        if name == @internal_name
+          return route.sub(name, @name)
+        end
+      end
+      route
+    end
+
+    def internal_farm_path(route)
+      if FARM_PATH_MATCHER =~ route
+        name = $MATCH
+        if name == @name or name == @internal_name
+          @internal_name
+        else
+          name
+        end
+      else
+        route
+      end
+    end
+
+    def public_farm_path(route)
+      if FARM_PATH_MATCHER =~ route
         name = $MATCH
         if name == @internal_name
           @name
@@ -88,12 +126,22 @@ module Droonga
       @sessions[id]
     end
 
-    def register_session(id, session)
+    def register_session(id, session, options={})
       @sessions[id] = session
       logger.trace("new session #{id} is registered. rest sessions=#{@sessions.size}")
+
+      timeout = options[:timeout] ||
+        @internal_connection_lifetime ||
+        DEFAULT_SESSION_TIMEOUT_SECONDS
+      session.set_timeout(@loop, timeout) do
+        logger.trace("session #{id} is timed out!")
+        unregister_session(id)
+      end
     end
 
     def unregister_session(id)
+      session = @sessions[id]
+      session.finish
       @sessions.delete(id)
       unless have_session?
         @on_finish.call if @on_finish
@@ -105,32 +153,6 @@ module Droonga
       not @sessions.empty?
     end
 
-    def all_nodes
-      @catalog.all_nodes
-    end
-
-    def live_nodes
-      @live_nodes || @catalog.all_nodes
-    end
-
-    def live_nodes=(nodes)
-      old_live_nodes = @live_nodes
-      @live_nodes = nodes
-      @dead_nodes = all_nodes - @live_nodes
-      @forwarder.resume if old_live_nodes != @live_nodes
-      @live_nodes
-    end
-
-    def remove_dead_routes(routes)
-      routes.reject do |route|
-        @dead_nodes.include?(farm_path(route))
-      end
-    end
-
-    def on_ready
-      @on_ready.call if @on_ready
-    end
-
     private
     def log_tag
       "engine_state"

data/lib/droonga/farm.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,11 +15,22 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "droonga/loggable"
+require "droonga/deferrable"
 require "droonga/slice"
 
 module Droonga
   class Farm
-    attr_writer :on_ready
+    include Loggable
+    include Deferrable
+
+    class NoSlice < StandardError
+      def initialize(message, extra_informations={})
+        message = "#{message}: #{extra_informations.inspect}"
+        super(message)
+      end
+    end
+
     def initialize(name, catalog, loop, options={})
       @name = name
       @catalog = catalog
@@ -29,7 +40,7 @@ module Droonga
       slices = @catalog.slices(name)
       slices.each do |slice_name, slice_options|
         dataset = @catalog.datasets[slice_options[:dataset]]
-        slice = Droonga::Slice.new(dataset,
+        slice = Droonga::Slice.new(slice_name, dataset,
                                    @loop,
                                    @options.merge(slice_options))
         @slices[slice_name] = slice
@@ -37,12 +48,18 @@ module Droonga
     end
 
     def start
+      n_slices = @slices.size
+      if n_slices.zero?
+        on_ready
+        return
+      end
+
       n_ready_slices = 0
       @slices.each_value do |slice|
         slice.on_ready = lambda do
           n_ready_slices += 1
-          if n_ready_slices == @slices.size
-            @on_ready.call if @on_ready
+          if n_ready_slices == n_slices
+            on_ready
           end
         end
         slice.start
@@ -50,13 +67,21 @@ module Droonga
     end
 
     def stop_gracefully
+      logger.trace("stop_gracefully: start")
       n_slices = @slices.size
+      if n_slices.zero?
+        yield if block_given?
+        logger.trace("stop_gracefully: done")
+        return
+      end
+
       n_done_slices = 0
       @slices.each_value do |slice|
         slice.stop_gracefully do
           n_done_slices += 1
           if n_done_slices == n_slices
             yield if block_given?
+            logger.trace("stop_gracefully: done")
           end
         end
       end
@@ -68,8 +93,22 @@ module Droonga
       end
     end
 
+    def refresh_node_reference
+      @slices.each_value do |slice|
+        slice.refresh_node_reference
+      end
+    end
+
     def process(slice_name, message)
+      unless @slices.key?(slice_name)
+        raise NoSlice.new(slice_name, :message => message, :slices => @slices.keys)
+      end
       @slices[slice_name].process(message)
     end
+
+    private
+    def log_tag
+      "farm"
+    end
   end
 end

data/lib/droonga/file_observer.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,15 +19,15 @@ require "coolio"
 
 require "droonga/path"
 require "droonga/loggable"
+require "droonga/changable"
 
 module Droonga
   class FileObserver
     include Loggable
+    include Changable
 
     CHECK_INTERVAL = 1
 
-    attr_accessor :on_change
-
     def initialize(loop, path)
       @loop = loop
       @path = path
@@ -36,25 +36,29 @@ module Droonga
       else
         @mtime = nil
       end
-      @on_change = nil
     end
 
     def start
-      @watcher = Coolio::TimerWatcher.new(CHECK_INTERVAL, true)
-      on_timer = lambda do
+      return if @timer
+      @timer = Coolio::TimerWatcher.new(CHECK_INTERVAL, true)
+      @timer.on_timer do
         if updated?
           @mtime = @path.mtime
-          @on_change.call if @on_change
+          on_change
         end
       end
-      @watcher.on_timer do
-        on_timer.call
-      end
-      @loop.attach(@watcher)
+      @loop.attach(@timer)
+      logger.trace("start: timer attached",
+                   :watcher => @timer,
+                   :path    => @path)
     end
 
     def stop
-      @watcher.detach
+      @timer.detach if @timer
+      logger.trace("stop: timer detached",
+                   :watcher => @timer,
+                   :path    => @path)
+      @timer = nil
     end
 
     private

data/lib/droonga/fluent_message_receiver.rb

@@ -24,6 +24,21 @@ module Droonga
   class FluentMessageReceiver
     include Loggable
 
+    class InvalidObject < StandardError
+      def initialize(object)
+        message = "no valid tag information: #{object.inspect}"
+        super(message)
+      end
+    end
+
+    class UnknownTypeEntries < StandardError
+      def initialize(object)
+        message = "unknown type message: couldn't detect entries: " +
+                    "#{object.inspect}"
+        super(message)
+      end
+    end
+
     def initialize(loop, options={}, &on_message)
       @loop = loop
       @listen_fd = options[:listen_fd]
@@ -31,6 +46,7 @@ module Droonga
       @server = nil
       @clients = []
       @on_message = on_message
+      @on_shutdown_ready = nil
     end
 
     def start
@@ -48,20 +64,34 @@ module Droonga
       logger.trace("stop_gracefully: done")
     end
 
+    def ensure_no_client(&block)
+      if @clients.empty?
+        logger.trace("ensure_no_client: no client")
+        yield
+      elsif block_given?
+        logger.trace("ensure_no_client: waiting for #{@clients.size} clients to be disconnected",
+                     :clients => @clients)
+        @on_shutdown_ready = lambda do
+          logger.trace("ensure_no_client: all clients are disconnected")
+          yield
+        end
+      end
+    end
+
     def stop_immediately
       logger.trace("stop_immediately: start")
       stop_gracefully
-      shutdown_clients
+      force_shutdown_clients
       logger.trace("stop_immediately: done")
     end
 
-    def shutdown_clients
+    private
+    def force_shutdown_clients
       @clients.dup.each do |client|
         client.close
       end
     end
 
-    private
     def start_heartbeat_receiver
       logger.trace("start_heartbeat_receiver: start")
       @heartbeat_receiver = HeartbeatReceiver.new(@loop, @heartbeat_fd)
@@ -80,15 +110,29 @@ module Droonga
 
       @clients = []
       @server = create_server do |connection|
+        logger.trace("Client: new connection", :connection => connection)
         client = Client.new(connection) do |tag, time, record|
+          logger.trace("Client: on_message: start")
           @on_message.call(tag, time, record)
+          logger.trace("Client: on_message: done")
         end
         client.on_close = lambda do
           @clients.delete(client)
+          if @on_shutdown_ready
+            logger.trace("Client: a client is disconnected. still waiting for #{@clients.size} clients.",
+                         :clients => @clients)
+            if @clients.empty?
+              @on_shutdown_ready.call
+            end
+          end
         end
         @clients << client
       end
       @loop.attach(@server)
+      logger.trace("start_server: server watcher attached",
+                   :watcher      => @server,
+                   :listen_fd    => @listen_fd,
+                   :heartbeat_fd => @heartbeat_fd)
 
       logger.trace("start_server: done")
     end
@@ -100,6 +144,8 @@ module Droonga
     def shutdown_server
       logger.trace("shutdown_server: start")
       @server.close
+      logger.trace("shutdown_server: server watcher detached",
+                   :watcher => @server)
       logger.trace("shutdown_server: done")
     end
 
@@ -108,6 +154,8 @@ module Droonga
     end
 
     class HeartbeatReceiver
+      include Loggable
+
       def initialize(loop, fd)
         @loop = loop
         @fd = fd
@@ -117,18 +165,21 @@ module Droonga
         @socket = UDPSocket.for_fd(@fd)
 
         @watcher = Coolio::IOWatcher.new(@socket, "r")
-        on_readable = lambda do
-          receive_heartbeat
-        end
         @watcher.on_readable do
-          on_readable.call
+          receive_heartbeat
         end
         @loop.attach(@watcher)
+        logger.trace("start: heartbeat IO watcher attached",
+                     :watcher => @watcher,
+                     :fd      => @fd)
       end
 
       def shutdown
         @socket.close
         @watcher.detach
+        logger.trace("shutdown: heartbeat watcher detached",
+                     :watcher => @watcher,
+                     :fd      => @fd)
       end
 
       private
@@ -153,6 +204,10 @@ module Droonga
         rescue SystemCallError
         end
       end
+
+      def log_tag
+        "heartbeat-receiver"
+      end
     end
 
     class Client
@@ -165,20 +220,14 @@ module Droonga
         @on_close = nil
         @unpacker = MessagePack::Unpacker.new
 
-        on_read = lambda do |data|
-          feed(data)
-        end
         @io.on_read do |data|
-          on_read.call(data)
+          feed(data)
         end
 
-        on_close = lambda do
+        @io.on_close do
           @io = nil
           @on_close.call if @on_close
         end
-        @io.on_close do
-          on_close.call
-        end
       end
 
       def close
@@ -187,22 +236,42 @@ module Droonga
 
       private
       def feed(data)
+        logger.trace("Client: feed: start", :data => data)
         @unpacker.feed_each(data) do |object|
-          tag = object[0]
-          case object[1]
-          when String # PackedForward message
-            entries = MessagePack.unpack(object[1])
-          when Array # Forward message
-            entries = object[1]
-          when Integer, Float # Message message
-            entries = [[object[1], object[2]]]
-          else
-            logger.error("unknown message", :message => object)
-            next
-          end
-          entries.each do |time, record|
-            @on_message.call(tag, time || Time.now.to_i, record)
+          logger.trace("Client: feed_each: start", :object => object)
+          begin
+            handle_feeded_object(object)
+          rescue InvalidObject, UnknownTypeEntries, MessagePack::MalformedFormatError => error
+            logger.exception("failed to process feeded object", error)
           end
+          logger.trace("Client: feed_each: done")
+        end
+        logger.trace("Client: feed: done")
+      end
+
+      def handle_feeded_object(object)
+        raise InvalidObject.new(object) unless object.is_a?(Array)
+
+        tag = object[0]
+        raise InvalidObject.new(object) unless tag.is_a?(String)
+
+        case object[1]
+        when String # PackedForward message [tag, "packed entries"]
+          raise InvalidObject.new(object) unless object.size == 2
+          entries = MessagePack.unpack(object[1])
+        when Array # Forward message [tag, [entry, entry...]]
+          raise InvalidObject.new(object) unless object.size == 2
+          entries = object[1]
+        when Integer, Float # Message message [tag, time, record]
+          raise InvalidObject.new(object) unless object.size == 3
+          entries = [[object[1], object[2]]]
+        else
+          raise UnknownTypeEntries.new(object)
+        end
+
+        raise InvalidObject.new(object) unless entries.is_a?(Array)
+        entries.each do |time, record|
+          @on_message.call(tag, time || Time.now.to_i, record)
         end
       end