droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/handler_messenger.rb

@@ -13,20 +13,24 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "time"
+
 require "droonga/replier"
 require "droonga/forwarder"
+require "droonga/timestamp"
 
 module Droonga
   class HandlerMessenger
-    attr_reader :database_name, :dispatcher, :engine_state
+    attr_reader :database_name, :dispatcher, :engine_state, :cluster
 
     def initialize(forwarder, message, options={})
       @forwarder = forwarder
-      @message = message
-      @options = options
-      @replier = Replier.new(@forwarder)
-      @dispatcher = options[:dispatcher]
-      @engine_state = options[:engine_state]
+      @message   = message
+      @options   = options
+      @replier   = Replier.new(@forwarder)
+      @dispatcher    = options[:dispatcher]
+      @engine_state  = options[:engine_state]
+      @cluster       = options[:cluster]
       @database_name = options[:database]
     end
 
@@ -105,6 +109,7 @@ module Droonga
     #
     # @see Forwarder#forward
     def forward(droonga_message, destination)
+      droonga_message["date"] ||= new_date
       @forwarder.forward(droonga_message, destination)
     end
 
@@ -116,5 +121,9 @@ module Droonga
     def log_tag
       "[#{Process.ppid}] handler_messenger"
     end
+
+    def new_date
+      Timestamp.stringify(Time.now)
+    end
   end
 end

data/lib/droonga/handler_runner.rb

@@ -28,6 +28,7 @@ module Droonga
       @loop = loop
       @options = options
       @name = options[:name]
+      @label = options[:label]
       @dataset_name = options[:dataset]
       @database_name = options[:database]
       prepare
@@ -112,7 +113,11 @@ module Droonga
       handler_message.validate
 
       messenger = HandlerMessenger.new(@forwarder, handler_message, @options)
-      handler = handler_class.new(@name, @context, messenger, @loop)
+      handler = handler_class.new(:name      => @name,
+                                  :label     => @label,
+                                  :context   => @context,
+                                  :messenger => messenger,
+                                  :loop      => @loop)
       begin
         result = handler.handle(handler_message)
         unless result.nil?

data/lib/droonga/internal_fluent_message_receiver.rb

@@ -38,12 +38,22 @@ module Droonga
       [@host, @port]
     end
 
-    def shutdown
-      logger.trace("shutdown: start")
-      shutdown_message_receiver
+    def shutdown_gracefully
+      logger.trace("shutdown_gracefully: start")
       shutdown_heartbeat_socket
       shutdown_listen_socket
-      logger.trace("shutdown: done")
+      shutdown_message_receiver_gracefully do
+        yield
+        logger.trace("shutdown_gracefully: done")
+      end
+    end
+
+    def shutdown_immediately
+      logger.trace("shutdown_immediately: start")
+      shutdown_heartbeat_socket
+      shutdown_listen_socket
+      shutdown_message_receiver_immediately
+      logger.trace("shutdown_immediately: done")
     end
 
     private
@@ -56,6 +66,7 @@ module Droonga
 
     def shutdown_listen_socket
       logger.trace("shutdown_listen_socket: start")
+      @listen_socket.close
       logger.trace("shutdown_listen_socket: done")
     end
 
@@ -73,6 +84,8 @@ module Droonga
 
     def shutdown_heartbeat_socket
       logger.trace("shutdown_heartbeat_socket: start")
+      #XXX don't close the heartbeat socket here, because it is used by the generic receiver.
+      # @heartbeat_socket.close
       logger.trace("shutdown_heartbeat_socket: done")
     end
 
@@ -87,11 +100,18 @@ module Droonga
       logger.trace("start_heartbeat_socket: done")
     end
 
-    # TODO: Use stop_gracefully/stop_immediately interface
-    def shutdown_message_receiver
-      logger.trace("shutdown_message_receiver: start")
+    def shutdown_message_receiver_gracefully
+      logger.trace("shutdown_message_receiver_gracefully: start")
+      @message_receiver.stop_gracefully do
+        yield
+        logger.trace("shutdown_message_receiver_gracefully: done")
+      end
+    end
+
+    def shutdown_message_receiver_immediately
+      logger.trace("shutdown_message_receiver_immediately: start")
       @message_receiver.stop_immediately
-      logger.trace("shutdown_message_receiver: done")
+      logger.trace("shutdown_message_receiver_immediately: done")
     end
 
     def log_tag

data/lib/droonga/job_pusher.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -37,15 +37,21 @@ module Droonga
       end
       FileUtils.chmod(0600, @socket_path)
       @loop.attach(@server)
+      logger.trace("connect: socket watcher attached",
+                   :watcher     => @socket,
+                   :socket_path => @socket_path)
     end
 
     def close
-      @server.close if @server
+      return unless @server
+      @server.close
+      logger.trace("close: socket watcher detached",
+                   :watcher => @socket)
     end
 
     def shutdown
       logger.trace("shutdown: start")
-      @server.close if @server
+      close
       @job_queue.close
       FileUtils.rm_f(@socket_path)
       logger.trace("shutdown: done")
@@ -178,13 +184,10 @@ module Droonga
 
       private
       def setup_connection
-        on_read = lambda do |data|
+        @connection.on_read do |data|
           @ready = (data == JobProtocol::READY_SIGNAL)
           @on_ready.call(self) if @on_ready
         end
-        @connection.on_read do |data|
-          on_read.call(data)
-        end
       end
     end
   end

data/lib/droonga/job_receiver.rb

@@ -33,19 +33,24 @@ module Droonga
       @receiver = Coolio::UNIXSocket.connect(@socket_path)
       setup_receive_handler(@receiver)
       @loop.attach(@receiver)
+      logger.trace("start: socket watcher attached",
+                   :watcher     => @receiver,
+                   :socket_path => @socket_path)
       logger.trace("start: done")
     end
 
     def shutdown
       logger.trace("shutdown: start")
       @receiver.close
+      logger.trace("shutdown: socket watcher attached",
+                   :watcher => @receiver)
       logger.trace("shutdown: done")
     end
 
     private
     def setup_receive_handler(connection)
       unpacker = MessagePack::Unpacker.new
-      on_read = lambda do |data|
+      connection.on_read do |data|
         logger.trace("on_read: start")
         unpacker.feed_each(data) do |message|
           @callback.call(message)
@@ -53,9 +58,6 @@ module Droonga
         logger.trace("on_read: done")
         send_ready(connection)
       end
-      connection.on_read do |data|
-        on_read.call(data)
-      end
       send_ready(connection)
     end
 

data/lib/droonga/logger.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -119,12 +119,14 @@ module Droonga
     def log(level, message, data)
       return unless target_level?(level)
       @output.print(build_log_line(level, message, data))
+      @output.flush if debug_level?
     end
 
     def log_backtrace(level, backtrace)
       return unless target_level?(level)
       backtrace.each do |message|
         @output.write(build_log_line(level, message))
+        @output.flush if debug_level?
       end
     end
 
@@ -132,6 +134,10 @@ module Droonga
       @level <= level
     end
 
+    def debug_level?
+      @level <= Level::DEBUG
+    end
+
     def build_log_line(level, message, data={})
       line = "#{Time.now.iso8601}[#{Process.pid}][#{Level.label(level)}]: "
       line << "#{@tag}: " if @tag

data/lib/droonga/node_name.rb

@@ -0,0 +1,90 @@
+# Copyright (C) 2015
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "socket"
+
+module Droonga
+  class NodeName
+    class << self
+      def parse(string)
+        return string if string.is_a?(self)
+
+        if /\A(.+):(\d+)\/([^.]+)\z/ =~ string
+          components = {
+            :host => $1,
+            :port => $2.to_i,
+            :tag  => $3,
+          }
+          new(components)
+        else
+          format = "${host_name}:${port_number}/${tag}"
+          message = "node name must be <#{format}> format: <#{string}>"
+          raise ArgumentError, message
+        end
+      end
+
+      def valid?(string)
+        begin
+          parse(string)
+          true
+        rescue ArgumentError
+          false
+        end
+      end
+    end
+
+    DEFAULT_HOST = Socket.gethostname
+    DEFAULT_HOST.force_encoding("US-ASCII") if DEFAULT_HOST.ascii_only?
+    DEFAULT_PORT = 10031
+    DEFAULT_TAG  = "droonga"
+
+    attr_reader :host
+    attr_reader :port
+    attr_reader :tag
+
+    def initialize(components={})
+      if components.is_a?(self.class)
+        node_name = components
+        components = {
+          :host => node_name.host,
+          :port => node_name.port,
+          :tag  => node_name.tag,
+        }
+      end
+
+      @host = components[:host] || DEFAULT_HOST
+      @port = components[:port] || DEFAULT_PORT
+      @tag  = components[:tag]  || DEFAULT_TAG
+    end
+
+    def to_s
+      node
+    end
+
+    def node
+      "#{@host}:#{@port}/#{@tag}"
+    end
+
+    def to_a
+      [@host, @port, @tag]
+    end
+
+    def ==(other)
+      if other.is_a?(String)
+        return to_s == other
+      end
+      other.is_a?(self.class) and to_a == other.to_a
+    end
+  end
+end

data/lib/droonga/node_role.rb

@@ -0,0 +1,72 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "json"
+
+require "droonga/path"
+require "droonga/serf/tag"
+
+module Droonga
+  class NodeRole
+    SERVICE_PROVIDER   = "service-provider".downcase
+    ABSORB_SOURCE      = "absorb-source".downcase
+    ABSORB_DESTINATION = "absorb-destination".downcase
+
+    ANY = "any".downcase
+
+    ROLES = [
+      SERVICE_PROVIDER,
+      ABSORB_SOURCE,
+      ABSORB_DESTINATION,
+    ]
+
+    class << self
+      def normalize(role)
+        new(role).to_s
+      end
+
+      def mine
+        if Path.serf_tags_file.exist?
+          tags = Path.serf_tags_file.read
+          tags = JSON.parse(tags)
+          role_from_tag = tags[Serf::Tag.node_role]
+          return role_from_tag.downcase if role_from_tag
+        end
+        SERVICE_PROVIDER
+      rescue Errno::ENOENT, JSON::ParserError
+        SERVICE_PROVIDER
+      end
+    end
+
+    def initialize(role)
+      @role = normalize(role)
+    end
+
+    def to_s
+      @role
+    end
+
+    private
+    def valid?(role)
+      ROLES.include?(role)
+    end
+
+    def normalize(role)
+      role = role.to_s.downcase
+      role = SERVICE_PROVIDER unless valid?(role)
+      role
+    end
+  end
+end

data/lib/droonga/path.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -44,12 +44,8 @@ module Droonga
         base + "state"
       end
 
-      def node_status
-        state + "status_file"
-      end
-
-      def live_nodes
-        state + "live-nodes.json"
+      def cluster_state
+        state + "cluster-state.json"
       end
 
       def config
@@ -65,8 +61,24 @@ module Droonga
         Pathname.new(base_file_name).expand_path(base)
       end
 
-      def buffer
-        state + "buffer"
+      def last_message_timestamp
+        base + "last-message-timestamp.txt"
+      end
+
+      def accidental_buffer
+        state + "buffer" + "accidental"
+      end
+
+      def intentional_buffer
+        state + "buffer" + "intentional"
+      end
+
+      def serf_command
+        base + "serf"
+      end
+
+      def serf_tags_file
+        state + "serf-tags.json"
       end
 
       def serf_event_handler_errors
@@ -80,6 +92,19 @@ module Droonga
                        now.hour, now.min, now.sec, now.nsec)
         serf_event_handler_errors + name
       end
+
+      def unique_file_path(directory, basename, suffix)
+        directory = Pathname(directory)
+        basename = basename.sub(/\.\z/, "")
+        suffix   = suffix.sub(/\A\./, "")
+        uniqueness_count = 0
+        path = nil
+        begin
+          path = directory + "#{basename}.#{uniqueness_count}.#{suffix}"
+          uniqueness_count += 1
+        end while path.exist?
+        path
+      end
     end
   end
 end