droonga-engine 1.0.3 → 1.0.4

data/lib/droonga/engine.rb

@@ -22,24 +22,21 @@ require "droonga/loggable"
 require "droonga/engine_state"
 require "droonga/catalog_loader"
 require "droonga/dispatcher"
-require "droonga/live_nodes_list_observer"
+require "droonga/file_observer"
+require "droonga/live_nodes_list_loader"
 
 module Droonga
   class Engine
     include Loggable
 
-    LAST_PROCESSED_TIMESTAMP = "last-processed.timestamp"
-    EFFECTIVE_MESSAGE_TIMESTAMP = "effective-message.timestamp"
-
     def initialize(loop, name, internal_name)
       @state = EngineState.new(loop, name, internal_name)
       @catalog = load_catalog
-      @live_nodes = @catalog.all_nodes
+      @state.catalog = @catalog
       @dispatcher = create_dispatcher
-      @live_nodes_list_observer = LiveNodesListObserver.new
-      @live_nodes_list_observer.on_update = lambda do |live_nodes|
-        @live_nodes = live_nodes
-        @dispatcher.live_nodes = live_nodes if @dispatcher
+      @live_nodes_list_observer = FileObserver.new(loop, Path.live_nodes)
+      @live_nodes_list_observer.on_change = lambda do
+        @state.live_nodes = load_live_nodes
       end
     end
 
@@ -95,22 +92,28 @@ module Droonga
       catalog
     end
 
+    def load_live_nodes
+      path = Path.live_nodes
+      loader = LiveNodesListLoader.new(path)
+      live_nodes = loader.load
+      logger.info("live-nodes loaded",
+                  :path  => path,
+                  :mtime => path.mtime)
+      live_nodes
+    end
+
     def create_dispatcher
-      dispatcher = Dispatcher.new(@state, @catalog)
-      dispatcher.live_nodes = @live_nodes
-      dispatcher
+      Dispatcher.new(@state, @catalog)
     end
 
     def output_last_processed_timestamp
-      File.open(last_processed_timestamp_file, "w") do |file|
+      path = Path.last_processed_timestamp
+      FileUtils.mkdir_p(path.dirname.to_s)
+      path.open("w") do |file|
         file.write(@last_processed_timestamp)
       end
     end
 
-    def last_processed_timestamp_file
-      @last_processed_timestamp_file ||= File.join(Droonga::Path.state, LAST_PROCESSED_TIMESTAMP)
-    end
-
     def effective_message?(message)
       effective_timestamp = effective_message_timestamp
       return true if effective_timestamp.nil?
@@ -118,14 +121,15 @@ module Droonga
       message_timestamp = Time.parse(message["date"])
       return false if effective_timestamp >= message_timestamp
 
-      FileUtils.rm(effective_message_timestamp_file)
+      FileUtils.rm(Path.effective_timestamp.to_s)
       true
     end
 
     def effective_message_timestamp
-      return nil unless File.exist?(effective_message_timestamp_file)
+      path = Path.effective_message_timestamp
+      return nil unless path.exist?
 
-      timestamp = File.read(effective_message_timestamp_file)
+      timestamp = path.read
       begin
         Time.parse(timestamp)
       rescue ArgumentError
@@ -133,10 +137,6 @@ module Droonga
       end
     end
 
-    def effective_message_timestamp_file
-      @effective_message_timestamp_file ||= File.join(Droonga::Path.state, EFFECTIVE_MESSAGE_TIMESTAMP)
-    end
-
     def log_tag
       "engine"
     end

data/lib/droonga/engine_state.rb

@@ -32,6 +32,8 @@ module Droonga
     attr_reader :forwarder
     attr_reader :replier
     attr_accessor :on_finish
+    attr_accessor :catalog
+    attr_writer :dead_nodes
     def initialize(loop, name, internal_name)
       @loop = loop
       @name = name
@@ -39,8 +41,11 @@ module Droonga
       @sessions = {}
       @current_id = 0
       @forwarder = Forwarder.new(@loop)
+      @forwarder.resume
       @replier = Replier.new(@forwarder)
       @on_finish = nil
+      @catalog = nil
+      @live_nodes = nil
     end
 
     def start
@@ -97,6 +102,24 @@ module Droonga
       not @sessions.empty?
     end
 
+    def all_nodes
+      @catalog.all_nodes
+    end
+
+    def live_nodes
+      @live_nodes || @catalog.all_nodes
+    end
+
+    def live_nodes=(nodes)
+      @live_nodes = nodes
+      @dead_nodes = all_nodes - @live_nodes
+      @live_nodes
+    end
+
+    def dead_nodes
+      @dead_nodes || []
+    end
+
     private
     def log_tag
       "engine_state"

data/lib/droonga/{catalog_observer.rb → file_observer.rb}

@@ -21,22 +21,26 @@ require "droonga/path"
 require "droonga/loggable"
 
 module Droonga
-  class CatalogObserver
+  class FileObserver
     include Loggable
 
     CHECK_INTERVAL = 1
 
     attr_accessor :on_change
 
-    def initialize(loop)
+    def initialize(loop, path)
       @loop = loop
-      @path = Path.catalog
-      @mtime = @path.mtime
+      @path = path
+      if @path.exist?
+        @mtime = @path.mtime
+      else
+        @mtime = nil
+      end
       @on_change = nil
     end
 
     def start
-      @watcher = Cool.io::TimerWatcher.new(CHECK_INTERVAL, true)
+      @watcher = Coolio::TimerWatcher.new(CHECK_INTERVAL, true)
       on_timer = lambda do
         if updated?
           @mtime = @path.mtime
@@ -55,11 +59,12 @@ module Droonga
 
     private
     def updated?
-      @path.mtime > @mtime
+      return false unless @path.exist?
+      @mtime.nil? or @path.mtime > @mtime
     end
 
     def log_tag
-      "catalog-observer"
+      "file-observer"
     end
   end
 end

data/lib/droonga/fluent_message_sender.rb

@@ -15,6 +15,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "fileutils"
 require "thread"
 
 require "cool.io"
@@ -22,6 +23,7 @@ require "cool.io"
 require "droonga/message-pack-packer"
 
 require "droonga/loggable"
+require "droonga/buffered_tcp_socket"
 
 module Droonga
   class FluentMessageSender
@@ -32,37 +34,40 @@ module Droonga
       @host = host
       @port = port
       @socket = nil
-      @buffer = []
-      @write_mutex = Mutex.new
     end
 
     def start
       logger.trace("start: start")
-      start_writer
       logger.trace("start: done")
     end
 
     def shutdown
       logger.trace("shutdown: start")
-      shutdown_writer
       shutdown_socket
       logger.trace("shutdown: done")
     end
 
     def send(tag, data)
       logger.trace("send: start")
-      fluent_message = [tag, Time.now.to_i, data]
-      packed_fluent_message = MessagePackPacker.pack(fluent_message)
-      @write_mutex.synchronize do
-        @buffer << packed_fluent_message
-        unless @signaling
-          @signaling = true
-          @writer.signal
-        end
-      end
+      packed_fluent_message = create_packed_fluent_message(tag, data)
+      connect unless connected?
+      @socket.write(packed_fluent_message)
       logger.trace("send: done")
     end
 
+    def reserve_send(tag, data)
+      logger.trace("reserve_send: start")
+      packed_fluent_message = create_packed_fluent_message(tag, data)
+      connect unless connected?
+      @socket.reserve_write(packed_fluent_message)
+      logger.trace("reserve_send: done")
+    end
+
+    def resume
+      connect
+      @socket.resume
+    end
+
     private
     def connected?
       not @socket.nil?
@@ -85,7 +90,9 @@ module Droonga
         @socket = nil
       end
 
-      @socket = Coolio::TCPSocket.connect(@host, @port)
+      data_directory = Path.buffer + "#{@host}:#{@port}"
+      FileUtils.mkdir_p(data_directory.to_s)
+      @socket = BufferedTCPSocket.connect(@host, @port, data_directory)
       @socket.on_write_complete do
         log_write_complete.call
       end
@@ -108,29 +115,9 @@ module Droonga
       @socket.close unless @socket.closed?
     end
 
-    def start_writer
-      @writer = Coolio::AsyncWatcher.new
-      @signaling = false
-
-      on_signal = lambda do
-        @write_mutex.synchronize do
-          @signaling = false
-          connect unless connected?
-          @buffer.each do |data|
-            @socket.write(data)
-          end
-          @buffer.clear
-        end
-      end
-      @writer.on_signal do
-        on_signal.call
-      end
-
-      @loop.attach(@writer)
-    end
-
-    def shutdown_writer
-      @writer.detach
+    def create_packed_fluent_message(tag, data)
+      fluent_message = [tag, Time.now.to_i, data]
+      MessagePackPacker.pack(fluent_message)
     end
 
     def log_tag

data/lib/droonga/forwarder.rb

@@ -16,6 +16,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 require "droonga/loggable"
+require "droonga/path"
 require "droonga/event_loop"
 require "droonga/fluent_message_sender"
 
@@ -46,15 +47,35 @@ module Droonga
       command = destination["type"]
       receiver = destination["to"]
       arguments = destination["arguments"]
-      output(receiver, message, command, arguments)
+      reserve = destination["reserve"]
+      output(receiver, message, command, arguments, :reserve => reserve)
       logger.trace("forward: done")
     end
 
+    def resume
+      return unless Path.buffer.exist?
+      Pathname.glob("#{Path.buffer}/*") do |path|
+        next unless path.directory?
+        next if Pathname.glob("#{path.to_s}/*").empty?
+
+        destination = path.basename.to_s
+        next if @senders.key?(destination)
+
+        components = destination.split(":")
+        port = components.pop.to_i
+        next if port.zero?
+        host = components.join(":")
+
+        sender = create_sender(host, port)
+        sender.resume
+        @senders[destination] = sender
+      end
+    end
+
     private
-    def output(receiver, message, command, arguments)
+    def output(receiver, message, command, arguments, options={})
       logger.trace("output: start")
-      # TODO: IMPROVE ME: Should not use "unless" and "and". It is confused.
-      unless receiver.is_a?(String) and command.is_a?(String)
+      if not receiver.is_a?(String) or not command.is_a?(String)
         logger.trace("output: abort: invalid argument",
                      :receiver => receiver,
                      :command  => command)
@@ -83,7 +104,11 @@ module Droonga
       output_tag = "#{tag}.message"
       log_info = "<#{receiver}>:<#{output_tag}>"
       logger.trace("output: post: start: #{log_info}")
-      sender.send(output_tag, message)
+      if options[:reserve]
+        sender.reserve_send(output_tag, message)
+      else
+        sender.send(output_tag, message)
+      end
       logger.trace("output: post: done: #{log_info}")
       logger.trace("output: done")
     end

data/lib/droonga/handler_messenger.rb

@@ -18,14 +18,15 @@ require "droonga/forwarder"
 
 module Droonga
   class HandlerMessenger
-    attr_reader :database_name
+    attr_reader :database_name, :dispatcher, :engine_state
 
     def initialize(forwarder, message, options={})
       @forwarder = forwarder
       @message = message
       @options = options
       @replier = Replier.new(@forwarder)
-      @dispatcher = @options[:dispatcher]
+      @dispatcher = options[:dispatcher]
+      @engine_state = options[:engine_state]
       @database_name = options[:database]
     end
 

data/lib/droonga/handler_runner.rb

@@ -16,7 +16,6 @@
 require "groonga"
 
 require "droonga/loggable"
-require "droonga/forwarder"
 require "droonga/handler_message"
 require "droonga/handler_messenger"
 require "droonga/step_runner"
@@ -36,18 +35,12 @@ module Droonga
 
     def start
       logger.trace("start: start")
-      @forwarder.start
       logger.trace("start: done")
     end
 
     def shutdown
       logger.trace("shutdown: start")
-      @forwarder.shutdown
-      if @database
-        @database.close
-        @context.close
-        @database = @context = nil
-      end
+      close_database if @database
       logger.trace("shutdown: done")
     end
 
@@ -62,12 +55,17 @@ module Droonga
     def process(message)
       logger.trace("process: start")
       type = message["type"]
-      handler_class = find_handler_class(type)
-      if handler_class.nil?
-        logger.trace("process: done: no handler: <#{type}>")
-        return
+      if type == "database.reopen"
+        handler_class = nil
+        reopen
+      else
+        handler_class = find_handler_class(type)
+        if handler_class.nil?
+          logger.trace("process: done: no handler: <#{type}>")
+          return
+        end
+        process_type(handler_class, type, message)
       end
-      process_type(handler_class, type, message)
       logger.trace("process: done: <#{type}>",
                    :handler => handler_class)
     end
@@ -75,13 +73,28 @@ module Droonga
     private
     def prepare
       if @database_name and !@database_name.empty?
-        @context = Groonga::Context.new
-        @database = @context.open_database(@database_name)
+        open_database
       end
       logger.debug("#{self.class.name}: activating plugins for the dataset \"#{@dataset_name}\": " +
                      "#{@options[:plugins].join(", ")}")
       @step_runner = StepRunner.new(nil, @options[:plugins] || [])
-      @forwarder = Forwarder.new(@loop)
+      @forwarder = @options[:forwarder]
+    end
+
+    def close_database
+      @database.close
+      @context.close
+      @database = @context = nil
+    end
+
+    def open_database
+      @context = Groonga::Context.new
+      @database = @context.open_database(@database_name)
+    end
+
+    def reopen
+      close_database
+      open_database
     end
 
     def find_handler_class(type)

data/lib/droonga/job_pusher.rb

@@ -57,6 +57,12 @@ module Droonga
       logger.trace("push: done")
     end
 
+    def broadcast(message)
+      logger.trace("broadcast start")
+      @job_queue.broadcast(message)
+      logger.trace("broadcast done")
+    end
+
     private
     def log_tag
       "job_pusher"
@@ -104,6 +110,12 @@ module Droonga
         end
       end
 
+      def broadcast(message)
+        @workers.each do |worker|
+          worker.write(message.to_msgpack)
+        end
+      end
+
       private
       def supply_job(worker)
         if @buffers.empty?

data/lib/droonga/line_buffer.rb

@@ -0,0 +1,42 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class LineBuffer
+    def initialize
+      @buffer = ""
+    end
+
+    def feed(data)
+      position = 0
+      loop do
+        new_line_position = data.index("\n", position)
+        if new_line_position.nil?
+          @buffer << data[position..-1]
+          break
+        end
+
+        line = data[position..new_line_position]
+        if position.zero?
+          yield(@buffer + line)
+          @buffer.clear
+        else
+          yield(line)
+        end
+        position = new_line_position + 1
+      end
+    end
+  end
+end

data/lib/droonga/logger.rb

@@ -49,12 +49,12 @@ module Droonga
           LABELS[level]
         end
 
-        def default
-          WARN
+        def value(label)
+          LABELS.index(label.to_s)
         end
 
-        def default_label
-          label(default)
+        def default
+          ENV["DROONGA_LOG_LEVEL"] || label(WARN)
         end
       end
     end
@@ -73,7 +73,7 @@ module Droonga
     def initialize(options={})
       @output = options[:output] || self.class.default_output
       @tag = options[:tag]
-      self.level = ENV["DROONGA_LOG_LEVEL"] || Level.default_label
+      self.level = options[:level] || Level.default
     end
 
     def level
@@ -81,7 +81,11 @@ module Droonga
     end
 
     def level=(level)
-      @level = Level::LABELS.index(level.to_s)
+      if level.is_a?(Numeric)
+        @level = level
+      else
+        @level = Level.value(level)
+      end
     end
 
     def trace(message, data={})

data/lib/droonga/path.rb

@@ -38,10 +38,26 @@ module Droonga
         base + "state"
       end
 
+      def live_nodes
+        state + "live-nodes.json"
+      end
+
+      def last_processed_timestamp
+        state + "last-processed.timestamp"
+      end
+
+      def effective_message_timestamp
+        state + "effective-message.timestamp"
+      end
+
       def catalog
         base_file_name = ENV["DROONGA_CATALOG"] || "catalog.json"
         Pathname.new(base_file_name).expand_path(base)
       end
+
+      def buffer
+        state + "buffer"
+      end
     end
   end
 end

data/lib/droonga/plugins/search/distributed_search_planner.rb

@@ -72,7 +72,7 @@ module Droonga
           query = @queries[name]
           return true if query["groupBy"]
           name = query["source"]
-          return false unless @queries.keys.include?(name)
+          return false unless @queries.key?(name)
           unifiable?(name)
         end
 

data/lib/droonga/plugins/system.rb

@@ -0,0 +1,50 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+
+module Droonga
+  module Plugins
+    module System
+      extend Plugin
+      register("system")
+
+      class StatusHandler < Droonga::Handler
+        action.synchronous = true
+
+        def handle(message)
+          engine_state = @messenger.engine_state
+          live_nodes = engine_state.live_nodes
+          nodes = {}
+          engine_state.all_nodes.collect do |identifier|
+            nodes[identifier] = {
+              "live" => live_nodes.include?(identifier),
+            }
+          end
+
+          {
+            "nodes" => nodes,
+          }
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "system.status"
+        step.handler = StatusHandler
+        step.collector = Collectors::Or
+      end
+    end
+  end
+end

data/lib/droonga/processor.rb

@@ -15,7 +15,6 @@
 
 require "droonga/loggable"
 require "droonga/handler_runner"
-require "fileutils"
 
 module Droonga
   class Processor
@@ -47,9 +46,9 @@ module Droonga
         synchronous = @handler_runner.prefer_synchronous?(type)
         if @n_workers.zero? or synchronous
           @handler_runner.process(message)
-          #XXX Workaround to restart system by any schema change.
-          #    This should be done more smartly...
-          FileUtils.touch(Path.catalog.to_s) if synchronous
+          if synchronous
+            @job_pusher.broadcast(database_reopen_message)
+          end
         else
           @job_pusher.push(message)
         end
@@ -60,6 +59,12 @@ module Droonga
     end
 
     private
+    def database_reopen_message
+      {
+        "type" => "database.reopen",
+      }
+    end
+
     def log_tag
       "processor"
     end