droonga-engine 1.0.4 → 1.0.5

data/lib/droonga/farm.rb

@@ -13,12 +13,13 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/slice"
 
 module Droonga
   class Farm
+    attr_writer :on_ready
     def initialize(name, catalog, loop, options={})
       @name = name
       @catalog = catalog
@@ -36,7 +37,14 @@ module Droonga
     end
 
     def start
+      n_ready_slices = 0
       @slices.each_value do |slice|
+        slice.on_ready = lambda do
+          n_ready_slices += 1
+          if n_ready_slices == @slices.size
+            @on_ready.call if @on_ready
+          end
+        end
         slice.start
       end
     end

data/lib/droonga/file_observer.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "coolio"
 

data/lib/droonga/fluent_message_receiver.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "socket"
 
@@ -40,12 +40,18 @@ module Droonga
       logger.trace("start: done")
     end
 
-    def shutdown
-      logger.trace("shutdown: start")
+    def stop_gracefully
+      logger.trace("stop_gracefully: start")
+      shutdown_heartbeat_receiver
       shutdown_server
+      logger.trace("stop_gracefully: done")
+    end
+
+    def stop_immediately
+      logger.trace("stop_immediately: start")
+      stop_gracefully
       shutdown_clients
-      shutdown_heartbeat_receiver
-      logger.trace("shutdown: done")
+      logger.trace("stop_immediately: done")
     end
 
     private

data/lib/droonga/fluent_message_sender.rb

@@ -13,12 +13,12 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "fileutils"
 require "thread"
 
-require "cool.io"
+require "coolio"
 
 require "droonga/message-pack-packer"
 
@@ -29,11 +29,12 @@ module Droonga
   class FluentMessageSender
     include Loggable
 
-    def initialize(loop, host, port)
+    def initialize(loop, host, port, options={})
       @loop = loop
       @host = host
       @port = port
       @socket = nil
+      @buffering = options[:buffering]
     end
 
     def start
@@ -55,17 +56,8 @@ module Droonga
       logger.trace("send: done")
     end
 
-    def reserve_send(tag, data)
-      logger.trace("reserve_send: start")
-      packed_fluent_message = create_packed_fluent_message(tag, data)
-      connect unless connected?
-      @socket.reserve_write(packed_fluent_message)
-      logger.trace("reserve_send: done")
-    end
-
     def resume
-      connect
-      @socket.resume
+      connect unless connected?
     end
 
     private
@@ -90,9 +82,14 @@ module Droonga
         @socket = nil
       end
 
-      data_directory = Path.buffer + "#{@host}:#{@port}"
-      FileUtils.mkdir_p(data_directory.to_s)
-      @socket = BufferedTCPSocket.connect(@host, @port, data_directory)
+      if @buffering
+        data_directory = Path.buffer + "#{@host}:#{@port}"
+        FileUtils.mkdir_p(data_directory.to_s)
+        @socket = BufferedTCPSocket.connect(@host, @port, data_directory)
+        @socket.resume
+      else
+        @socket = Coolio::TCPSocket.connect(@host, @port)
+      end
       @socket.on_write_complete do
         log_write_complete.call
       end
@@ -121,7 +118,7 @@ module Droonga
     end
 
     def log_tag
-      "[#{Process.ppid}][#{Process.pid}] fluent-message-sender"
+      "[#{Process.ppid}] fluent-message-sender"
     end
   end
 end

data/lib/droonga/forwarder.rb

@@ -13,24 +13,27 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/loggable"
 require "droonga/path"
 require "droonga/event_loop"
+require "droonga/buffered_tcp_socket"
 require "droonga/fluent_message_sender"
 
 module Droonga
   class Forwarder
     include Loggable
 
-    def initialize(loop)
+    def initialize(loop, options={})
       @loop = loop
+      @buffering = options[:buffering]
       @senders = {}
     end
 
     def start
       logger.trace("start: start")
+      resume
       logger.trace("start: done")
     end
 
@@ -47,8 +50,7 @@ module Droonga
       command = destination["type"]
       receiver = destination["to"]
       arguments = destination["arguments"]
-      reserve = destination["reserve"]
-      output(receiver, message, command, arguments, :reserve => reserve)
+      output(receiver, message, command, arguments)
       logger.trace("forward: done")
     end
 
@@ -56,10 +58,19 @@ module Droonga
       return unless Path.buffer.exist?
       Pathname.glob("#{Path.buffer}/*") do |path|
         next unless path.directory?
-        next if Pathname.glob("#{path.to_s}/*").empty?
 
         destination = path.basename.to_s
-        next if @senders.key?(destination)
+        sender = @senders[destination]
+        if sender
+          sender.resume
+          next
+        end
+
+        chunk_loader = BufferedTCPSocket::ChunkLoader.new(path)
+        unless chunk_loader.have_any_chunk?
+          #FileUtils.rm_rf(path.to_s) # TODO re-enable this
+          next
+        end
 
         components = destination.split(":")
         port = components.pop.to_i
@@ -104,11 +115,7 @@ module Droonga
       output_tag = "#{tag}.message"
       log_info = "<#{receiver}>:<#{output_tag}>"
       logger.trace("output: post: start: #{log_info}")
-      if options[:reserve]
-        sender.reserve_send(output_tag, message)
-      else
-        sender.send(output_tag, message)
-      end
+      sender.send(output_tag, message)
       logger.trace("output: post: done: #{log_info}")
       logger.trace("output: done")
     end
@@ -132,13 +139,16 @@ module Droonga
     end
 
     def create_sender(host, port)
-      sender = FluentMessageSender.new(@loop, host, port)
+      options = {
+        :buffering => @buffering,
+      }
+      sender = FluentMessageSender.new(@loop, host, port, options)
       sender.start
       sender
     end
 
     def log_tag
-      "[#{Process.ppid}][#{Process.pid}] forwarder"
+      "[#{Process.ppid}] forwarder"
     end
   end
 end

data/lib/droonga/handler.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/pluggable"
 require "droonga/loggable"

data/lib/droonga/handler_message.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class HandlerMessage

data/lib/droonga/handler_messenger.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/replier"
 require "droonga/forwarder"
@@ -114,7 +114,7 @@ module Droonga
 
     private
     def log_tag
-      "[#{Process.ppid}][#{Process.pid}] handler_messenger"
+      "[#{Process.ppid}] handler_messenger"
     end
   end
 end

data/lib/droonga/handler_runner.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "groonga"
 
@@ -132,7 +132,7 @@ module Droonga
     end
 
     def log_tag
-      "[#{Process.ppid}][#{Process.pid}] handler"
+      "[#{Process.ppid}] handler"
     end
   end
 end

data/lib/droonga/input_message.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class InputMessage

data/lib/droonga/internal_fluent_message_receiver.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "socket"
 require "ipaddr"
@@ -87,9 +87,10 @@ module Droonga
       logger.trace("start_heartbeat_socket: done")
     end
 
+    # TODO: Use stop_gracefully/stop_immediately interface
     def shutdown_message_receiver
       logger.trace("shutdown_message_receiver: start")
-      @message_receiver.shutdown
+      @message_receiver.stop_immediately
       logger.trace("shutdown_message_receiver: done")
     end
 

data/lib/droonga/job_protocol.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module JobProtocol

data/lib/droonga/job_pusher.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "msgpack"
 

data/lib/droonga/job_receiver.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "msgpack"
 

data/lib/droonga/line_buffer.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class LineBuffer

data/lib/droonga/live_nodes_list_loader.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "pathname"
 require "json"

data/lib/droonga/loggable.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/logger"
 

data/lib/droonga/logger.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class << self
@@ -110,7 +110,7 @@ module Droonga
 
     def exception(message, exception, data={})
       log(Level::ERROR,
-          "#{message}: #{exception.message}(#{exception.class})",
+          "#{message}: #{exception.class}: #{exception.message}",
           data)
       log_backtrace(Level::ERROR, exception.backtrace)
     end
@@ -133,7 +133,7 @@ module Droonga
     end
 
     def build_log_line(level, message, data={})
-      line = "#{Time.now.iso8601}[#{Level.label(level)}]: "
+      line = "#{Time.now.iso8601}[#{Process.pid}][#{Level.label(level)}]: "
       line << "#{@tag}: " if @tag
       line << message
       data.each do |key, value|

data/lib/droonga/message_matcher.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   # It checks whether the pattern matches against a message.

data/lib/droonga/output_message.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/status_code"
 

data/lib/droonga/path.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "pathname"
 
@@ -55,6 +55,10 @@ module Droonga
         Pathname.new(base_file_name).expand_path(base)
       end
 
+      def published(suffix)
+        base + "published-#{suffix}"
+      end
+
       def buffer
         state + "buffer"
       end

data/lib/droonga/planner.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/loggable"
 require "droonga/distributed_command_planner"

data/lib/droonga/pluggable.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Pluggable

data/lib/droonga/plugin/metadata/adapter_input_message.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Plugin

data/lib/droonga/plugin/metadata/adapter_output_message.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Plugin

data/lib/droonga/plugin/metadata/collector_message.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Plugin

data/lib/droonga/plugin/metadata/handler_action.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Plugin

data/lib/droonga/plugin/metadata/input_message.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Plugin

data/lib/droonga/plugin.rb

@@ -11,13 +11,14 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/plugin_registry"
 require "droonga/single_step_definition"
 require "droonga/adapter"
 require "droonga/planner"
 require "droonga/handler"
+require "droonga/collector"
 require "droonga/collectors"
 
 module Droonga

data/lib/droonga/plugin_loader.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "pathname"
 

data/lib/droonga/plugin_registry.rb

@@ -11,7 +11,9 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "droonga/error"
 
 module Droonga
   class PluginRegistry

data/lib/droonga/plugins/basic.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/plugin"
 require "droonga/reducer"

data/lib/droonga/plugins/crud.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "groonga"
 

data/lib/droonga/plugins/dump.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "groonga"
 
@@ -20,6 +20,17 @@ require "droonga/error_messages"
 
 module Droonga
   module Plugins
+    # TODO: Implement it by superstep:
+    #  * Schema: Choose one slice in a replica.
+    #    Because all slices has the same schema.
+    #    (Should we add "select" type to Dataset#copmute_routes for the case?)
+    #  * Fact table: Choose all slices in a replica.
+    #    Because records for the fact table are distributed in all slices.
+    #    (Should we add "gather" type to Dataset#copmute_routes for the case?)
+    #  * Other tables: Choose one slice in a replica.
+    #    Because all slices has all records for other tables.
+    #    (Should we add "select" type to Dataset#copmute_routes for the case?)
+
     module Dump
       extend Plugin
       register("dump")
@@ -324,7 +335,7 @@ module Droonga
         end
 
         def log_tag
-          "[#{Process.ppid}][#{Process.pid}] dumper"
+          "[#{Process.ppid}] dumper"
         end
       end
 

data/lib/droonga/plugins/error.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/plugin"
 

data/lib/droonga/plugins/groonga/column_create.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "groonga/command/column-create"