droonga-engine 1.0.2 → 1.0.3

data/lib/droonga/engine/version.rb

@@ -15,6 +15,6 @@
 
 module Droonga
   class Engine
-    VERSION = "1.0.2"
+    VERSION = "1.0.3"
   end
 end

data/lib/droonga/engine_state.rb

@@ -13,6 +13,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "English"
+
 require "coolio"
 
 require "droonga/loggable"
@@ -26,15 +28,19 @@ module Droonga
 
     attr_reader :loop
     attr_reader :name
+    attr_reader :internal_name
     attr_reader :forwarder
     attr_reader :replier
-    def initialize(loop, name)
+    attr_accessor :on_finish
+    def initialize(loop, name, internal_name)
       @loop = loop
       @name = name
+      @internal_name = internal_name
       @sessions = {}
       @current_id = 0
       @forwarder = Forwarder.new(@loop)
       @replier = Replier.new(@forwarder)
+      @on_finish = nil
     end
 
     def start
@@ -50,13 +56,26 @@ module Droonga
     end
 
     def local_route?(route)
-      route.start_with?(@name)
+      route.start_with?(@name) or route.start_with?(@internal_name)
+    end
+
+    def farm_path(route)
+      if /\A[^:]+:\d+\/[^.]+/ =~ route
+        name = $MATCH
+        if name == @internal_name
+          @name
+        else
+          name
+        end
+      else
+        route
+      end
     end
 
     def generate_id
       id = @current_id
       @current_id = id.succ
-      return [@name, id].join(".#")
+      return [@internal_name, id].join(".#")
     end
 
     def find_session(id)
@@ -69,6 +88,13 @@ module Droonga
 
     def unregister_session(id)
       @sessions.delete(id)
+      unless have_session?
+        @on_finish.call if @on_finish
+      end
+    end
+
+    def have_session?
+      not @sessions.empty?
     end
 
     private

data/lib/droonga/internal_fluent_message_receiver.rb

@@ -0,0 +1,100 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "socket"
+require "ipaddr"
+
+require "droonga/fluent_message_receiver"
+
+module Droonga
+  class InternalFluentMessageReceiver
+    include Loggable
+
+    def initialize(loop, host, &on_message)
+      @loop = loop
+      @host = host
+      @on_message = on_message
+    end
+
+    def start
+      logger.trace("start: start")
+      start_listen_socket
+      start_heartbeat_socket
+      start_message_receiver
+      logger.trace("start: done")
+
+      [@host, @port]
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      shutdown_message_receiver
+      shutdown_heartbeat_socket
+      shutdown_listen_socket
+      logger.trace("shutdown: done")
+    end
+
+    private
+    def start_listen_socket
+      logger.trace("start_listen_socket: start")
+      @listen_socket = TCPServer.new(@host, 0)
+      @port = @listen_socket.addr[1]
+      logger.trace("start_listen_socket: done")
+    end
+
+    def shutdown_listen_socket
+      logger.trace("shutdown_listen_socket: start")
+      logger.trace("shutdown_listen_socket: done")
+    end
+
+    def address_family
+      ip_address = IPAddr.new(IPSocket.getaddress(@host))
+      ip_address.family
+    end
+
+    def start_heartbeat_socket
+      logger.trace("start_heartbeat_socket: start")
+      @heartbeat_socket = UDPSocket.new(address_family)
+      @heartbeat_socket.bind(@host, @port)
+      logger.trace("start_heartbeat_socket: done")
+    end
+
+    def shutdown_heartbeat_socket
+      logger.trace("shutdown_heartbeat_socket: start")
+      logger.trace("shutdown_heartbeat_socket: done")
+    end
+
+    def start_message_receiver
+      logger.trace("start_heartbeat_socket: start")
+      options = {
+        :listen_fd    => @listen_socket.fileno,
+        :heartbeat_fd => @heartbeat_socket.fileno,
+      }
+      @message_receiver = FluentMessageReceiver.new(@loop, options, &@on_message)
+      @message_receiver.start
+      logger.trace("start_heartbeat_socket: done")
+    end
+
+    def shutdown_message_receiver
+      logger.trace("shutdown_message_receiver: start")
+      @message_receiver.shutdown
+      logger.trace("shutdown_message_receiver: done")
+    end
+
+    def log_tag
+      "internal-fluent-message-receiver"
+    end
+  end
+end

data/lib/droonga/live_nodes_list_loader.rb

@@ -0,0 +1,48 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "pathname"
+require "json"
+
+module Droonga
+  class LiveNodesListLoader
+    def initialize(path)
+      @path = path
+    end
+
+    def load
+      list = parse
+      list.keys
+    end
+
+    private
+    def parse
+      return default_list unless @path.exist?
+
+      contents = @path.read
+      return default_list if contents.empty?
+
+      begin
+        JSON.parse(contents)
+      rescue JSON::ParserError
+        default_list
+      end
+    end
+
+    def default_list
+      {}
+    end
+  end
+end

data/lib/droonga/live_nodes_list_observer.rb

@@ -0,0 +1,72 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "fileutils"
+require "listen"
+
+require "droonga/path"
+require "droonga/loggable"
+require "droonga/live_nodes_list_loader"
+
+module Droonga
+  class LiveNodesListObserver
+    class << self
+      FILE_NAME = "live-nodes.json"
+
+      def path
+        Path.state + FILE_NAME
+      end
+    end
+
+    include Loggable
+
+    attr_accessor :on_update
+
+    def initialize
+    end
+
+    def start
+      path = self.class.path
+      file_name = path.expand_path.to_s
+      directory = path.dirname.to_s
+      FileUtils.mkdir_p(directory)
+      @listener = Listen.to(directory) do |modified, added, removed|
+        if added.include?(file_name) or
+             modified.include?(file_name)
+          load_list!
+        end
+      end
+      @listener.start
+    end
+
+    def stop
+      @listener.stop
+    end
+
+    def load_list!
+      path = self.class.path
+      loader = LiveNodesListLoader.new(path)
+      live_nodes = loader.load
+      logger.info("loaded", :path => path.to_s, :live_nodes => live_nodes)
+
+      on_update.call(live_nodes) if on_update
+    end
+
+    private
+    def log_tag
+      "live-nodes-list-observer"
+    end
+  end
+end

data/lib/droonga/path.rb

@@ -0,0 +1,47 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "pathname"
+
+module Droonga
+  module Path
+    BASE_DIR_ENV_NAME = "DROONGA_BASE_DIR"
+
+    class << self
+      def setup
+        base_dir = ENV[BASE_DIR_ENV_NAME] || Dir.pwd
+        ENV[BASE_DIR_ENV_NAME] = File.expand_path(base_dir)
+      end
+
+      def base
+        @base ||= Pathname.new(ENV[BASE_DIR_ENV_NAME] || Dir.pwd).expand_path
+      end
+
+      def base=(new_base)
+        @base = nil
+        ENV[BASE_DIR_ENV_NAME] = new_base
+      end
+
+      def state
+        base + "state"
+      end
+
+      def catalog
+        base_file_name = ENV["DROONGA_CATALOG"] || "catalog.json"
+        Pathname.new(base_file_name).expand_path(base)
+      end
+    end
+  end
+end

data/lib/droonga/plugins/dump.rb

@@ -26,65 +26,306 @@ module Droonga
 
       class Handler < Droonga::Handler
         def handle(message)
-          replyTo = (message.raw["replyTo"] || {})["to"]
-          return false unless replyTo
-
-          request = message.request || {}
-          messages_per_seconds = request["messagesPerSecond"] || 10000
-          messages_per_seconds = [10, messages_per_seconds.to_i].max
-          messages_per_100msec = messages_per_seconds / 10
-          dumper = Enumerator.new do |yielder|
-            n = 0
-            each_table do |table|
-              table.each do |record|
-                values = {}
-                record.attributes.each do |key, value|
-                  values[key] = value unless key.start_with?("_")
-                end
-                dump_message = {
-                  "dataset" => message.raw["dataset"],
-                  "body" => {
-                    "table" => table.name,
-                    "key" => record.key,
-                    "values" => values,
-                  },
-                }
-                messenger.forward(dump_message,
-                                  "to" => replyTo,
-                                  "type" => "dump.record")
-                n = (n + 1) % messages_per_100msec
-                yielder << nil if n.zero?
-              end
-            end
+          request = Request.new(message)
+          if request.need_dump?
+            dumper = Dumper.new(@context, loop, messenger, request)
+            dumper.start_dump
+            true
+          else
+            false
+          end
+        end
+      end
+
+      class Request
+        def initialize(message)
+          @message = message
+        end
+
+        def need_dump?
+          reply_to
+        end
+
+        def id
+          @message["id"]
+        end
+
+        def dataset
+          @message.raw["dataset"]
+        end
+
+        def reply_to
+          (@message.raw["replyTo"] || {})["to"]
+        end
+
+        def messages_per_seconds
+          request = (@message.request || {})
+          minimum_messages_per_seconds = 10
+          [
+            minimum_messages_per_seconds,
+            (request["messagesPerSecond"] || 10000).to_i,
+          ].max
+        end
+      end
+
+      class Dumper
+        include Loggable
+
+        def initialize(context, loop, messenger, request)
+          @context = context
+          @loop = loop
+          @messenger = messenger
+          @request = request
+        end
+
+        def start_dump
+          setup_forward_data
+
+          forward("dump.start")
+
+          dumper = Fiber.new do
+            dump_schema
+            dump_records
+            dump_indexes
+            forward("dump.end")
+          end
+
+          on_error = lambda do |exception|
+            message = "failed to dump"
+            logger.exception(message, $!)
+            error("DumpFailure", message)
           end
 
           timer = Coolio::TimerWatcher.new(0.1, true)
           timer.on_timer do
             begin
-              dumper.next
-            rescue StopIteration
+              dumper.resume
+            rescue FiberError
               timer.detach
+            rescue
+              timer.detach
+              on_error.call($!)
             end
           end
-          loop.attach(timer)
 
-          true
+          @loop.attach(timer)
         end
 
         private
+        def setup_forward_data
+          @base_forward_message = {
+            "inReplyTo" => @request.id,
+            "dataset"   => @request.dataset,
+          }
+          @forward_to = @request.reply_to
+          @n_forwarded_messages = 0
+          @messages_per_100msec = @request.messages_per_seconds / 10
+        end
+
+        def error(name, message)
+          message = {
+            "statusCode" => ErrorMessages::InternalServerError::STATUS_CODE,
+            "body" => {
+              "name"    => name,
+              "message" => message,
+            },
+          }
+          error_message = @base_forward_message.merge(message)
+          @messenger.forward(error_message,
+                             "to"   => @forward_to,
+                             "type" => "dump.error")
+        end
+
+        def forward(type, body=nil)
+          forward_message = @base_forward_message
+          if body
+            forward_message = forward_message.merge("body" => body)
+          end
+          @messenger.forward(forward_message,
+                             "to"   => @forward_to,
+                             "type" => type)
+
+          @n_forwarded_messages += 1
+          @n_forwarded_messages %= @messages_per_100msec
+          Fiber.yield if @n_forwarded_messages.zero?
+        end
+
+        def dump_schema
+          reference_tables = []
+          each_table do |table|
+            if reference_table?(table)
+              reference_tables << table
+              next
+            end
+            dump_table(table)
+          end
+
+          reference_tables.each do |table|
+            dump_table(table)
+          end
+        end
+
+        def dump_table(table)
+          forward("dump.table", table_body(table))
+
+          columns = table.columns.sort_by(&:name)
+          columns.each do |column|
+            next if index_column?(column)
+            dump_column(column)
+          end
+        end
+
+        def table_body(table)
+          body = {
+            "type" => table_type(table),
+            "name" => table.name,
+          }
+          if table.support_key?
+            body["keyType"] = table.domain.name
+          end
+          if body["keyType"] == "ShortText"
+            if table.default_tokenizer
+              body["tokenizer"] = table.default_tokenizer.name
+            end
+            if table.normalizer
+              body["normalizer"] = table.normalizer.name
+            end
+          end
+          body
+        end
+
+        def table_type(table)
+          table.class.name.split(/::/).last
+        end
+
+        def dump_column(column)
+          forward("dump.column", column_body(column))
+        end
+
+        def column_body(column)
+          body = {
+            "table"     => column.domain.name,
+            "name"      => column.local_name,
+            "type"      => column_type(column),
+            "valueType" => column.range.name,
+          }
+          case body["type"]
+          when "Index"
+            body["indexOptions"] = {
+              "section"  => column.with_section?,
+              "weight"   => column.with_weight?,
+              "position" => column.with_position?,
+              "sources"  => index_column_sources(column),
+            }
+          when "Vector"
+            body["vectorOptions"] = {
+              "weight" => column.with_weight?,
+            }
+          end
+          body
+        end
+
+        def column_type(column)
+          if index_column?(column)
+            "Index"
+          elsif column.vector?
+            "Vector"
+          else
+            "Scalar"
+          end
+        end
+
+        def index_column_sources(index_column)
+          index_column.sources.collect do |source|
+            if source.is_a?(::Groonga::Table)
+              "_key"
+            else
+              source.local_name
+            end
+          end
+        end
+
+        def dump_records
+          each_table do |table|
+            next if index_only_table?(table)
+            table.each do |record|
+              values = {}
+              record.attributes.each do |key, value|
+                next if key.start_with?("_")
+                values[key] = normalize_record_value(value)
+              end
+              body = {
+                "table"  => table.name,
+                "key"    => record.key,
+                "values" => values,
+              }
+              forward("dump.record", body)
+            end
+          end
+        end
+
+        def normalize_record_value(value)
+          case value
+          when Array
+            value.collect do |element|
+              case element
+              when Hash
+                element["_key"]
+              else
+                element
+              end
+            end
+          else
+            value
+          end
+        end
+
+        def dump_indexes
+          each_index_columns do |column|
+            dump_column(column)
+          end
+        end
+
         def each_table
-          @context.database.each(:ignore_missing_object => true) do |object|
-            next unless object.is_a?(::Groonga::Table)
-            next if index_only_table?(object)
+          options = {
+            :ignore_missing_object => true,
+            :order_by => :key,
+          }
+          @context.database.each(options) do |object|
+            next unless table?(object)
             yield(object)
           end
         end
 
+        def table?(object)
+          object.is_a?(::Groonga::Table)
+        end
+
         def index_only_table?(table)
           table.columns.all? do |column|
-            column.is_a?(::Groonga::IndexColumn)
+            index_column?(column)
           end
         end
+
+        def reference_table?(table)
+          table.support_key? and table?(table.domain)
+        end
+
+        def index_column?(column)
+          column.is_a?(::Groonga::IndexColumn)
+        end
+
+        def each_index_columns
+          each_table do |table|
+            table.columns.each do |column|
+              yield(column) if index_column?(column)
+            end
+          end
+        end
+
+        def log_tag
+          "[#{Process.ppid}][#{Process.pid}] dumper"
+        end
       end
 
       define_single_step do |step|