droonga-engine 1.0.5 → 1.0.6

data/lib/droonga/catalog/version1.rb

@@ -14,6 +14,8 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "English"
+require "pathname"
+require "fileutils"
 
 require "droonga/catalog/base"
 require "droonga/catalog/dataset"
@@ -51,11 +53,14 @@ module Droonga
             part["partitions"].each do |range, partitions|
               partitions.each do |partition|
                 if partition =~ pattern
-                  path = File.join([device, $POSTMATCH, "db"])
-                  path = File.expand_path(path, base_path)
+                  database_name = $POSTMATCH
+                  path = File.join([device, Path.databases.basename.to_s, database_name, "db"])
+                  path = Pathname(path).expand_path(base_path)
+                  migrate_database_location(path, :device => device,
+                                                  :name   => database_name)
                   options = {
                     :dataset => dataset_name,
-                    :database => path,
+                    :database => path.to_s,
                     :n_workers => workers,
                     :plugins => plugins
                   }
@@ -384,6 +389,14 @@ module Droonga
         end
       end
 
+      def migrate_database_location(path, params)
+        old_path = File.join([params[:device], params[:name], "db"])
+        old_path = Pathname(old_path).expand_path(base_path)
+        if old_path.exist? and not path.exist?
+          FileUtils.move(old_path.to_s, path.to_s)
+        end
+      end
+
       class Dataset < Catalog::Dataset
         def compute_routes(args, live_nodes=nil)
           routes = []

data/lib/droonga/catalog/version2.rb

@@ -13,6 +13,9 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "pathname"
+require "fileutils"
+
 require "droonga/address"
 require "droonga/catalog/base"
 require "droonga/catalog/dataset"
@@ -41,11 +44,13 @@ module Droonga
             volume.slices.each do |slice|
               volume_address = slice.volume.address
               if volume_address.node == node
-                path = File.join([device, volume_address.name, "db"])
-                path = File.expand_path(path, base_path)
+                path = File.join([device, Path.databases.basename.to_s, volume_address.name, "db"])
+                path = Pathname(path).expand_path(base_path)
+                migrate_database_location(path, :device => device,
+                                                :name   => volume_address.name)
                 options = {
                   :dataset => dataset_name,
-                  :database => path,
+                  :database => path.to_s,
                   :n_workers => n_workers,
                   :plugins => plugins
                 }
@@ -81,6 +86,14 @@ module Droonga
         end
         nodes.sort.uniq
       end
+
+      def migrate_database_location(path, params)
+        old_path = File.join([params[:device], params[:name], "db"])
+        old_path = Pathname(old_path).expand_path(base_path)
+        if old_path.exist? and not path.exist?
+          FileUtils.move(old_path.to_s, path.to_s)
+        end
+      end
     end
   end
 end

data/lib/droonga/catalog_fetcher.rb

@@ -0,0 +1,51 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "droonga/client"
+
+require "droonga/address"
+require "droonga/catalog/dataset"
+
+module Droonga
+  class CatalogFetcher
+    def initialize(client_options)
+      @client_options = default_options.merge(client_options)
+    end
+
+    def fetch(options={})
+      message = {
+        "dataset" => options[:dataset] || Catalog::Dataset::DEFAULT_NAME,
+        "type"    => "catalog.fetch"
+      }
+      Droonga::Client.open(@client_options) do |client|
+        response = client.request(message)
+        response["body"]
+      end
+    end
+
+    private
+    def default_options
+      {
+        :host          => "127.0.0.1",
+        :port          => Address::DEFAULT_PORT,
+        :tag           => Address::DEFAULT_TAG,
+        :protocol      => :droonga,
+        :timeout       => 1,
+        :receiver_host => "127.0.0.1",
+        :receiver_port => 0,
+      }
+    end
+  end
+end

data/lib/droonga/catalog_generator.rb

@@ -15,17 +15,18 @@
 
 require "time"
 
+require "droonga/address"
 require "droonga/catalog/dataset"
 
 module Droonga
   class CatalogGenerator
-    DEFAULT_DATASET = "Default"
-    DEFAULT_HOSTS = ["127.0.0.1"]
+    DEFAULT_DATASET = Catalog::Dataset::DEFAULT_NAME
+    DEFAULT_HOSTS = [Address::DEFAULT_HOST]
     DEFAULT_N_WORKERS = 4
     DEFAULT_N_SLICES = 1
-    DEFAULT_PLUGINS = ["groonga", "search", "crud", "dump", "system"]
-    DEFAULT_PORT = 10031
-    DEFAULT_TAG = "droonga"
+    DEFAULT_PLUGINS = ["groonga", "search", "crud", "dump", "system", "catalog"]
+    DEFAULT_PORT = Address::DEFAULT_PORT
+    DEFAULT_TAG = Address::DEFAULT_TAG
 
     attr_reader :datasets
 

data/lib/droonga/catalog_modifier.rb

@@ -0,0 +1,45 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "json"
+
+require "droonga/path"
+require "droonga/catalog_generator"
+require "droonga/safe_file_writer"
+
+module Droonga
+  class CatalogModifier
+    class << self
+      def modify
+        new.modify do |generator, file|
+          yield(generator, file)
+        end
+      end
+    end
+
+    def initialize
+      @generator = CatalogGenerator.new
+      @catalog = JSON.parse(Path.catalog.read)
+      @generator.load(@catalog)
+    end
+
+    def modify
+      SafeFileWriter.write(Path.catalog) do |output, file|
+        yield(@generator, file)
+        output.puts(JSON.pretty_generate(@generator.generate))
+      end
+    end
+  end
+end

data/lib/droonga/command/droonga_engine.rb

@@ -17,12 +17,16 @@ require "optparse"
 require "socket"
 require "ipaddr"
 require "fileutils"
+require "yaml"
 
 require "coolio"
 require "sigdump"
 
+require "droonga/engine/version"
 require "droonga/path"
+require "droonga/address"
 require "droonga/serf"
+require "droonga/node_status"
 require "droonga/file_observer"
 require "droonga/process_supervisor"
 
@@ -59,6 +63,7 @@ module Droonga
       private
       def parse_command_line_arguments!(command_line_arguments)
         parser = OptionParser.new
+        parser.version = Engine::VERSION
         @configuration.add_command_line_options(parser)
         parser.parse!(command_line_arguments)
       end
@@ -108,19 +113,74 @@ module Droonga
       end
 
       class Configuration
-        DEFAULT_HOST = Socket.gethostname
-        DEFAULT_PORT = 10031
-
         attr_reader :host, :port, :tag, :log_file, :pid_file_path
         attr_reader :ready_notify_fd
         def initialize
-          @host = DEFAULT_HOST
-          @port = DEFAULT_PORT
-          @tag = "droonga"
-          @log_file = nil
-          @daemon = false
-          @pid_file_path = nil
+          config = load_config
+
+          @host = config["host"] || Address::DEFAULT_HOST
+          @port = config["port"] || Address::DEFAULT_PORT
+          @tag  = config["tag"]  || Address::DEFAULT_TAG
+
+          @given_values = {}
+
+          @log_file        = nil
+          @daemon          = false
+          @pid_file_path   = nil
           @ready_notify_fd = nil
+
+          if have_config_file?
+            if config.include?("daemon")
+              @daemon = config["daemon"]
+            else
+              @daemon = true
+            end
+            if @daemon
+              self.pid_file_path = config["pid_file"] || Path.default_pid_file
+            end
+            self.log_file  = config["log_file"] || Path.default_log_file
+            self.log_level = config["log_level"] if config.include?("log_level")
+          end
+        end
+
+        def have_config_file?
+          File.exist?(Path.config)
+        end
+
+        def have_given_host?
+          @given_values.include?(:host)
+        end
+
+        def have_given_port?
+          @given_values.include?(:port)
+        end
+
+        def have_given_tag?
+          @given_values.include?(:tag)
+        end
+
+        def have_given_daemon?
+          @given_values.include?(:daemon)
+        end
+
+        def have_given_log_file?
+          @given_values.include?(:log_file)
+        end
+
+        def have_given_log_level?
+          @given_values.include?(:log_level)
+        end
+
+        def have_given_pid_file?
+          @given_values.include?(:pid_file)
+        end
+
+        def load_config
+          if have_config_file?
+            YAML.load_file(Path.config)
+          else
+            {}
+          end
         end
 
         def engine_name
@@ -136,6 +196,18 @@ module Droonga
           ENV["DROONGA_LOG_LEVEL"] || Logger::Level.default
         end
 
+        def log_level=(level)
+          ENV["DROONGA_LOG_LEVEL"] = level
+        end
+
+        def log_file=(file)
+          @log_file = File.expand_path(file)
+        end
+
+        def pid_file_path=(path)
+          @pid_file_path = Pathname.new(path).expand_path
+        end
+
         def daemon?
           @daemon
         end
@@ -171,16 +243,19 @@ module Droonga
                     "The host name of the Droonga engine",
                     "(#{@host})") do |host|
             @host = host
+            @given_values[:host] = host
           end
           parser.on("--port=PORT", Integer,
                     "The port number of the Droonga engine",
                     "(#{@port})") do |port|
             @port = port
+            @given_values[:port] = port
           end
           parser.on("--tag=TAG",
                     "The tag of the Droonga engine",
                     "(#{@tag})") do |tag|
             @tag = tag
+            @given_values[:tag] = tag
           end
         end
 
@@ -193,11 +268,13 @@ module Droonga
                     "The log level of the Droonga engine",
                     "[#{levels_label}]",
                     "(#{log_level})") do |level|
-            ENV["DROONGA_LOG_LEVEL"] = level
+            self.log_level = level
+            @given_values[:log_level] = log_level
           end
           parser.on("--log-file=FILE",
                     "Output logs to FILE") do |file|
-            @log_file = File.expand_path(file)
+            self.log_file = file
+            @given_values[:log_file] = file
           end
         end
 
@@ -207,10 +284,17 @@ module Droonga
           parser.on("--daemon",
                     "Run as a daemon") do
             @daemon = true
+            @given_values[:daemon] = true
+          end
+          parser.on("--no-daemon",
+                    "Run as a regular process") do
+            @daemon = false
+            @given_values[:daemon] = false
           end
           parser.on("--pid-file=PATH",
                     "Put PID to PATH") do |path|
-            @pid_file_path = Pathname.new(path).expand_path
+            self.pid_file_path = path
+            @given_values[:pid_file] = path
           end
         end
 
@@ -248,7 +332,6 @@ module Droonga
 
         def run
           @serf = run_serf
-          @serf_status_observer = run_serf_status_observer
           @service_runner = run_service
           setup_initial_on_ready
           @catalog_observer = run_catalog_observer
@@ -298,7 +381,6 @@ module Droonga
         def stop_gracefully
           @command_runner.stop
           @serf.stop
-          @serf_status_observer.stop
           @catalog_observer.stop
           @service_runner.stop_gracefully
         end
@@ -306,7 +388,6 @@ module Droonga
         def stop_immediately
           @command_runner.stop
           @serf.stop
-          @serf_status_observer.stop
           @catalog_observer.stop
           @service_runner.stop_immediately
         end
@@ -342,20 +423,6 @@ module Droonga
           serf
         end
 
-        def restart_serf
-          @serf.stop if @serf
-          @serf = run_serf
-        end
-
-        def run_serf_status_observer
-          serf_status_observer = FileObserver.new(@loop, Serf.status_file)
-          serf_status_observer.on_change = lambda do
-            restart_serf
-          end
-          serf_status_observer.start
-          serf_status_observer
-        end
-
         def run_catalog_observer
           catalog_observer = FileObserver.new(@loop, Path.catalog)
           catalog_observer.on_change = lambda do

data/lib/droonga/command/droonga_engine_service.rb

@@ -204,11 +204,16 @@ module Droonga
 
       def stop_gracefully
         return if @stopping
+        logger.trace("stop_gracefully: start")
         @stopping = true
         @receiver.stop_gracefully
         @engine.stop_gracefully do
           shutdown_worker_process_agent
           shutdown_internal_message_receiver
+          @receiver.shutdown_clients
+          logger.trace("stop_gracefully: done",
+                       :n_rest_watchers => @loop.watchers.size,
+                       :rest_watchers   => @loop.watchers)
         end
       end