droonga-engine 1.0.5 → 1.0.6

data/lib/droonga/path.rb

@@ -34,6 +34,10 @@ module Droonga
         ENV[BASE_DIR_ENV_NAME] = new_base
       end
 
+      def databases
+        base + "database"
+      end
+
       def state
         base + "state"
       end
@@ -50,18 +54,38 @@ module Droonga
         state + "effective-message.timestamp"
       end
 
+      def config
+        base + "droonga-engine.yaml"
+      end
+
+      def default_pid_file
+        base + "droonga-engine.pid"
+      end
+
+      def default_log_file
+        base + "droonga-engine.log"
+      end
+
       def catalog
         base_file_name = ENV["DROONGA_CATALOG"] || "catalog.json"
         Pathname.new(base_file_name).expand_path(base)
       end
 
-      def published(suffix)
-        base + "published-#{suffix}"
-      end
-
       def buffer
         state + "buffer"
       end
+
+      def serf_event_handler_errors
+        state + "serf-event-handler-errors"
+      end
+
+      def serf_event_handler_error_file
+        now = Time.now
+        name = sprintf("%04d-%02d-%02d_%02d-%02d-%02d.%d.error",
+                       now.year, now.month, now.day,
+                       now.hour, now.min, now.sec, now.nsec)
+        serf_event_handler_errors + name
+      end
     end
   end
 end

data/lib/droonga/plugins/catalog.rb

@@ -0,0 +1,40 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "droonga/plugin"
+require "droonga/path"
+
+module Droonga
+  module Plugins
+    module Catalog
+      extend Plugin
+      register("catalog")
+
+      class FetchHandler < Droonga::Handler
+        action.synchronous = false
+
+        def handle(message)
+          JSON.parse(Path.catalog.read)
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "catalog.fetch"
+        step.handler = FetchHandler
+        step.collector = Collectors::Or
+      end
+    end
+  end
+end

data/lib/droonga/safe_file_writer.rb

@@ -26,7 +26,7 @@ module Droonga
         FileUtils.mkdir_p(path.dirname.to_s)
         Tempfile.open(path.basename.to_s, path.dirname.to_s, "w") do |output|
           if block_given?
-            yield(output)
+            yield(output, output.path)
           else
             output.write(contents)
           end

data/lib/droonga/searcher.rb

@@ -82,26 +82,14 @@ module Droonga
     def search(queries)
       outputs = nil
       logger.trace("search: start", :queries => queries)
-      # TODO: THIS IS JUST A WORKAROUND! We should remove it ASAP!
-      disable_gc do
-        @context.push_memory_pool do
-          outputs = process_queries(queries)
-        end
+      @context.push_memory_pool do
+        outputs = process_queries(queries)
       end
       logger.trace("search: done")
-      return outputs
+      outputs
     end
 
     private
-    def disable_gc
-      GC.disable
-      begin
-        yield
-      ensure
-        GC.enable
-      end
-    end
-
     def process_queries(queries)
       logger.trace("process_queries: start")
       if queries.nil? or queries.empty?

data/lib/droonga/serf.rb

@@ -22,6 +22,7 @@ require "open3"
 require "droonga/path"
 require "droonga/loggable"
 require "droonga/catalog_loader"
+require "droonga/node_status"
 require "droonga/serf_downloader"
 require "droonga/line_buffer"
 
@@ -43,32 +44,6 @@ module Droonga
       def path
         Droonga::Path.base + "serf"
       end
-
-      def status_file
-        Droonga::Path.state + "status_file"
-      end
-
-      def load_status
-        if status_file.exist?
-          contents = status_file.read
-          unless contents.empty?
-            return JSON.parse(contents, :symbolize_names => true)
-          end
-        end
-        {}
-      end
-
-      def status(key)
-        load_status[key]
-      end
-
-      def send_query(name, query, payload)
-        new(nil, name).send_query(query, payload)
-      end
-
-      def live_nodes(name)
-        new(nil, name).live_nodes
-      end
     end
 
     include Loggable
@@ -111,11 +86,20 @@ module Droonga
       logger.trace("stop: done")
     end
 
+    def join(*hosts)
+      ensure_serf
+      nodes = hosts.collect do |host|
+        "#{host}:#{port}"
+      end
+      run_once("join", *nodes)
+    end
+
     def send_query(query, payload)
       ensure_serf
       options = ["-format", "json"] + additional_options_from_payload(payload)
       options += [query, JSON.generate(payload)]
       result = run_once("query", *options)
+      result[:result] = JSON.parse(result[:result])
       if payload["node"]
         responses = result[:result]["Responses"]
         response = responses[payload["node"]]
@@ -135,7 +119,8 @@ module Droonga
     def live_nodes
       ensure_serf
       nodes = {}
-      result= run_once("members", "-format", "json")
+      result = run_once("members", "-format", "json")
+      result[:result] = JSON.parse(result[:result])
       members = result[:result]
       members["members"].each do |member|
         if member["status"] == "alive"
@@ -212,13 +197,13 @@ module Droonga
       "#{extract_host(@name)}:7373"
     end
 
-    def status
-      @status ||= self.class.load_status
+    def node_status
+      @node_status ||= NodeStatus.new
     end
 
     def role
-      if status[:role]
-        role = status[:role].to_sym
+      if node_status.have?(:role)
+        role = node_status.get(:role).to_sym
         if self.class::ROLE.key?(role)
           return role
         end
@@ -282,7 +267,7 @@ module Droonga
       def run_once
         stdout, stderror, status = Open3.capture3(@serf, @command, *@options, :pgroup => true)
         {
-          :result => JSON.parse(stdout),
+          :result => stdout,
           :error  => stderror,
           :status => status,
         }

data/lib/droonga/serf_downloader.rb

@@ -27,8 +27,15 @@ module Droonga
   class SerfDownloader
     include Loggable
 
+    class DownloadFailed < StandardError
+    end
+
+    MAX_RETRY_COUNT = 5
+    RETRY_INTERVAL  = 10
+
     def initialize(output_path)
       @output_path = output_path
+      @retry_count = 0
     end
 
     def download
@@ -49,6 +56,24 @@ module Droonga
         FileUtils.mv("#{dir}/serf", absolete_output_path.to_s)
         FileUtils.chmod(0755, absolete_output_path.to_s)
       end
+    rescue Archive::Zip::UnzipError => archive_error
+      logger.warn("Downloaded zip file is broken.")
+      if @retry_count < MAX_RETRY_COUNT
+        @retry_count += 1
+        sleep(RETRY_INTERVAL * @retry_count)
+        download
+      else
+        raise DownloadFailed.new("Couldn't download serf executable. Try it later.")
+      end
+    rescue Faraday::ConnectionFailed => network_error
+      logger.warn("Connection failed.")
+      if @retry_count < MAX_RETRY_COUNT
+        @retry_count += 1
+        sleep(RETRY_INTERVAL * @retry_count)
+        download
+      else
+        raise DownloadFailed.new("Couldn't download serf executable. Try it later.")
+      end
     end
 
     private
@@ -77,7 +102,7 @@ module Droonga
       when /x86_64|x64/
         @architecture = "amd64"
       when /i\d86/
-        @architecture = "i386"
+        @architecture = "386"
       else
         raise "Unsupported architecture: #{RUBY_PLATFORM}"
       end

data/lib/droonga/service_installation.rb

@@ -0,0 +1,123 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "fileutils"
+
+require "droonga/path"
+
+module Droonga
+  class ServiceInstallation
+    class << self
+    end
+
+    class NotInstalledAsService < StandardError
+    end
+
+    def user_name
+      "droonga-engine"
+    end
+
+    def group_name
+      "droonga"
+    end
+
+    def base_directory
+      @base_directory ||= Pathname("/home/#{user_name}/droonga")
+    end
+
+    def ensure_using_service_base_directory
+      if user_exist?
+        Path.base = base_directory.to_s
+      end
+    end
+
+    def have_read_permission?
+      test_file = Path.config
+      begin
+        test_file.read
+      rescue Errno::EACCES => error
+        return false
+      end
+      true
+    end
+
+    def have_write_permission?
+      test_file = Path.base + "#{Time.now.to_i}.test"
+      begin
+        FileUtils.mkdir_p(Path.base)
+        FileUtils.touch(test_file.to_s)
+      rescue Errno::EACCES => error
+      end
+      unless test_file.exist?
+        return false
+      end
+      FileUtils.rm_f(test_file.to_s)
+      true
+    end
+
+    def user_exist?
+      system("id", user_name,
+             :out => "/dev/null",
+             :err => "/dev/null")
+    end
+
+    def installed_as_service?
+      return false unless user_exist?
+    
+      #TODO: we should support systemd also...
+      succeeded = system("service", "droonga-engine", "status",
+                         :out => "/dev/null",
+                         :err => "/dev/null")
+      return true if succeeded
+    
+      #TODO: we should support systemd also...
+      result = `service droonga-engine status`
+      result.include?("running") or \
+        result.include?("droonga-engine is stopped") or \
+        result.include?("droonga-engine dead")
+    end
+
+    def ensure_correct_file_permission(file)
+      if user_exist?
+        FileUtils.chown_R(user_name, group_name, file)
+        FileUtils.chmod_R("go+r", file)
+      end
+    end
+
+    def running?(pid_file_path=nil)
+      raise NotInstalledAsService.new unless installed_as_service?
+      #TODO: we should support systemd also...
+      system("service", "droonga-engine", "status",
+             :out => "/dev/null",
+             :err => "/dev/null")
+    end
+
+    def start
+      raise NotInstalledAsService.new unless installed_as_service?
+      #TODO: we should support systemd also...
+      system("service", "droonga-engine", "start",
+             :out => "/dev/null",
+             :err => "/dev/null")
+    end
+
+    def stop
+      raise NotInstalledAsService.new unless installed_as_service?
+      #TODO: we should support systemd also...
+      system("service", "droonga-engine", "stop",
+             :out => "/dev/null",
+             :err => "/dev/null")
+    end
+  end
+end

data/lib/droonga/session.rb

@@ -13,8 +13,12 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "droonga/loggable"
+
 module Droonga
   class Session
+    include Loggable
+
     def initialize(id, dispatcher, collector_runner, tasks, inputs)
       @id = id
       @dispatcher = dispatcher

data/lib/droonga/slice.rb

@@ -24,7 +24,7 @@ module Droonga
   class Slice
     include Loggable
 
-    attr_accessor :on_ready
+    attr_writer :on_ready
     def initialize(dataset, loop, options={})
       @dataset = dataset
       @loop = loop
@@ -44,12 +44,29 @@ module Droonga
       start_supervisor
     end
 
-    def shutdown
-      logger.trace("shutdown: start")
-      shutdown_supervisor if @supervisor
+    def stop_gracefully
+      logger.trace("stop_gracefully: start")
+      on_stop = lambda do
+        @job_pusher.shutdown
+        @processor.shutdown
+        yield if block_given?
+      end
+      if @supervisor
+        @supervisor.stop_gracefully do
+          on_stop.call
+        end
+      else
+        on_stop.call
+      end
+      logger.trace("stop_gracefully: done")
+    end
+
+    def stop_immediately
+      logger.trace("stop_immediately: start")
+      @supervisor.stop_immediately if @supervisor
       @job_pusher.shutdown
       @processor.shutdown
-      logger.trace("shutdown: done")
+      logger.trace("stop_immediately: done")
     end
 
     def process(message)
@@ -106,13 +123,6 @@ module Droonga
       @supervisor.start
     end
 
-    def shutdown_supervisor
-      logger.trace("supervisor: shutdown: start")
-      @supervisor.stop_gracefully
-      logger.trace("supervisor: shutdown: done")
-    end
-
-    private
     def on_ready
       @on_ready.call if @on_ready
     end

data/lib/droonga/supervisor.rb

@@ -45,9 +45,18 @@ module Droonga
     end
 
     def stop_gracefully
+      logger.trace("stop_gracefully: start")
+      n_worker_runners = @worker_runners.size
+      n_done_worker_runners = 0
       @worker_runners.each do |worker_runner|
-        worker_runner.stop_gracefully
+        worker_runner.stop_gracefully do
+          n_done_worker_runners += 1
+          if n_done_worker_runners == n_worker_runners
+            yield if block_given?
+          end
+        end
       end
+      logger.trace("stop_gracefully: done")
     end
 
     def stop_immediately
@@ -87,6 +96,7 @@ module Droonga
         @config = config
         @on_ready = nil
         @on_failure = nil
+        @stop_gracefully_callback = nil
       end
 
       def start
@@ -117,8 +127,11 @@ module Droonga
         @supervisor.start
       end
 
-      def stop_gracefully
+      def stop_gracefully(&block)
+        logger.trace("stop_gracefully: start")
         @supervisor.stop_gracefully
+        @stop_gracefully_callback = block
+        logger.trace("stop_gracefully: done")
       end
 
       def stop_immediately
@@ -159,6 +172,7 @@ module Droonga
         @success = status.success?
         @supervisor.stop
         on_failure unless success?
+        @stop_gracefully_callback.call if @stop_gracefully_callback
       end
 
       private

data/lib/droonga/worker_process_agent.rb

@@ -17,10 +17,12 @@ require "coolio"
 
 require "droonga/process_control_protocol"
 require "droonga/line_buffer"
+require "droonga/loggable"
 
 module Droonga
   class WorkerProcessAgent
     include ProcessControlProtocol
+    include Loggable
 
     def initialize(loop, input, output)
       @loop = loop
@@ -31,20 +33,26 @@ module Droonga
     end
 
     def start
+      logger.trace("start: start")
       @loop.attach(@input)
       @loop.attach(@output)
+      logger.trace("start: done")
     end
 
     def stop
+      logger.trace("stop: start")
       if @output
         @output, output = nil, @output
         output.write(Messages::FINISH)
-        output.close
+        output.on_write_complete do
+          output.close
+        end
       end
       if @input
         @input, input = nil, @input
         input.close
       end
+      logger.trace("stop: done")
     end
 
     def ready
@@ -107,5 +115,9 @@ module Droonga
     def on_stop_immediately
       @on_stop_immediately.call if @on_stop_immediately
     end
+
+    def log_tag
+      "worker_process_agent"
+    end
   end
 end

data/sample/droonga-engine.yaml

@@ -0,0 +1,5 @@
+host:      192.168.0.10
+port:      10031
+tag:       droonga
+log_file:  droonga-engine.log
+log_level: info

data/test/command/config/default/catalog.json

@@ -4,7 +4,7 @@
   "datasets": {
     "Default": {
       "nWorkers": 2,
-      "plugins": ["groonga", "crud", "search", "dump", "system"],
+      "plugins": ["groonga", "crud", "search", "dump", "system", "catalog"],
       "replicas": [
         {
           "dimension": "_key",

data/test/command/config/default/droonga-engine.yaml

@@ -0,0 +1,4 @@
+host:   127.0.0.1
+port:   10031
+tag:    droonga
+daemon: false

data/test/command/config/version1/catalog.json

@@ -11,7 +11,7 @@
   "datasets": {
     "Default": {
       "workers": 2,
-      "plugins": ["groonga", "crud", "search", "dump", "system"],
+      "plugins": ["groonga", "crud", "search", "dump", "system", "catalog"],
       "number_of_replicas": 2,
       "number_of_partitions": 2,
       "partition_key": "_key",

data/test/command/suite/catalog/fetch.expected

@@ -0,0 +1,64 @@
+{
+  "inReplyTo": "request-id",
+  "statusCode": 200,
+  "type": "catalog.fetch.result",
+  "body": {
+    "version": 2,
+    "effectiveDate": "2014-02-28T00:00:00Z",
+    "datasets": {
+      "Default": {
+        "nWorkers": 2,
+        "plugins": [
+          "groonga",
+          "crud",
+          "search",
+          "dump",
+          "system",
+          "catalog"
+        ],
+        "replicas": [
+          {
+            "dimension": "_key",
+            "slicer": "hash",
+            "slices": [
+              {
+                "label": "slice000",
+                "weight": 50,
+                "volume": {
+                  "address": "127.0.0.1:23003/droonga.000"
+                }
+              },
+              {
+                "label": "slice001",
+                "weight": 50,
+                "volume": {
+                  "address": "127.0.0.1:23003/droonga.001"
+                }
+              }
+            ]
+          },
+          {
+            "dimension": "_key",
+            "slicer": "hash",
+            "slices": [
+              {
+                "label": "slice010",
+                "weight": 50,
+                "volume": {
+                  "address": "127.0.0.1:23003/droonga.010"
+                }
+              },
+              {
+                "label": "slice011",
+                "weight": 50,
+                "volume": {
+                  "address": "127.0.0.1:23003/droonga.011"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    }
+  }
+}

data/test/command/suite/catalog/fetch.test

@@ -0,0 +1,6 @@
+#@require-catalog-version 2
+{
+  "type": "catalog.fetch",
+  "dataset": "Default",
+  "body": {}
+}