droonga-engine 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e783348c7bfbef6e04de3de388c902839e8633b
-  data.tar.gz: 01b3367f3c22fb2a40245ca9bedc4920b42a494a
+  metadata.gz: 3e2a22b317f8d67460835b19b148b87065317f62
+  data.tar.gz: 2dd51cbb6b2ed3f667474d45f0735f5626a989e7
 SHA512:
-  metadata.gz: 399c8b21fd8368610a2ad498246584614f4bde6bc99bed78f8ba705164627a920ea59e32311ef5bf565fcaeb838e58837ab200a3d2ed0b2ee56f7ca26f1efa0e
-  data.tar.gz: 30bdef9c53baa70dd2095869656283488517d579942bc214c9267ff7c095e2c0aac9ff0d0e01db59263f4d4c86cf6f2165e5d4681db9af21726a2d69f4d798af
+  metadata.gz: 64d02366c1dc7945091afeec62787ea1082091edb08e34f844d32cbc39ebed5839ff36cf840c4dbd7c89dff92d95e5037fbdfeadaee58af129c1aeaea8316a5c
+  data.tar.gz: b7c88b9cab4a8f5b438d789cb78cee50080efb4d78b826829b1057a3412d7b930fe99ab52ee2e2d899e98ca3725c4df244461671598e51adf455dafb99d55ce7

data/bin/droonga-engine-absorb-data

@@ -79,7 +79,8 @@ end
 destination_node = "#{options.destination_host}:#{options.port}/#{options.tag}"
 
 def run_remote_command(target, command, options)
-  result = Droonga::Serf.send_query(target, command, options)
+  serf = Droonga::Serf.new(nil, target)
+  result = serf.send_query(command, options)
   puts result[:result]
   puts result[:error] unless result[:error].empty?
   result[:response]

data/bin/droonga-engine-catalog-generate

@@ -23,6 +23,10 @@ require "pathname"
 require "droonga/engine/version"
 require "droonga/catalog_generator"
 require "droonga/safe_file_writer"
+require "droonga/service_installation"
+
+service_installation = Droonga::ServiceInstallation.new
+service_installation.ensure_using_service_base_directory
 
 generator = Droonga::CatalogGenerator.new
 current_dataset = {}
@@ -31,7 +35,11 @@ datasets = {
 }
 
 options = OpenStruct.new
-options.output_path = "-"
+if service_installation.user_exist?
+  options.output_path = Droonga::Path.catalog
+else
+  options.output_path = "-"
+end
 parser = OptionParser.new
 parser.version = Droonga::Engine::VERSION
 parser.on("--output=PATH",
@@ -98,21 +106,29 @@ if datasets[Droonga::CatalogGenerator::DEFAULT_DATASET].empty?
   datasets.delete(Droonga::CatalogGenerator::DEFAULT_DATASET)
 end
 
+if service_installation.user_exist? and
+     options.output_path == Droonga::Path.catalog
+  options.for_service = true
+end
+
 datasets.each do |name, options|
   generator.add_dataset(name, options)
 end
 
 def open_output(path)
   if path == "-"
-    yield($stdout)
+    yield($stdout, nil)
   else
-    Droonga::SafeFileWriter.write(path) do |output|
-      yield(output)
+    Droonga::SafeFileWriter.write(path) do |output, file|
+      yield(output, file)
     end
   end
 end
 
 catalog = generator.generate
-open_output(options.output_path) do |output|
+open_output(options.output_path) do |output, file|
   output.puts(JSON.pretty_generate(catalog))
+  if file and options.for_service
+    service_installation.ensure_correct_file_permission(file)
+  end
 end

data/bin/droonga-engine-catalog-modify

@@ -23,6 +23,10 @@ require "pathname"
 require "droonga/engine/version"
 require "droonga/catalog_generator"
 require "droonga/safe_file_writer"
+require "droonga/service_installation"
+
+service_installation = Droonga::ServiceInstallation.new
+service_installation.ensure_using_service_base_directory
 
 generator = Droonga::CatalogGenerator.new
 current_dataset = {}
@@ -31,9 +35,14 @@ datasets = {
 }
 
 options = OpenStruct.new
-options.source_path = "./catalog.json"
 options.output_path = "-"
-options.update      = false
+if service_installation.user_exist?
+  options.source_path = Droonga::Path.catalog
+  options.update      = true
+else
+  options.source_path = "./catalog.json"
+  options.update      = false
+end
 parser = OptionParser.new
 parser.version = Droonga::Engine::VERSION
 parser.on("--source=PATH",
@@ -76,6 +85,10 @@ parser.parse!(ARGV)
 if options.source_path != "-" and options.update
   options.output_path = options.source_path
 end
+if service_installation.user_exist? and
+     options.output_path == Droonga::Path.catalog
+  options.for_service = true
+end
 
 def load_source(path)
   source = nil
@@ -94,15 +107,18 @@ generator.modify(datasets)
 
 def open_output(path)
   if path == "-"
-    yield($stdout)
+    yield($stdout, nil)
   else
-    Droonga::SafeFileWriter.write(path) do |output|
-      yield(output)
+    Droonga::SafeFileWriter.write(path) do |output, file|
+      yield(output, file)
     end
   end
 end
 
 catalog = generator.generate
-open_output(options.output_path) do |output|
+open_output(options.output_path) do |output, file|
   output.puts(JSON.pretty_generate(catalog))
+  if file and options.for_service
+    service_installation.ensure_correct_file_permission(file)
+  end
 end

data/bin/droonga-engine-configure

@@ -0,0 +1,215 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+require "fileutils"
+require "yaml"
+
+require "droonga/path"
+require "droonga/command/droonga_engine"
+require "droonga/safe_file_writer"
+require "droonga/service_installation"
+
+options = {
+  :quiet => nil,
+  :clear => nil,
+  :reset_config  => nil,
+  :reset_catalog => nil,
+}
+
+configuration = Droonga::Command::DroongaEngine::Configuration.new
+parser = OptionParser.new
+parser.on("--quiet", "Run with no prompt.") do |host|
+  options[:quiet] = true
+end
+parser.on("--clear", "Clear any existing data.") do |host|
+  options[:clear] = true
+end
+parser.on("--reset-config", "Regenerate the configuration file \"droonga-engine.yaml\".") do |host|
+  options[:reset_config] = true
+end
+parser.on("--reset-catalog", "Regenerate the \"catalog.json\".") do |host|
+  options[:reset_catalog] = true
+end
+configuration.add_command_line_options(parser)
+parser.parse!(ARGV)
+
+
+def unjoin(configuration)
+  system("droonga-engine-unjoin",
+         "--host", configuration.host,
+         :out => "/dev/null",
+         :err => "/dev/null")
+end
+
+def input(message, default_value=nil)
+  print "#{message} [#{default_value}]: "
+  response = gets.strip
+  if response.empty?
+    default_value
+  else
+    response
+  end
+end
+
+def confirmed?(message)
+  while true
+    print "#{message} (y/N): "
+    response = gets
+    case response
+    when /\Ay/i
+      return true
+    when /\An/i, /^$/
+      return false
+    end
+  end
+end
+
+def path_from_base_dir(path)
+  path = path.to_s
+  base = "#{Droonga::Path.base.to_s}/"
+  if path.start_with?(base)
+    path.sub(base, "")
+  else
+    path
+  end
+end
+
+
+service_installation = Droonga::ServiceInstallation.new
+service_installation.ensure_using_service_base_directory
+
+running = false
+begin
+  if service_installation.running?
+    if !options[:quiet]
+      puts("The droonga-engine service is now running.")
+      puts("Before reconfiguration, the service is going to be stopped " +
+             "and this node will be unjoined from the cluster.")
+      unless confirmed?("Are you sure you want to continue reconfiguration?")
+        exit(false)
+      end
+    end
+    running = true
+  end
+rescue Droonga::ServiceInstallation::NotInstalledAsService
+  puts("Not installed as a service yet.")
+end
+
+unless service_installation.have_write_permission?
+  puts("You have no permission to write files under " +
+         "<#{Droonga::Path.base.to_s}>.")
+  puts("Try again with right permission.")
+  exit(false)
+end
+
+
+data_files = [
+  Droonga::Path.databases,
+  Droonga::Path.state,
+]
+have_data = data_files.any?(&:exist?)
+options[:clear] = false unless have_data
+
+if !options[:quiet] and options[:clear].nil?
+  options[:clear] = confirmed?("Do you want all data to be cleared?")
+end
+
+
+options[:reset_config] = true unless Droonga::Path.config.exist?
+if !options[:quiet] and options[:reset_config].nil?
+  options[:reset_config] = confirmed?("Do you want the configuration file " +
+                                        "\"droonga-engine.yaml\" to be regenerated?")
+end
+
+options[:reset_catalog] = true unless Droonga::Path.catalog.exist?
+if !options[:quiet] and options[:reset_catalog].nil?
+  options[:reset_catalog] = confirmed?("Do you want the file \"catalog.json\" " +
+                                         "to be regenerated?")
+end
+
+
+if running
+  unjoin(configuration)
+  service_installation.stop
+end
+
+if options[:clear]
+  data_files.each do |file|
+    FileUtils.rm_rf(file.to_s)
+  end
+end
+
+if options[:reset_config] or options[:reset_catalog]
+  if configuration.have_given_host? or options[:quiet]
+    host = configuration.host
+  else
+    host = input("host", configuration.host)
+  end
+
+  if configuration.have_given_port? or options[:quiet]
+    port = configuration.port
+  else
+    port = input("port", configuration.port).to_i
+  end
+
+  if configuration.have_given_tag? or options[:quiet]
+    tag = configuration.tag
+  else
+    tag = input("tag", configuration.tag)
+  end
+end
+
+if options[:reset_config]
+  if configuration.have_given_log_level? or options[:quiet]
+    log_level = configuration.log_level
+  else
+    log_level = input("log level", configuration.log_level)
+  end
+
+  new_configuration = {
+    "host"      => host,
+    "port"      => port,
+    "tag"       => tag,
+    "log_file"  => path_from_base_dir(Droonga::Path.default_log_file),
+    "log_level" => log_level,
+  }
+  Droonga::SafeFileWriter.write(Droonga::Path.config,
+                                YAML.dump(new_configuration))
+end
+
+if options[:reset_catalog]
+  system("droonga-engine-catalog-generate",
+         "--output", Droonga::Path.catalog.to_s,
+         "--hosts",  host,
+         "--port",   port.to_s,
+         "--tag",    tag)
+end
+
+if options[:reset_config] or options[:reset_catalog]
+  service_installation.ensure_correct_file_permission(Droonga::Path.base)
+end
+
+if running
+  successfully_started = service_installation.start
+  unless successfully_started
+    puts("The droonga-engine service is still stopped.")
+    puts("You need to start the service again manually.")
+  end
+end
+
+exit(true)

data/bin/droonga-engine-join

@@ -15,8 +15,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "ostruct"
-require "optparse"
+require "slop"
 require "json"
 require "pathname"
 
@@ -27,144 +26,70 @@ require "droonga/safe_file_writer"
 require "droonga/data_absorber"
 require "droonga/serf"
 
-options = OpenStruct.new
-options.base_dir = ENV[Droonga::Path::BASE_DIR_ENV_NAME] || Dir.pwd
-options.drndump  = "drndump"
-options.client   = "droonga-request"
-options.copy     = true
-options.port     = Droonga::CatalogGenerator::DEFAULT_PORT
-options.tag      = Droonga::CatalogGenerator::DEFAULT_TAG
-
-parser = OptionParser.new
-parser.version = Droonga::Engine::VERSION
-
-parser.on("--base-dir=PATH",
-          "Path to the base directory the catalog.json is located in.",
-          "(#{options.base_dir})") do |path|
-  options.base_dir = path
-end
-
-parser.on("--[no-]copy",
-          "Do or don't copy data from the source cluster.",
-          "(#{options.copy})") do |copy|
-  options.copy = copy
-end
-
-parser.separator("")
-parser.separator("Connections:")
-parser.on("--host=HOST",
-          "Host name of the node to be joined.") do |host|
-  options.joining_host = host
-end
-parser.on("--replica-source-host=HOST",
-          "Host name of the soruce cluster to be connected.") do |host|
-  options.replica_source_host = host
-end
-parser.on("--dataset=DATASET",
-          "Tag dataset name of the cluster to be joined as a node.",
-          "(#{options.dataset})") do |dataset|
-  options.dataset = dataset
-end
-parser.on("--port=PORT", Integer,
-          "Port number of the source cluster to be connected.",
-          "(#{options.port})") do |port|
-  options.port = port
-end
-parser.on("--tag=TAG",
-          "Tag name of the soruce cluster to be connected.",
-          "(#{options.tag})") do |tag|
-  options.tag = tag
-end
-
-parser.parse!(ARGV)
-
-
-base_dir = Pathname(options.base_dir).expand_path
-ENV[Droonga::Path::BASE_DIR_ENV_NAME] = base_dir.to_s
-
-catalog_path = Droonga::Path.catalog
-unless catalog_path.exist?
-  raise "Cannot load 'catalog.json'. You must specify correct path " +
-          "to the base directory via --base-dir option."
-end
-source_catalog = JSON.parse(catalog_path.read)
-
-
-unless options.joining_host
-  raise "You must specify the host name or the IP address of the node " +
-          "to be joined via --host option."
-end
-unless options.replica_source_host
-  raise "You must specify the host name or the IP address of a node " +
-           "of an existing cluster via --replica-source-host option."
-end
-
-
-generator = Droonga::CatalogGenerator.new
-generator.load(source_catalog)
-
-dataset = generator.dataset_for_host(options.replica_source_host)
-if dataset
-  if generator.dataset_for_host(options.joining_host)
-    raise "The joining node is already a member of the cluster. " +
-            "You cannot join a member twice."
+options = nil
+begin
+  options = Slop.parse(:help => true) do |option|
+    option.on("no-copy", "Don't copy data from the source cluster.",
+              :default => false)
+
+    option.separator("Connections:")
+    option.on(:host=,
+              "Host name of the node to be joined.",
+              :required => true)
+    option.on("replica-source-host=",
+              "Host name of the soruce cluster to be connected.",
+              :required => true)
+    option.on(:dataset=,
+              "Tag dataset name of the cluster to be joined as a node.",
+              :default => Droonga::CatalogGenerator::DEFAULT_DATASET)
+    option.on(:port=,
+              "Port number of the source cluster to be connected.",
+              :as => Integer,
+              :default => Droonga::CatalogGenerator::DEFAULT_PORT)
+    option.on(:tag=,
+              "Tag name of the soruce cluster to be connected.",
+              :default => Droonga::CatalogGenerator::DEFAULT_TAG)
   end
-
-  options.dataset = dataset.name
-  options.tag     = dataset.replicas.tag
-  options.port    = dataset.replicas.port
+rescue Slop::MissingOptionError => e
+  $stderr.puts(e)
+  exit(false)
 end
 
-options.joining_node = "#{options.joining_host}:#{options.port}/#{options.tag}"
-options.source_node  = "#{options.replica_source_host}:#{options.port}/#{options.tag}"
+joining_node = "#{options[:host]}:#{options[:port]}/#{options[:tag]}"
+source_node  = "#{options["replica-source-host"]}:#{options[:port]}/#{options[:tag]}"
 
 def run_remote_command(target, command, options)
-  result = Droonga::Serf.send_query(target, command, options)
-  puts result[:result]
-  puts result[:error] unless result[:error].empty?
+  serf = Droonga::Serf.new(nil, target)
+  result = serf.send_query(command, options)
+  puts(result[:result])
+  puts(result[:error]) unless result[:error].empty?
   result[:response]
 end
 
-=begin
-XXX disable fetching until it become working
-puts "Preparing to fetch cluster information..."
-publish_port = 10032 + rand(10000)
-run_remote_command(options.source_node, "publish_catalog",
-                   "node" => options.source_node,
-                   "port" => publish_port)
-sleep(3) # wait until the HTTP server becomes ready
-=end
-
-puts "Joining new replica to the cluster..."
-run_remote_command(options.joining_node, "join",
-                   "node"   => options.joining_node,
-                   "type"   => "replica",
-                   "source" => options.source_node,
-#                   "fetch_port" => publish_port,
-                   "copy"   => options.copy)
+puts("Joining new replica to the cluster...")
+run_remote_command(joining_node, "join",
+                   "node"    => joining_node,
+                   "type"    => "replica",
+                   "source"  => source_node,
+                   "dataset" => options[:dataset],
+                   "copy"    => !options["no-copy"])
 sleep(5) #TODO: wait for restarting of the joining node. this should be done more safely.
 
 while true
   sleep(3)
-  response = run_remote_command(options.joining_node, "report_status",
-                                "node" => options.joining_node,
-                                "key" => "absorbing")
+  response = run_remote_command(joining_node, "report_status",
+                                "node" => joining_node,
+                                "key"  => "absorbing")
   absorbing = response["value"]
   break unless absorbing
 end
 
-puts "Update existing hosts in the cluster..."
-run_remote_command(options.source_node, "add_replicas",
-                   "dataset" => options.dataset,
-                   "hosts"   => [options.joining_host])
+puts("Update existing hosts in the cluster...")
+run_remote_command(source_node, "add_replicas",
+                   "dataset" => options[:dataset],
+                   "hosts"   => [options[:host]])
 
-=begin
-XXX disable fetching until it become working
-run_remote_command(options.source_node, "unpublish_catalog",
-                   "node" => options.source_node,
-                   "port" => publish_port)
-=end
 
-puts "Done."
+puts("Done.")
 
 exit(true)

data/bin/droonga-engine-unjoin

@@ -24,8 +24,12 @@ require "droonga/engine/version"
 require "droonga/path"
 require "droonga/catalog_generator"
 require "droonga/safe_file_writer"
+require "droonga/service_installation"
 require "droonga/serf"
 
+service_installation = Droonga::ServiceInstallation.new
+service_installation.ensure_using_service_base_directory
+
 options = OpenStruct.new
 options.base_dir = ENV[Droonga::Path::BASE_DIR_ENV_NAME] || Dir.pwd
 
@@ -48,6 +52,7 @@ parser.parse!(ARGV)
 base_dir = Pathname(options.base_dir).expand_path
 ENV[Droonga::Path::BASE_DIR_ENV_NAME] = base_dir.to_s
 
+
 catalog_path = Droonga::Path.catalog
 unless catalog_path.exist?
   raise "Cannot load 'catalog.json'. You must specify correct path " +
@@ -59,6 +64,13 @@ unless options.replica_remove_host
           "be removed from the cluster via --replica-remove-host option."
 end
 
+unless service_installation.have_read_permission?
+  puts("You have no permission to read files under " +
+         "<#{Droonga::Path.base.to_s}>.")
+  puts("Try again with right permission.")
+  exit(false)
+end
+
 source_catalog = JSON.parse(catalog_path.read)
 generator = Droonga::CatalogGenerator.new
 generator.load(source_catalog)
@@ -80,7 +92,8 @@ options.remaining_node = "#{remaining_host}:#{options.port}/#{options.tag}"
 
 
 def run_remote_command(target, command, options)
-  result = Droonga::Serf.send_query(target, command, options)
+  serf = Droonga::Serf.new(nil, target)
+  result = serf.send_query(command, options)
   puts result[:result]
   puts result[:error] unless result[:error].empty?
 end

data/doc/text/news.md

@@ -1,5 +1,26 @@
 # News
 
+## 1.0.6: 2014-09-29
+
+ * The installation script is now available.
+   It automatically installs required softwares and configure the `droonga-engine` as a system service.
+   Currently it works only for Debian, Ubuntu, and CentOS 7.
+ * The service works as a process belonging to a user `droonga-engine` who is specific for the service.
+   The configuration directory for the service is placed under the home directory of the user.
+ * A static configuration file to define default parameters (`host` and so on) is now available.
+   It must be placed into the directory same to `catalog.json`.
+   You don't have to run `droonga-engine` command with many options, anymore.
+ * `droonga-engine-join` now automatically fetches `catalog.json` from the specified source replica node.
+   Now you don't have to copy `catalog.json` from another node before you run `droonga-engine-join` anymore.
+ * A new `catalog` plugin is introduced as one of default plugins, to fetch `catalog.json` from existing cluster.
+   The list of plugins in your `catalog.json` must include it.
+ * A new command line utility `droonga-engine-configure` is available.
+   It generates the static configuration file, the `catalog.json` for the service.
+   Moreover, it clears old stored data to make the node empty.
+ * Some options for utility commands become optional.
+   Important parameters are automatically detected.
+ * Restarts server processes more gracefully.
+
 ## 1.0.5: 2014-07-29
 
  * Restarts server processes more gracefully.

data/droonga-engine.gemspec

@@ -35,22 +35,24 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_dependency "rroonga", ">= 4.0.1"
-  gem.add_dependency "groonga-command-parser"
-  gem.add_dependency "json"
+  gem.add_dependency "archive-zip"
   gem.add_dependency "cool.io"
-  gem.add_dependency "droonga-message-pack-packer", ">= 1.0.1"
+  gem.add_dependency "drndump"
+  gem.add_dependency "droonga-client", ">= 0.1.9"
+  gem.add_dependency "droonga-message-pack-packer", ">= 1.0.2"
+  gem.add_dependency "groonga-command-parser"
   gem.add_dependency "faraday"
   gem.add_dependency "faraday_middleware"
-  gem.add_dependency "archive-zip"
+  gem.add_dependency "json"
+  gem.add_dependency "rroonga", ">= 4.0.4"
   gem.add_dependency "sigdump"
-  gem.add_dependency "droonga-client", ">= 0.1.9"
-  gem.add_dependency "drndump"
-  gem.add_development_dependency "rake"
+  gem.add_dependency "slop"
+  gem.add_dependency "sys-proctable"
+  gem.add_development_dependency "kramdown"
   gem.add_development_dependency "bundler"
+  gem.add_development_dependency "packnga"
+  gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
   gem.add_development_dependency "test-unit-notify"
   gem.add_development_dependency "test-unit-rr"
-  gem.add_development_dependency "packnga"
-  gem.add_development_dependency "kramdown"
 end