droonga-engine 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d592eda2e4916fb9d0d4fbf48ad538cbff3c1d34
-  data.tar.gz: 62255e98b9141709a42db836e525737a8e254343
+  metadata.gz: 333168fcdecd92b88dfc590746445e0ed31319c4
+  data.tar.gz: bb082dd818cfd171a131e0e45dc75269a1cd814d
 SHA512:
-  metadata.gz: 163a844fae7ba67af1dc934380a374756439744e844de81e53fe9dc141f6a7b1ad457db25a0e7a1fda988d6709aab34e55b9ecd0efc3dabb6c6be508bf6fb466
-  data.tar.gz: 183c9afe679f550c198e65f9f7f5dfab74b10ded4e84bfd40d1b93239227c571063e05920aa6e514eaa11a46a35cdad8f320265f60534b6944071c2132e79e94
+  metadata.gz: 38167299101e872c86cb07ac35449f023028ffaaa405477423c7882fd981c90677f4483671ea3c326e71338274326acbbfab644f668c77b7602d932b3c19717e
+  data.tar.gz: 41fe28134710faf85c694abe7c702e120d33188f321611e8627788b80ceabc5cca295f7d590621a85d1a87b936746a7d90e63cfeb938eb932287110c5548cfe1

data/Gemfile

@@ -71,3 +71,10 @@ if File.exist?(grn2drn_dir)
 else
   gem "grn2drn", :github => "droonga/grn2drn"
 end
+
+drndump_dir = File.join(parent_dir, "drndump")
+if File.exist?(drndump_dir)
+  gem "drndump", :path => drndump_dir
+else
+  gem "drndump", :github => "droonga/drndump"
+end

data/bin/droonga-engine-absorb-data

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "ostruct"
+require "optparse"
+require "open3"
+
+require "droonga/engine/version"
+require "droonga/catalog_generator"
+require "droonga/serf"
+
+options = OpenStruct.new
+options.port    = Droonga::CatalogGenerator::DEFAULT_PORT
+options.tag     = Droonga::CatalogGenerator::DEFAULT_TAG
+options.dataset = Droonga::CatalogGenerator::DEFAULT_DATASET
+parser = OptionParser.new
+parser.version = Droonga::Engine::VERSION
+
+parser.separator("")
+parser.separator("Connection:")
+parser.on("--source-host=HOST",
+          "Host name of the source cluster to be connected.") do |host|
+  options.source_host = host
+end
+parser.on("--destination-host=HOST",
+          "Host name of this cluster to be connected.") do |host|
+  options.destination_host = host
+end
+parser.on("--port=PORT", Integer,
+          "Port number of the source cluster to be connected.",
+          "(#{options.port})") do |port|
+  options.port = port
+end
+
+parser.separator("")
+parser.separator("Data:")
+parser.on("--tag=TAG",
+          "Tag name to be used to communicate with Droonga system.",
+          "(#{options.tag})") do |tag|
+  options.tag = tag
+end
+parser.on("--dataset=DATASET",
+          "Dataset to be absorbed.",
+          "(#{options.dataset})") do |dataset|
+  options.dataset = dataset
+end
+
+parser.parse!(ARGV)
+
+unless options.source_host
+  raise "You must specify the source host via --source-host option."
+end
+unless options.destination_host
+  raise "You must specify the destination host via --destination-host option."
+end
+
+destination_node = "#{options.destination_host}:#{options.port}/#{options.tag}"
+
+puts "Absorbing data..."
+Droonga::Serf.send_query(destination_node, "absorb_data",
+                         "node"    => destination_node,
+                         "source"  => options.source_host,
+                         "port"    => options.port,
+                         "tag"     => options.tag,
+                         "dataset" => options.dataset)
+puts "Done."
+
+exit(true)

data/bin/droonga-engine-catalog-generate

@@ -18,11 +18,11 @@
 require "ostruct"
 require "optparse"
 require "json"
-require "tempfile"
 require "pathname"
 
 require "droonga/engine/version"
 require "droonga/catalog_generator"
+require "droonga/safe_file_writer"
 
 generator = Droonga::CatalogGenerator.new
 current_dataset = {}
@@ -42,31 +42,38 @@ parser.on("--output=PATH",
 end
 parser.on("--dataset=NAME",
           "Add a dataset its name is NAME.",
-          "And set the NAME to the current dataset.") do |name|
+          "And set the NAME to the current dataset.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_DATASET})") do |name|
   current_dataset = datasets[name] = {}
 end
 parser.on("--n-workers=N", Integer,
-          "Use N workers for the current dataset.") do |n|
+          "Use N workers for the current dataset.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_N_WORKERS})") do |n|
   current_dataset[:n_workers] = n
 end
 parser.on("--hosts=NAME1,NAME2,...", Array,
-          "Use given hosts for replicas of the current dataset.") do |hosts|
+          "Use given hosts for replicas of the current dataset.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_HOSTS.join(",")})") do |hosts|
   current_dataset[:hosts] = hosts
 end
 parser.on("--port=PORT", Integer,
-          "Use the PORT as the port for the current dataset.") do |port|
+          "Use the PORT as the port for the current dataset.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_PORT})") do |port|
   current_dataset[:port] = port
 end
 parser.on("--tag=TAG",
-          "Use the TAG as the tag for the current dataset.") do |tag|
+          "Use the TAG as the tag for the current dataset.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_TAG})") do |tag|
   current_dataset[:tag] = tag
 end
 parser.on("--n-slices=N", Integer,
-          "Use N slices for each replica.") do |n|
+          "Use N slices for each replica.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_N_SLICES})") do |n|
   current_dataset[:n_slices] = n
 end
 parser.on("--plugins=PLUGIN1,PLUGIN2,...", Array,
-          "Use PLUGINS for the current dataset.") do |plugins|
+          "Use PLUGINS for the current dataset.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_PLUGINS.join(",")})") do |plugins|
   current_dataset[:plugins] = plugins
 end
 parser.on("--schema=PATH",
@@ -99,12 +106,8 @@ def open_output(path)
   if path == "-"
     yield($stdout)
   else
-    # Don't output the file directly to prevent loading of incomplete file!
-    path = Pathname(path).expand_path
-    Tempfile.open(path.basename.to_s, path.parent.to_s, "w") do |output|
+    Droonga::SafeFileWriter.write(path) do |output|
       yield(output)
-      output.flush
-      File.rename(output.path, path.to_s)
     end
   end
 end

data/bin/droonga-engine-catalog-modify

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "ostruct"
+require "optparse"
+require "json"
+require "pathname"
+
+require "droonga/engine/version"
+require "droonga/catalog_generator"
+require "droonga/safe_file_writer"
+
+generator = Droonga::CatalogGenerator.new
+current_dataset = {}
+datasets = {
+  Droonga::CatalogGenerator::DEFAULT_DATASET => current_dataset
+}
+
+options = OpenStruct.new
+options.source_path = "./catalog.json"
+options.output_path = "-"
+options.update      = false
+parser = OptionParser.new
+parser.version = Droonga::Engine::VERSION
+parser.on("--source=PATH",
+          "Path to an existing catalog.json.",
+          "\"-\" means the standard input.",
+          "(#{options.source_path})") do |path|
+  options.source_path = path
+end
+parser.on("--output=PATH",
+          "Output catalog.json to PATH.",
+          "\"-\" means the standard output.",
+          "(#{options.output_path})") do |path|
+  options.output_path = path
+end
+parser.on("--[no-]update",
+          "Update the source file itself, or not.",
+          "(#{options.update})") do |update|
+  options.update = update
+end
+parser.on("--dataset=NAME",
+          "Add a dataset its name is NAME.",
+          "And set the NAME to the current dataset.",
+          "(#{Droonga::CatalogGenerator::DEFAULT_DATASET})") do |name|
+  current_dataset = datasets[name] = {}
+end
+parser.on("--replica-hosts=NAME1,NAME2,...", Array,
+          "Use given hosts as replicas for the current dataset.") do |hosts|
+  current_dataset[:replica_hosts] = hosts
+end
+parser.on("--add-replica-hosts=NAME1,NAME2,...", Array,
+          "Use given hosts to be added as replicas to the current dataset.") do |hosts|
+  current_dataset[:add_replica_hosts] = hosts
+end
+parser.on("--remove-replica-hosts=NAME1,NAME2,...", Array,
+          "Use given hosts to be removed as replicas from the current dataset.") do |hosts|
+  current_dataset[:remove_replica_hosts] = hosts
+end
+parser.parse!(ARGV)
+
+if options.source_path != "-" and options.update
+  options.output_path = options.source_path
+end
+
+def load_source(path)
+  source = nil
+  if path == "-"
+    source = $stdin.read
+  else
+    source_path = Pathname(path).expand_path
+    source = source_path.read
+  end
+  JSON.parse(source)
+end
+
+source_catalog = load_source(options.source_path)
+generator.load(source_catalog)
+generator.modify(datasets)
+
+def open_output(path)
+  if path == "-"
+    yield($stdout)
+  else
+    Droonga::SafeFileWriter.write(path) do |output|
+      yield(output)
+    end
+  end
+end
+
+catalog = generator.generate
+open_output(options.output_path) do |output|
+  output.puts(JSON.pretty_generate(catalog))
+end

data/bin/droonga-engine-join

@@ -0,0 +1,115 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "ostruct"
+require "optparse"
+require "json"
+require "pathname"
+
+require "droonga/engine/version"
+require "droonga/path"
+require "droonga/catalog_generator"
+require "droonga/safe_file_writer"
+require "droonga/data_absorber"
+require "droonga/serf"
+
+options = OpenStruct.new
+options.base_dir = ENV[Droonga::Path::BASE_DIR_ENV_NAME] || Dir.pwd
+options.drndump  = "drndump"
+options.client   = "droonga-request"
+options.copy     = true
+
+parser = OptionParser.new
+parser.version = Droonga::Engine::VERSION
+
+parser.on("--base-dir=PATH",
+          "Path to the base directory the catalog.json is located in.",
+          "(#{options.base_dir})") do |path|
+  options.base_dir = path
+end
+
+parser.on("--[no-]copy",
+          "Do or don't copy data from the source cluster.",
+          "(#{options.copy})") do |copy|
+  options.copy = copy
+end
+
+parser.separator("")
+parser.separator("Connections:")
+parser.on("--host=HOST",
+          "Host name of the node to be joined.") do |host|
+  options.joining_host = host
+end
+parser.on("--replica-source-host=HOST",
+          "Host name of the soruce cluster to be connected.") do |host|
+  options.replica_source_host = host
+end
+
+parser.parse!(ARGV)
+
+
+base_dir = Pathname(options.base_dir).expand_path
+ENV[Droonga::Path::BASE_DIR_ENV_NAME] = base_dir.to_s
+
+catalog_path = Droonga::Path.catalog
+unless catalog_path.exist?
+  raise "Cannot load 'catalog.json'. You must specify correct path " +
+          "to the base directory via --base-dir option."
+end
+source_catalog = JSON.parse(catalog_path.read)
+
+
+unless options.joining_host
+  raise "You must specify the host name or the IP address of the node " +
+          "to be joined via --host option."
+end
+unless options.replica_source_host
+  raise "You must specify the host name or the IP address of a node " +
+           "of an existing cluster via --replica-source-host option."
+end
+
+
+generator = Droonga::CatalogGenerator.new
+generator.load(source_catalog)
+
+dataset = generator.dataset_for_host(options.replica_source_host)
+unless dataset
+  raise "Specified source host #{options.replica_source_host} is not a " +
+          "member of the cluster. You must specify correct host via " +
+          "--replica-source-host option."
+end
+
+if generator.dataset_for_host(options.joining_host)
+  raise "The joining node is already a member of the cluster. " +
+          "You cannot join a member twice."
+end
+
+options.tag  = dataset.replicas.tag
+options.port = dataset.replicas.port
+
+options.joining_node = "#{options.joining_host}:#{options.port}/#{options.tag}"
+
+puts "Joining new replica to the cluster..."
+Droonga::Serf.send_query(options.joining_node, "join",
+                         "node"   => options.joining_node,
+                         "type"   => "replica",
+                         "source" => options.replica_source_host,
+                         "copy"   => options.copy)
+
+puts "Done."
+
+exit(true)

data/bin/droonga-engine-unjoin

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "ostruct"
+require "optparse"
+require "json"
+require "pathname"
+
+require "droonga/engine/version"
+require "droonga/path"
+require "droonga/catalog_generator"
+require "droonga/safe_file_writer"
+require "droonga/serf"
+
+options = OpenStruct.new
+options.base_dir = ENV[Droonga::Path::BASE_DIR_ENV_NAME] || Dir.pwd
+
+parser = OptionParser.new
+parser.version = Droonga::Engine::VERSION
+
+parser.on("--base-dir=PATH",
+          "Path to the base directory the catalog.json is located in.",
+          "(#{options.base_dir})") do |path|
+  options.base_dir = path
+end
+parser.on("--host=HOST",
+          "Host name of the replica removed from cluster.") do |host|
+  options.replica_remove_host = host
+end
+
+parser.parse!(ARGV)
+
+
+base_dir = Pathname(options.base_dir).expand_path
+ENV[Droonga::Path::BASE_DIR_ENV_NAME] = base_dir.to_s
+
+catalog_path = Droonga::Path.catalog
+unless catalog_path.exist?
+  raise "Cannot load 'catalog.json'. You must specify correct path " +
+          "to the base directory via --base-dir option."
+end
+
+unless options.replica_remove_host
+  raise "You must specify the host name or the IP address of a node to " +
+          "be removed from the cluster via --replica-remove-host option."
+end
+
+source_catalog = JSON.parse(catalog_path.read)
+generator = Droonga::CatalogGenerator.new
+generator.load(source_catalog)
+
+dataset = generator.dataset_for_host(options.replica_remove_host)
+unless dataset
+  raise "Specified host #{options.replica_remove_host} is not a member of "+
+          "the cluster. You must specify correct host via --replica-remove-host " +
+          "option."
+end
+
+options.dataset     = dataset.name
+options.tag         = dataset.replicas.tag
+options.port        = dataset.replicas.port
+options.other_hosts = dataset.replicas.hosts
+
+remaining_host = options.other_hosts.first || options.replica_remove_host
+options.remaining_node = "#{remaining_host}:#{options.port}/#{options.tag}"
+
+
+puts "Unjoining replica from the cluster..."
+
+Droonga::Serf.send_query(options.remaining_node, "remove_replicas",
+                         "dataset" => options.dataset,
+                         "hosts"   => [options.replica_remove_host])
+
+puts "Done."
+
+exit(true)

data/doc/text/news.md

@@ -1,5 +1,13 @@
 # News
 
+## 1.0.4: 2014-06-29
+
+ * New command (and plugin) [`status`](http://droonga.org/reference/1.0.4/commands/status/) is now available.
+ * New command line tools are available.
+   * `droonga-engine-join` and `droonga-engine-unjoin` help you to modify cluster composition. See [the tutorial to add/remove replica](http://droonga.org/tutorial/1.0.4/add-replica/).
+   * `droonga-engine-absorb-data` helps you to duplicate clusters. See [the tutorial for dump/restore](http://droonga.org/tutorial/1.0.4/dump-restore/).
+   * `droonga-engine-catalog-modify` helps you to modify existing `catalog.json`.
+
 ## 1.0.3: 2014-05-29
 
  * Alive monitoring (based on [Serf](http://serfdom.io/)) lands.

data/droonga-engine.gemspec

@@ -41,10 +41,11 @@ Gem::Specification.new do |gem|
   gem.add_dependency "cool.io"
   gem.add_dependency "serverengine"
   gem.add_dependency "droonga-message-pack-packer", ">= 1.0.1"
-  gem.add_dependency "listen", "~> 2.7"
   gem.add_dependency "faraday"
   gem.add_dependency "faraday_middleware"
   gem.add_dependency "archive-zip"
+  gem.add_dependency "sigdump"
+  gem.add_dependency "drndump"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "test-unit"

data/lib/droonga/buffered_tcp_socket.rb

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "cool.io"
+
+require "droonga/loggable"
+
+module Droonga
+  class BufferedTCPSocket < Coolio::TCPSocket
+    include Loggable
+
+    def initialize(socket, data_directory)
+      super(socket)
+      @data_directory = data_directory
+      @_write_buffer = []
+    end
+
+    def on_connect
+      logger.trace("connected to #{@remote_host}:#{@remote_port}")
+    end
+
+    def write(data)
+      reserve_write(data)
+      schedule_write
+      data.bytesize
+    end
+
+    def reserve_write(data)
+      chunk = Chunk.new(@data_directory, data, Time.now, 0)
+      chunk.buffering
+      @_write_buffer << chunk
+      data.bytesize
+    end
+
+    def on_writable
+      begin
+        chunk = @_write_buffer.shift
+        written_size = @_io.write(chunk.data)
+        if written_size == chunk.data.bytesize
+          chunk.written
+        else
+          chunk.written_partial(written_size)
+          @_write_buffer.unshift(chunk)
+        end
+      rescue Errno::EINTR
+        return
+      rescue SystemCallError, IOError, SocketError
+        return close
+      end
+
+      if @_write_buffer.empty?
+        disable_write_watcher
+        on_write_complete
+      end
+    end
+
+    def resume
+      @_write_buffer = (load_chunks + @_write_buffer).sort_by do |chunk|
+        chunk.time_stamp
+      end
+    end
+
+    private
+    def load_chunks
+      FileUtils.mkdir_p(@data_directory.to_s)
+      Pathname.glob("#{@data_directory}/*.chunk").collect do |chunk_path|
+        Chunk.load(chunk_path)
+      end
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] buffered-tcp-socket"
+    end
+
+    class Chunk
+      class << self
+        def load(path)
+          data_directory = path.dirname
+          time_stamp1, time_stamp2, revision, = path.basename.to_s.split(".", 4)
+          data = path.open("rb") {|file| file.read}
+          time_stamp = Time.iso8601("#{time_stamp1}.#{time_stamp2}")
+          revision = Integer(revision)
+          new(data_directory, data, time_stamp, revision)
+        end
+      end
+
+      attr_reader :data, :time_stamp
+      def initialize(data_directory, data, time_stamp, revision)
+        @data_directory = data_directory
+        @data = data
+        @time_stamp = time_stamp.utc
+        @revision = revision
+      end
+
+      def buffering
+        path.open("wb") do |file|
+          file.write(@data)
+        end
+      end
+
+      def written
+        FileUtils.rm_f(path.to_s)
+      end
+
+      def written_partial(size)
+        written
+        @data = @data[size..-1]
+        @revision += 1
+        buffering
+      end
+
+      private
+      def path
+        @data_directory + "#{@time_stamp.iso8601(6)}.#{@revision}.chunk"
+      end
+    end
+  end
+end