droonga-engine 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd5aad47933f58aee93072477c535d458556b183
-  data.tar.gz: fc6e82289c99cc3ef4cdc269d0153ca6588dec7f
+  metadata.gz: 695a658bbd48a05e64b9e53f2764549212c63786
+  data.tar.gz: 799ed768d48b6e49150fa63ca4ccc7f956837495
 SHA512:
-  metadata.gz: 08909575ab516f97f404396904a53a0920bdc7a6ad7ab8673e7065d681698dee2f7029c71cb74731824892df0eeb87b8de4383d4aa9d0107823e7a00f363a7dc
-  data.tar.gz: fc2acfa0c8c133bc70ebad99f7d326e96fcc316fdedfb45347431e5ec3a96a751d4f00db03b9c21c4c44d03d26b8da5b0263615926e9321d50f44c60277f0cf9
+  metadata.gz: a4141f7a2c89b0cc5558ea24fe11a52c6aa4ea93bf323065866befabc32ca2b9060f72dc3b5a4a5c9ef5e8204027d2e67df0ffbea5612011a414caee7d7a2a07
+  data.tar.gz: 50a6459b3f7db24bdc112536f88aac95c6b6e1c69423293135fe03d06a3cde96a686b54ffb112517bfc6aa6e2a9979d36f66999c0cfa4c78c2c71fbb466d52e0

data/.travis.yml

@@ -4,7 +4,8 @@ notifications:
 env:
   - DEFAULT_TEST_TASK=test:unit
   - DEFAULT_TEST_TASK=test:command:default
-  - DEFAULT_TEST_TASK=test:command:version1
+  - DEFAULT_TEST_TASK=test:command:single_slice
+#  - DEFAULT_TEST_TASK=test:command:version1
 rvm:
   - 1.9.3
   - 2.0.0

data/Rakefile

@@ -50,6 +50,11 @@ namespace :test do
       run_command_test
     end
 
+    desc "Run command test: single slice"
+    task :single_slice do
+      run_command_test("--config", "single_slice")
+    end
+
     desc "Run command test: version1"
     task :version1 do
       run_command_test("--config", "version1")
@@ -61,6 +66,7 @@ desc "Run test"
 task :test => [
   "test:unit",
   "test:command:default",
+  "test:command:single_slice",
   "test:command:version1",
 ]
 

data/bin/droonga-engine-absorb-data

@@ -83,46 +83,46 @@ module Droonga
       parser.version = Engine::VERSION
 
       parser.separator("")
-      parser.separator("Destination node:")
+      parser.separator("Destination:")
       parser.on("--host=HOST",
-                "Host name of the destination node.") do |host|
+                "Host name of the destination engine node to copy data.") do |host|
         options.host = host
       end
       parser.on("--port=PORT", Integer,
-                "Port number of the destination node.",
+                "Port number to communicate with the destination engine node.",
                 "(#{options.port})") do |port|
         options.port = port
       end
       parser.on("--tag=TAG", Integer,
-                "Tag name of the destination node.",
+                "Tag name to communicate with the destination engine node.",
                 "(#{options.tag})") do |tag|
         options.tag = tag
       end
       parser.on("--dataset=DATASET",
-                "Name of the destination dataset.",
+                "Name of the destination dataset for copying data.",
                 "(#{options.dataset})") do |dataset|
         options.dataset = dataset
       end
 
       parser.separator("")
-      parser.separator("Source node:")
+      parser.separator("Source:")
       parser.on("--source-host=HOST",
-                "Host name of the source node.",
+                "Host name of the soruce engine node to copy data.",
                 "(#{options.source_host})") do |host|
         options.source_host = host
       end
       parser.on("--source-port=PORT", Integer,
-                "Port number of the source node.",
+                "Port number to communicate with the soruce engine node.",
                 "(#{options.source_port})") do |host|
         options.source_host = host
       end
       parser.on("--source-tag=TAG",
-                "Tag name of the source node.",
+                "Tag name to communicate with the soruce engine node.",
                 "(#{options.source_tag})") do |tag|
         options.source_tag = tag
       end
       parser.on("--source-dataset=DATASET",
-                "Name of the source dataset.",
+                "Name of the soruce dataset for copying data.",
                 "(#{options.source_dataset})") do |dataset|
         options.source_dataset = dataset
       end
@@ -130,7 +130,7 @@ module Droonga
       parser.separator("")
       parser.separator("Connection:")
       parser.on("--receiver-host=HOST",
-                "Host name of this computer.",
+                "Host name of the computer you are running this command.",
                 "(#{options.receiver_host})") do |host|
         options.receiver_host = host
       end
@@ -138,18 +138,18 @@ module Droonga
       parser.separator("")
       parser.separator("Miscellaneous:")
       parser.on("--records-per-second=N", Integer,
-                "Maximum number of records per second to be absorbed.",
+                "Maximum number of records to be copied per one second.",
                 "'#{Client::RateLimiter::NO_LIMIT}' means no limit.",
                 "(#{options.messages_per_second})") do |n|
         options.messages_per_second = n
       end
       parser.on("--progress-interval-seconds=N", Integer,
-                "Interval seconds to report progress.",
+                "Interval seconds to report progress of data copying.",
                 "(#{options.progress_interval_seconds})") do |n|
         options.progress_interval_seconds = n
       end
       parser.on("--[no-]verbose",
-                "Output details for internal operations.",
+                "Output details for internal operations or not.",
                 "(#{options.verbose})") do |verbose|
         options.verbose = verbose
       end

data/bin/droonga-engine-catalog-generate

@@ -43,59 +43,71 @@ end
 parser = OptionParser.new
 parser.version = Droonga::Engine::VERSION
 parser.on("--output=PATH",
-          "Output catalog.json to PATH.",
+          "The output path of generated catalog.json to be saved as.",
           "\"-\" means the standard output.",
+          "Any existing file at the specified path will be overwritten without confirmation.",
           "(#{options.output_path})") do |path|
   options.output_path = path
 end
 parser.on("--dataset=NAME",
-          "Add a dataset its name is NAME.",
-          "And set the NAME to the current dataset.",
+          "The name of a new dataset.",
+          "This can be specified multiple times to define multiple datasets.",
           "(#{Droonga::Catalog::Generator::DEFAULT_DATASET})") do |name|
   current_dataset = datasets[name] = {}
 end
 parser.on("--n-workers=N", Integer,
-          "Use N workers for the current dataset.",
+          "Number of workers for each volume in the dataset ",
+          "specified by the preceding --dataset option.",
           "(#{Droonga::Catalog::Generator::DEFAULT_N_WORKERS})") do |n|
   current_dataset[:n_workers] = n
 end
 parser.on("--hosts=NAME1,NAME2,...", Array,
-          "Use given hosts for replicas of the current dataset.",
+          "Host names of engine nodes to be used as replicas in the dataset ",
+          "specified by the preceding --dataset option.",
           "(#{Droonga::Catalog::Generator::DEFAULT_HOSTS.join(",")})") do |hosts|
   current_dataset[:hosts] = hosts
 end
 parser.on("--port=PORT", Integer,
-          "Use the PORT as the port for the current dataset.",
+          "Port number to communicate with engine nodes in the dataset ",
+          "specified by the preceding --dataset option.",
           "(#{Droonga::Catalog::Generator::DEFAULT_PORT})") do |port|
   current_dataset[:port] = port
 end
 parser.on("--tag=TAG",
-          "Use the TAG as the tag for the current dataset.",
+          "Tag name to communicate with engine nodes in the dataset ",
+          "specified by the preceding --dataset option.",
           "(#{Droonga::Catalog::Generator::DEFAULT_TAG})") do |tag|
   current_dataset[:tag] = tag
 end
 parser.on("--n-slices=N", Integer,
-          "Use N slices for each replica.",
+          "Number of slices for each replica in the dataset ",
+          "specified by the preceding --dataset option.",
           "(#{Droonga::Catalog::Generator::DEFAULT_N_SLICES})") do |n|
   current_dataset[:n_slices] = n
 end
 parser.on("--plugins=PLUGIN1,PLUGIN2,...", Array,
-          "Use PLUGINS for the current dataset.",
+          "Plugin names activated for the dataset ",
+          "specified by the preceding --dataset option.",
           "(#{Droonga::Catalog::Generator::DEFAULT_PLUGINS.join(",")})") do |plugins|
   current_dataset[:plugins] = plugins
 end
 parser.on("--schema=PATH",
-          "Use schema in JSON at PATH for the current dataset.") do |path|
+          "The path to a JSON file including schema definition for the dataset ",
+          "specified by the preceding --dataset option.") do |path|
   File.open(path) do |input|
     current_dataset[:schema] = JSON.parse(input.read)
   end
 end
 parser.on("--fact=TABLE",
-          "Use TABLE as the fact table for the current dataset.") do |table|
+          "Name of the fact table in the dataset ",
+          "specified by the preceding --dataset option.") do |table|
   current_dataset[:fact] = table
 end
 parser.on("--replicas=PATH",
-          "Use replicas in JSON at PATH for the current dataset.") do |path|
+          "The path to a JSON file including replicas definition for the dataset ",
+          "specified by the preceding --dataset option.",
+          "If this option is used, other options to define replicas in the dataset ",
+          "(--hosts, --port, --tag and --n-slices) are ignored.") do |path|
   File.open(path) do |input|
     current_dataset[:replicas] = JSON.parse(input.read)
   end

data/bin/droonga-engine-catalog-modify

@@ -46,14 +46,15 @@ end
 parser = OptionParser.new
 parser.version = Droonga::Engine::VERSION
 parser.on("--source=PATH",
-          "Path to an existing catalog.json.",
+          "The path to the catalog.json to be modified.",
           "\"-\" means the standard input.",
           "(#{options.source_path})") do |path|
   options.source_path = path
 end
 parser.on("--output=PATH",
-          "Output catalog.json to PATH.",
+          "The output path of modified catalog.json to be saved as.",
           "\"-\" means the standard output.",
+          "Any existing file at the specified path will be overwritten without confirmation.",
           "(#{options.output_path})") do |path|
   options.output_path = path
 end
@@ -63,21 +64,26 @@ parser.on("--[no-]update",
   options.update = update
 end
 parser.on("--dataset=NAME",
-          "Add a dataset its name is NAME.",
-          "And set the NAME to the current dataset.",
+          "The name of an existing dataset to be modified.",
+          "This can be specified multiple times to modify multiple datasets.",
           "(#{Droonga::Catalog::Generator::DEFAULT_DATASET})") do |name|
   current_dataset = datasets[name] = {}
 end
 parser.on("--replica-hosts=NAME1,NAME2,...", Array,
-          "Use given hosts as replicas for the current dataset.") do |hosts|
+          "Host names of engine nodes to be used as replicas in the dataset ",
+          "specified by the preceding --dataset option.",
+          "If you specify this option, all existing replica nodes ",
+          "defined in the dataset are replaced.") do |hosts|
   current_dataset[:replica_hosts] = hosts
 end
 parser.on("--add-replica-hosts=NAME1,NAME2,...", Array,
-          "Use given hosts to be added as replicas to the current dataset.") do |hosts|
+          "Host names of engine nodes to be added to the cluster as replicas, ",
+          "in the dataset specified by the preceding --dataset option.") do |hosts|
   current_dataset[:add_replica_hosts] = hosts
 end
 parser.on("--remove-replica-hosts=NAME1,NAME2,...", Array,
-          "Use given hosts to be removed as replicas from the current dataset.") do |hosts|
+          "Host names of engine nodes to be removed from the cluster, ",
+          "in the dataset specified by the preceding --dataset option.") do |hosts|
   current_dataset[:remove_replica_hosts] = hosts
 end
 parser.parse!(ARGV)

data/bin/droonga-engine-join

@@ -92,7 +92,7 @@ module Droonga
     private
     def parse_options
       options = Slop.parse(:help => true) do |option|
-        option.on("no-copy", "Don't copy data from the source cluster.",
+        option.on("no-copy", "Don't copy data from the source node.",
                   :default => false)
 
         option.separator("Target:")
@@ -100,33 +100,33 @@ module Droonga
                   "Host name of the new node to be joined.",
                   :required => true)
         option.on("replica-source-host=",
-                  "Host name of the soruce node in the cluster to be connected.",
+                  "Host name of the soruce node in the cluster to join.",
                   :required => true)
 
         option.on(:port=,
-                  "Port number of the source cluster to be connected.",
+                  "Port number to communicate with engine nodes.",
                   :as => Integer,
                   :default => NodeName::DEFAULT_PORT)
         option.on(:tag=,
-                  "Tag name of the soruce cluster to be connected.",
+                  "Tag name to communicate with engine nodes.",
                   :default => NodeName::DEFAULT_TAG)
         option.on(:dataset=,
-                  "Dataset name of for the node to be joined.",
+                  "Dataset name the node is going to join as a replica in.",
                   :default => Catalog::Dataset::DEFAULT_NAME)
 
         option.separator("Connections:")
         option.on("receiver-host=",
-                  "Host name of this host.",
+                  "Host name of the computer you are running this command.",
                   :default => Socket.gethostname)
 
         option.separator("Miscellaneous:")
         option.on("records-per-second=",
-                  "Maximum number of records per second to be copied. " +
+                  "Maximum number of records to be copied per one second. " +
                     "'#{Client::RateLimiter::NO_LIMIT}' means no limit.",
                   :as => Integer,
                   :default => DataAbsorberClient::DEFAULT_MESSAGES_PER_SECOND)
         option.on("progress-interval-seconds=",
-                  "Interval seconds to report progress.",
+                  "Interval seconds to report progress of data copying.",
                   :as => Integer,
                   :default => DataAbsorberClient::DEFAULT_PROGRESS_INTERVAL_SECONDS)
         option.on(:verbose, "Output details for internal operations.",

data/bin/droonga-engine-set-role

@@ -23,7 +23,7 @@ module Droonga
       def run
         parse_options do |option|
           option.on(:role=,
-                    "New role for the target node.",
+                    "New role for the engine node.",
                     :required => true)
         end
 

data/bin/droonga-engine-unjoin

@@ -31,10 +31,10 @@ module Droonga
       def run
         parse_options do |option|
           option.on("receiver-host=",
-                    "Host name of this host.",
+                    "Host name of the computer you are running this command.",
                     :default => Socket.gethostname)
           option.on(:dataset=,
-                    "Dataset name of for the node to be unjoined.",
+                    "Dataset name the node is going to be removed from.",
                     :default => Catalog::Dataset::DEFAULT_NAME)
         end
 

data/lib/droonga/address.rb

@@ -68,6 +68,9 @@ module Droonga
     end
 
     def ==(other)
+      if other.is_a?(String)
+        return to_s == other
+      end
       other.is_a?(self.class) and to_a == other.to_a
     end
   end

data/lib/droonga/cluster.rb

@@ -17,6 +17,7 @@ require "droonga/loggable"
 require "droonga/changable"
 require "droonga/path"
 require "droonga/file_observer"
+require "droonga/address"
 require "droonga/engine_node"
 require "droonga/differ"
 
@@ -63,12 +64,15 @@ module Droonga
 
     attr_accessor :catalog
 
-    def initialize(loop, params)
-      @loop = loop
+    def initialize(params)
+      @loop = params[:loop]
 
       @params = params
       @catalog = params[:catalog]
-      @state = nil
+      @state = params[:state] || {}
+
+      @engine_nodes = nil
+      @on_change = nil
 
       reload
     end
@@ -158,7 +162,7 @@ module Droonga
 
     def forward(message, destination)
       receiver = destination["to"]
-      receiver_node_name = receiver.match(/\A[^:]+:\d+\/[^.]+/).to_s
+      receiver_node_name = Address.parse(receiver).node
       raise NotStartedYet.new unless @engine_nodes
       @engine_nodes.each do |node|
         if node.name == receiver_node_name
@@ -215,21 +219,23 @@ module Droonga
     end
 
     def all_node_names
-      raise NoCatalogLoaded.new unless @catalog
       @catalog.all_nodes
     end
 
     def create_engine_nodes
       all_node_names.collect do |name|
         node_state = @state[name] || {}
-        EngineNode.new(@loop,
-                       name,
-                       node_state,
-                       :auto_close_timeout =>
-                         @params[:internal_connection_lifetime])
+        create_engine_node(:name  => name,
+                           :state => node_state)
       end
     end
 
+    def create_engine_node(params)
+      EngineNode.new(params.merge(:loop => @loop,
+                                  :auto_close_timeout =>
+                                    @params[:internal_connection_lifetime]))
+    end
+
     def log_tag
       "cluster_state"
     end

data/lib/droonga/command/droonga_engine_service.rb

@@ -157,8 +157,11 @@ module Droonga
       end
 
       def run_engine
-        @engine = Engine.new(@loop, @engine_name, @internal_engine_name,
-                             :internal_connection_lifetime => @internal_connection_lifetime)
+        @engine = Engine.new(:loop          => @loop,
+                             :name          => @engine_name,
+                             :internal_name => @internal_engine_name,
+                             :internal_connection_lifetime =>
+                               @internal_connection_lifetime)
         @engine.on_ready = lambda do
           @worker_process_agent.ready
         end

data/lib/droonga/command/remote_command_base.rb

@@ -31,14 +31,14 @@ module Droonga
 
           option.separator("Connections:")
           option.on(:host=,
-                    "Host name of the target node.",
+                    "Host name of the node to be operated.",
                     :required => true)
           option.on(:port=,
-                    "Port number of the source cluster to be connected.",
+                    "Port number to communicate with the engine node.",
                     :as => Integer,
                     :default => NodeName::DEFAULT_PORT)
           option.on(:tag=,
-                    "Tag name of the soruce cluster to be connected.",
+                    "Tag name to communicate with the engine node.",
                     :default => NodeName::DEFAULT_TAG)
 
           option.separator("Miscellaneous:")