droonga-engine 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 333168fcdecd92b88dfc590746445e0ed31319c4
-  data.tar.gz: bb082dd818cfd171a131e0e45dc75269a1cd814d
+  metadata.gz: 5e783348c7bfbef6e04de3de388c902839e8633b
+  data.tar.gz: 01b3367f3c22fb2a40245ca9bedc4920b42a494a
 SHA512:
-  metadata.gz: 38167299101e872c86cb07ac35449f023028ffaaa405477423c7882fd981c90677f4483671ea3c326e71338274326acbbfab644f668c77b7602d932b3c19717e
-  data.tar.gz: 41fe28134710faf85c694abe7c702e120d33188f321611e8627788b80ceabc5cca295f7d590621a85d1a87b936746a7d90e63cfeb938eb932287110c5548cfe1
+  metadata.gz: 399c8b21fd8368610a2ad498246584614f4bde6bc99bed78f8ba705164627a920ea59e32311ef5bf565fcaeb838e58837ab200a3d2ed0b2ee56f7ca26f1efa0e
+  data.tar.gz: 30bdef9c53baa70dd2095869656283488517d579942bc214c9267ff7c095e2c0aac9ff0d0e01db59263f4d4c86cf6f2165e5d4681db9af21726a2d69f4d798af

data/.gitignore

@@ -3,5 +3,6 @@
 /doc/
 /.yardoc/
 /pkg/
-/sample/cluster/0*
-/sample/cluster/state/
+/sample/cluster/0/
+/sample/cluster/1/
+/sample/cluster/2/

data/Gemfile

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 source "https://rubygems.org"
 

data/LICENSE.txt

@@ -11,4 +11,4 @@ Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

data/Rakefile

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "bundler/gem_helper"
 require "packnga"

data/benchmark/benchmark.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/client"
 

data/benchmark/utils.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "json"
 require "thread"

data/benchmark/watch/benchmark-notify.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 # this benchmark must be done by benchmark-notify.sh.
 

data/benchmark/watch/benchmark-publish.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "benchmark"
 require "fileutils"

data/benchmark/watch/benchmark-scan.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "benchmark"
 require "fileutils"

data/bin/droonga-engine

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/command/droonga_engine"
 

data/bin/droonga-engine-absorb-data

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "ostruct"
 require "optparse"
@@ -21,12 +21,15 @@ require "open3"
 
 require "droonga/engine/version"
 require "droonga/catalog_generator"
+require "droonga/path"
+require "droonga/data_absorber"
 require "droonga/serf"
 
 options = OpenStruct.new
 options.port    = Droonga::CatalogGenerator::DEFAULT_PORT
 options.tag     = Droonga::CatalogGenerator::DEFAULT_TAG
 options.dataset = Droonga::CatalogGenerator::DEFAULT_DATASET
+options.remote  = true
 parser = OptionParser.new
 parser.version = Droonga::Engine::VERSION
 
@@ -45,6 +48,11 @@ parser.on("--port=PORT", Integer,
           "(#{options.port})") do |port|
   options.port = port
 end
+parser.on("--[no-]remote",
+          "Run command in remote node or not.",
+          "(#{options.remote})") do |remote|
+  options.remote = remote
+end
 
 parser.separator("")
 parser.separator("Data:")
@@ -70,13 +78,45 @@ end
 
 destination_node = "#{options.destination_host}:#{options.port}/#{options.tag}"
 
-puts "Absorbing data..."
-Droonga::Serf.send_query(destination_node, "absorb_data",
-                         "node"    => destination_node,
-                         "source"  => options.source_host,
-                         "port"    => options.port,
-                         "tag"     => options.tag,
-                         "dataset" => options.dataset)
+def run_remote_command(target, command, options)
+  result = Droonga::Serf.send_query(target, command, options)
+  puts result[:result]
+  puts result[:error] unless result[:error].empty?
+  result[:response]
+end
+
+puts "Start to absorb data from #{options.source_host}"
+puts "                       to #{options.destination_host}"
+puts "  dataset = #{options.dataset}"
+puts "  port    = #{options.port}"
+puts "  tag     = #{options.tag}"
+puts ""
+puts "Absorbing..."
+
+if options.remote
+  run_remote_command(destination_node, "absorb_data",
+                     "node"    => destination_node,
+                     "source"  => options.source_host,
+                     "port"    => options.port,
+                     "tag"     => options.tag,
+                     "dataset" => options.dataset)
+  while true
+    sleep(3)
+    response = run_remote_command(destination_node, "report_status",
+                                  "node" => destination_node,
+                                  "key" => "absorbing")
+    absorbing = response["value"]
+    break unless absorbing
+  end
+else
+  Droonga::DataAbsorber.absorb(:dataset          => options.dataset,
+                               :source_host      => options.source_host,
+                               :destination_host => options.destination_host,
+                               :port             => options.port,
+                               :tag              => options.tag) do |output|
+    puts output
+  end
+end
 puts "Done."
 
 exit(true)

data/bin/droonga-engine-catalog-generate

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "ostruct"
 require "optparse"

data/bin/droonga-engine-catalog-modify

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "ostruct"
 require "optparse"

data/bin/droonga-engine-data-publisher

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "ostruct"
+require "optparse"
+require "pathname"
+require "fileutils"
+require "webrick"
+
+require "droonga/engine/version"
+require "droonga/path"
+
+options = OpenStruct.new
+options.base_dir = ENV[Droonga::Path::BASE_DIR_ENV_NAME] || Dir.pwd
+options.port = 10032
+
+parser = OptionParser.new
+parser.version = Droonga::Engine::VERSION
+
+parser.on("--base-dir=PATH",
+          "Path to the base directory the catalog.json is located in.",
+          "(#{options.base_dir})") do |path|
+  options.base_dir = path
+end
+parser.on("--port=PORT", Integer,
+          "Port number to listen.",
+          "(#{options.port})") do |port|
+  options.port = port
+end
+parser.on("--published-file=PATH",
+          "Path to the file to be published.") do |path|
+  options.published_file = path
+end
+
+parser.parse!(ARGV)
+
+include WEBrick
+
+published_dir = Droonga::Path.published(options.port)
+FileUtils.mkdir_p(published_dir.to_s)
+published_file = Pathname(options.published_file).expand_path
+FileUtils.copy(published_file.to_s, published_dir + published_file.basename)
+
+server = HTTPServer.new(:Port => options.port,
+                        :DocumentRoot => published_dir)
+
+trap("INT") do
+  server.shutdown
+  FileUtils.rm_rf(published_dir.to_s)
+end
+
+server.start

data/bin/droonga-engine-join

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "ostruct"
 require "optparse"
@@ -32,6 +32,8 @@ options.base_dir = ENV[Droonga::Path::BASE_DIR_ENV_NAME] || Dir.pwd
 options.drndump  = "drndump"
 options.client   = "droonga-request"
 options.copy     = true
+options.port     = Droonga::CatalogGenerator::DEFAULT_PORT
+options.tag      = Droonga::CatalogGenerator::DEFAULT_TAG
 
 parser = OptionParser.new
 parser.version = Droonga::Engine::VERSION
@@ -58,6 +60,21 @@ parser.on("--replica-source-host=HOST",
           "Host name of the soruce cluster to be connected.") do |host|
   options.replica_source_host = host
 end
+parser.on("--dataset=DATASET",
+          "Tag dataset name of the cluster to be joined as a node.",
+          "(#{options.dataset})") do |dataset|
+  options.dataset = dataset
+end
+parser.on("--port=PORT", Integer,
+          "Port number of the source cluster to be connected.",
+          "(#{options.port})") do |port|
+  options.port = port
+end
+parser.on("--tag=TAG",
+          "Tag name of the soruce cluster to be connected.",
+          "(#{options.tag})") do |tag|
+  options.tag = tag
+end
 
 parser.parse!(ARGV)
 
@@ -87,28 +104,66 @@ generator = Droonga::CatalogGenerator.new
 generator.load(source_catalog)
 
 dataset = generator.dataset_for_host(options.replica_source_host)
-unless dataset
-  raise "Specified source host #{options.replica_source_host} is not a " +
-          "member of the cluster. You must specify correct host via " +
-          "--replica-source-host option."
+if dataset
+  if generator.dataset_for_host(options.joining_host)
+    raise "The joining node is already a member of the cluster. " +
+            "You cannot join a member twice."
+  end
+
+  options.dataset = dataset.name
+  options.tag     = dataset.replicas.tag
+  options.port    = dataset.replicas.port
 end
 
-if generator.dataset_for_host(options.joining_host)
-  raise "The joining node is already a member of the cluster. " +
-          "You cannot join a member twice."
-end
+options.joining_node = "#{options.joining_host}:#{options.port}/#{options.tag}"
+options.source_node  = "#{options.replica_source_host}:#{options.port}/#{options.tag}"
 
-options.tag  = dataset.replicas.tag
-options.port = dataset.replicas.port
+def run_remote_command(target, command, options)
+  result = Droonga::Serf.send_query(target, command, options)
+  puts result[:result]
+  puts result[:error] unless result[:error].empty?
+  result[:response]
+end
 
-options.joining_node = "#{options.joining_host}:#{options.port}/#{options.tag}"
+=begin
+XXX disable fetching until it become working
+puts "Preparing to fetch cluster information..."
+publish_port = 10032 + rand(10000)
+run_remote_command(options.source_node, "publish_catalog",
+                   "node" => options.source_node,
+                   "port" => publish_port)
+sleep(3) # wait until the HTTP server becomes ready
+=end
 
 puts "Joining new replica to the cluster..."
-Droonga::Serf.send_query(options.joining_node, "join",
-                         "node"   => options.joining_node,
-                         "type"   => "replica",
-                         "source" => options.replica_source_host,
-                         "copy"   => options.copy)
+run_remote_command(options.joining_node, "join",
+                   "node"   => options.joining_node,
+                   "type"   => "replica",
+                   "source" => options.source_node,
+#                   "fetch_port" => publish_port,
+                   "copy"   => options.copy)
+sleep(5) #TODO: wait for restarting of the joining node. this should be done more safely.
+
+while true
+  sleep(3)
+  response = run_remote_command(options.joining_node, "report_status",
+                                "node" => options.joining_node,
+                                "key" => "absorbing")
+  absorbing = response["value"]
+  break unless absorbing
+end
+
+puts "Update existing hosts in the cluster..."
+run_remote_command(options.source_node, "add_replicas",
+                   "dataset" => options.dataset,
+                   "hosts"   => [options.joining_host])
+
+=begin
+XXX disable fetching until it become working
+run_remote_command(options.source_node, "unpublish_catalog",
+                   "node" => options.source_node,
+                   "port" => publish_port)
+=end
 
 puts "Done."
 

data/bin/droonga-engine-serf-event-handler

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/command/serf_event_handler"
 

data/bin/droonga-engine-service

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/command/droonga_engine_service"
 

data/bin/droonga-engine-unjoin

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "ostruct"
 require "optparse"
@@ -79,11 +79,18 @@ remaining_host = options.other_hosts.first || options.replica_remove_host
 options.remaining_node = "#{remaining_host}:#{options.port}/#{options.tag}"
 
 
+def run_remote_command(target, command, options)
+  result = Droonga::Serf.send_query(target, command, options)
+  puts result[:result]
+  puts result[:error] unless result[:error].empty?
+end
+
+
 puts "Unjoining replica from the cluster..."
 
-Droonga::Serf.send_query(options.remaining_node, "remove_replicas",
-                         "dataset" => options.dataset,
-                         "hosts"   => [options.replica_remove_host])
+run_remote_command(options.remaining_node, "remove_replicas",
+                   "dataset" => options.dataset,
+                   "hosts"   => [options.replica_remove_host])
 
 puts "Done."
 

data/bin/droonga-engine-worker

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "droonga/command/droonga_engine_worker"
+
+exit(Droonga::Command::DroongaEngineWorker.run(ARGV))

data/doc/text/news.md

@@ -1,5 +1,13 @@
 # News
 
+## 1.0.5: 2014-07-29
+
+ * Restarts server processes more gracefully.
+ * Works with search results with vector reference column values correctly.
+ * Messages forwarded to other nodes are always buffered for now.
+ * Works again for the case: `nWorkers` == `0`
+ * droonga-engine-join: Works correctly and safely for databases with much records.
+
 ## 1.0.4: 2014-06-29
 
  * New command (and plugin) [`status`](http://droonga.org/reference/1.0.4/commands/status/) is now available.

data/droonga-engine.gemspec

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 base_dir = File.dirname(__FILE__)
 $LOAD_PATH.unshift(File.join(base_dir, "lib"))
@@ -35,16 +35,16 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_dependency "rroonga", ">= 3.1.0"
+  gem.add_dependency "rroonga", ">= 4.0.1"
   gem.add_dependency "groonga-command-parser"
   gem.add_dependency "json"
   gem.add_dependency "cool.io"
-  gem.add_dependency "serverengine"
   gem.add_dependency "droonga-message-pack-packer", ">= 1.0.1"
   gem.add_dependency "faraday"
   gem.add_dependency "faraday_middleware"
   gem.add_dependency "archive-zip"
   gem.add_dependency "sigdump"
+  gem.add_dependency "droonga-client", ">= 0.1.9"
   gem.add_dependency "drndump"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "bundler"

data/lib/droonga/adapter.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/pluggable"
 require "droonga/loggable"

data/lib/droonga/adapter_runner.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/loggable"
 require "droonga/message_matcher"

data/lib/droonga/address.rb

@@ -0,0 +1,69 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Droonga
+  class Address
+    class << self
+      def parse(string)
+        if /\A(.+):(\d+)\/([^.]+)(?:\.(.+))?\z/ =~ string
+          components = {
+            :host => $1,
+            :port => $2.to_i,
+            :tag  => $3,
+            :name => $4
+          }
+          new(components)
+        else
+          format = "${host_name}:${port_number}/${tag}.${name}"
+          message = "volume address must be <#{format}> format: <#{string}>"
+          raise ArgumentError, message
+        end
+      end
+    end
+
+    DEFAULT_HOST = "127.0.0.1"
+    DEFAULT_PORT = 10031
+    DEFAULT_TAG  = "droonga"
+
+    attr_reader :host
+    attr_reader :port
+    attr_reader :tag
+    attr_reader :name
+    def initialize(components={})
+      @host = components[:host] || DEFAULT_HOST
+      @port = components[:port] || DEFAULT_PORT
+      @tag  = components[:tag]  || DEFAULT_TAG
+      @name = components[:name]
+    end
+
+    def to_s
+      string = node
+      string << ".#{@name}" if @name
+      string
+    end
+
+    def to_a
+      [@host, @port, @tag, @name]
+    end
+
+    def ==(other)
+      other.is_a?(self.class) and to_a == other.to_a
+    end
+
+    def node
+      "#{@host}:#{@port}/#{@tag}"
+    end
+  end
+end