droonga-engine 1.0.2 → 1.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d592eda2e4916fb9d0d4fbf48ad538cbff3c1d34
+  data.tar.gz: 62255e98b9141709a42db836e525737a8e254343
+SHA512:
+  metadata.gz: 163a844fae7ba67af1dc934380a374756439744e844de81e53fe9dc141f6a7b1ad457db25a0e7a1fda988d6709aab34e55b9ecd0efc3dabb6c6be508bf6fb466
+  data.tar.gz: 183c9afe679f550c198e65f9f7f5dfab74b10ded4e84bfd40d1b93239227c571063e05920aa6e514eaa11a46a35cdad8f320265f60534b6944071c2132e79e94

data/.gitignore

@@ -4,3 +4,4 @@
 /.yardoc/
 /pkg/
 /sample/cluster/0*
+/sample/cluster/state/

data/.travis.yml

@@ -13,3 +13,6 @@ rvm:
 before_install:
 #  - GROONGA_MASTER=yes curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh
   - curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh
+  - curl --location --remote-name https://dl.bintray.com/mitchellh/serf/0.6.0_linux_amd64.zip
+  - unzip 0.6.0_linux_amd64.zip
+  - sudo install serf /usr/local/bin/

data/Gemfile

@@ -37,6 +37,13 @@ elsif ENV["TRAVIS"] == "true"
   end
 end
 
+droonga_message_pack_packer_dir = File.join(parent_dir, "droonga-message-pack-packer-ruby")
+if File.exist?(droonga_message_pack_packer_dir)
+  gem "droonga-message-pack-packer", :path => droonga_message_pack_packer_dir
+else
+  gem "droonga-message-pack-packer", :github => "droonga/droonga-message-pack-packer-ruby"
+end
+
 droonga_client_dir = File.join(parent_dir, "droonga-client-ruby")
 if File.exist?(droonga_client_dir)
   gem "droonga-client", :path => droonga_client_dir

data/Rakefile

@@ -30,8 +30,12 @@ Packnga::DocumentTask.new(spec) do |task|
 end
 
 def run_command_test(*options)
-  ruby(File.join(File.dirname(__FILE__), "test", "command", "run-test.rb",),
-       *options)
+  default_options = []
+  if ENV["TRAVIS"] == "true"
+    default_options.push("--timeout", "3")
+  end
+  ruby(File.join(File.dirname(__FILE__), "test", "command", "run-test.rb"),
+       *(default_options + options))
 end
 
 namespace :test do

data/bin/droonga-engine

@@ -15,6 +15,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/engine/command/droonga_engine"
+require "droonga/command/droonga_engine"
 
-exit(Droonga::Engine::Command::DroongaEngine::Supervisor.run(ARGV))
+exit(Droonga::Command::DroongaEngine.run(ARGV))

data/bin/{droonga-catalog-generate → droonga-engine-catalog-generate}

@@ -18,13 +18,17 @@
 require "ostruct"
 require "optparse"
 require "json"
+require "tempfile"
+require "pathname"
 
 require "droonga/engine/version"
 require "droonga/catalog_generator"
 
 generator = Droonga::CatalogGenerator.new
-current_dataset = nil
-datasets = {}
+current_dataset = {}
+datasets = {
+  Droonga::CatalogGenerator::DEFAULT_DATASET => current_dataset
+}
 
 options = OpenStruct.new
 options.output_path = "-"
@@ -83,6 +87,10 @@ parser.on("--replicas=PATH",
 end
 parser.parse!(ARGV)
 
+if datasets[Droonga::CatalogGenerator::DEFAULT_DATASET].empty?
+  datasets.delete(Droonga::CatalogGenerator::DEFAULT_DATASET)
+end
+
 datasets.each do |name, options|
   generator.add_dataset(name, options)
 end
@@ -91,8 +99,12 @@ def open_output(path)
   if path == "-"
     yield($stdout)
   else
-    File.open(path, "w") do |output|
+    # Don't output the file directly to prevent loading of incomplete file!
+    path = Pathname(path).expand_path
+    Tempfile.open(path.basename.to_s, path.parent.to_s, "w") do |output|
       yield(output)
+      output.flush
+      File.rename(output.path, path.to_s)
     end
   end
 end

data/bin/droonga-engine-serf-event-handler

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/command/serf_event_handler"
+
+exit(Droonga::Command::SerfEventHandler.run)

data/bin/droonga-engine-service

@@ -15,6 +15,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/engine/command/droonga_engine"
+require "droonga/command/droonga_engine_service"
 
-exit(Droonga::Engine::Command::DroongaEngine::Service.run(ARGV))
+exit(Droonga::Command::DroongaEngineService.run(ARGV))

data/doc/text/news.md

@@ -1,6 +1,26 @@
 # News
 
-## 1.0.2: 2014-04-29 (planned)
+## 1.0.3: 2014-05-29
+
+ * Alive monitoring (based on [Serf](http://serfdom.io/)) lands.
+   Now, nodes in a cluster observe each other, and the cluster keeps working, even if one of replicas is dead.
+ * New commands to dump whole contents of an existing cluster are available.
+   They are used by [`drndump`](https://github.com/droonga/drndump) internally.
+ * The command line tool `droonga-catalog-generate` is renamed to `droonga-engine-catalog-generate`.
+ * Use `Default` as the name of the default dataset for a `catalog.json`, generated by `droonga-catalog-generate` .
+ * The path of the configuration directory is now specified via an environment variable `DROONGA_BASE_DIR`.
+ * Fix incompatibilities of the `select` Groonga command.
+   * The default value of the `drilldown_output_columns` option becomes same to Groonga's one.
+   * Column values of `Time` type clumns are returned as float numbers correctly.
+   * The message structure of results becomes same to Groonga.
+     In previous version, records are wrongly wrapped in an array.
+ * Improve features of the `select` command..
+   * The request parameter `"attributes"` for `"elements"` in `"output"` is now available.
+   * The special value `"*"` for `"attributes"` in `"output"` is now available, to export all columns.
+ * Server process does shutdown/restart gracefully.
+ * Restart itself automatically when the `catalog.json` is updated.
+
+## 1.0.2: 2014-04-29
 
 The most important topic on this release is that the core component aka Droonga Engine becomes fluentd-free.
 As the result, the project (and gem package) `fluent-plugin-droonga` is renamed to `droonga-engine`.

data/droonga-engine.gemspec

@@ -40,10 +40,13 @@ Gem::Specification.new do |gem|
   gem.add_dependency "json"
   gem.add_dependency "cool.io"
   gem.add_dependency "serverengine"
-  gem.add_dependency "droonga-message-pack-packer"
+  gem.add_dependency "droonga-message-pack-packer", ">= 1.0.1"
+  gem.add_dependency "listen", "~> 2.7"
+  gem.add_dependency "faraday"
+  gem.add_dependency "faraday_middleware"
+  gem.add_dependency "archive-zip"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "bundler"
-  gem.add_development_dependency "droonga-client"
   gem.add_development_dependency "test-unit"
   gem.add_development_dependency "test-unit-notify"
   gem.add_development_dependency "test-unit-rr"

data/lib/droonga/catalog/collection_volume.rb

@@ -69,6 +69,10 @@ module Droonga
         slicer == "hash"
       end
 
+      def all_nodes
+        @all_nodes ||= collect_all_nodes
+      end
+
       private
       def compute_continuum
         total_weight = compute_total_weight
@@ -101,6 +105,14 @@ module Droonga
           key
         end
       end
+
+      def collect_all_nodes
+        nodes = []
+        slices.each do |slice|
+          nodes += slice.all_nodes
+        end
+        nodes.uniq.sort
+      end
     end
   end
 end

data/lib/droonga/catalog/dataset.rb

@@ -58,6 +58,31 @@ module Droonga
         @replicas ||= VolumeCollection.new(create_volumes(@data["replicas"]))
       end
 
+      def all_nodes
+        @all_nodes ||= replicas.all_nodes
+      end
+
+      def get_routes(args, live_nodes=nil)
+        routes = []
+        case args["type"]
+        when "broadcast"
+          volumes = replicas.select(args["replica"].to_sym, live_nodes)
+          volumes.each do |volume|
+            slices = volume.select_slices
+            slices.each do |slice|
+              routes << slice.volume.address
+            end
+          end
+        when "scatter"
+          volumes = replicas.select(args["replica"].to_sym, live_nodes)
+          volumes.each do |volume|
+            slice = volume.choose_slice(args["record"])
+            routes << slice.volume.address
+          end
+        end
+        routes
+      end
+
       private
       def create_volumes(raw_volumes)
         raw_volumes.collect do |raw_volume|

data/lib/droonga/catalog/single_volume.rb

@@ -23,6 +23,16 @@ module Droonga
       def address
         @data["address"]
       end
+
+      def node
+        ip_address_and_port, path = address.split("/")
+        tag = path.split(".").first
+        "#{ip_address_and_port}/#{tag}"
+      end
+
+      def all_nodes
+        @all_nodes ||= [node]
+      end
     end
   end
 end

data/lib/droonga/catalog/slice.rb

@@ -36,6 +36,10 @@ module Droonga
       def volume
         @volume ||= Volume.create(@dataset, @data["volume"])
       end
+
+      def all_nodes
+        @all_nodes ||= volume.all_nodes
+      end
     end
   end
 end

data/lib/droonga/catalog/version1.rb

@@ -13,6 +13,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "English"
+
 require "droonga/catalog/base"
 require "droonga/catalog/dataset"
 
@@ -66,53 +68,8 @@ module Droonga
         return results
       end
 
-      def get_routes(name, args)
-        routes = []
-        dataset = dataset(name)
-        case args["type"]
-        when "broadcast"
-          dataset["ring"].each do |key, partition|
-            select_range_and_replicas(partition, args, routes)
-          end
-        when "scatter"
-          name = get_partition(dataset, args["record"]["_key"])
-          partition = dataset["ring"][name]
-          select_range_and_replicas(partition, args, routes)
-        end
-        return routes
-      end
-
-      def get_partition(dataset, key)
-        continuum = dataset["continuum"]
-        return dataset["ring"].keys[0] unless continuum
-        hash = Zlib.crc32(key)
-        min = 0
-        max = continuum.size - 1
-        while (min < max) do
-          index = (min + max) / 2
-          value, key = continuum[index]
-          return key if value == hash
-          if value > hash
-            max = index
-          else
-            min = index + 1
-          end
-        end
-        return continuum[max][1]
-      end
-
-      def select_range_and_replicas(partition, args, routes)
-        date_range = args["date_range"] || 0..-1
-        partition["partitions"].sort[date_range].each do |time, replicas|
-          case args["replica"]
-          when "top"
-            routes << replicas[0]
-          when "random"
-            routes << replicas[rand(replicas.size)]
-          when "all"
-            routes.concat(replicas)
-          end
-        end
+      def all_nodes
+        @all_nodes ||= collect_all_nodes
       end
 
       private
@@ -142,6 +99,10 @@ module Droonga
         end
       end
 
+      def collect_all_nodes
+        @data["zones"].sort
+      end
+
       def validate
         do_validation do
           validate_effective_date
@@ -408,8 +369,8 @@ module Droonga
                   unless partition =~ valid_farms_matcher
                     raise UnknownFarmForPartition.new(name, partition, @path)
                   end
+                  directory_name = $POSTMATCH
                   do_validation do
-                    directory_name = $POSTMATCH
                     if directory_name.nil? or directory_name.empty?
                       message = "\"#{partition}\" has no database name. " +
                                   "You mus specify a database name for \"#{name}\"."
@@ -422,6 +383,56 @@ module Droonga
           end
         end
       end
+
+      class Dataset < Catalog::Dataset
+        def get_routes(args, live_nodes=nil)
+          routes = []
+          case args["type"]
+          when "broadcast"
+            self["ring"].each do |key, partition|
+              select_range_and_replicas(partition, args, routes)
+            end
+          when "scatter"
+            name = get_partition(args["record"]["_key"])
+            partition = self["ring"][name]
+            select_range_and_replicas(partition, args, routes)
+          end
+          return routes
+        end
+
+        def get_partition(key)
+          continuum = self["continuum"]
+          return self["ring"].keys[0] unless continuum
+          hash = Zlib.crc32(key)
+          min = 0
+          max = continuum.size - 1
+          while (min < max) do
+            index = (min + max) / 2
+            value, key = continuum[index]
+            return key if value == hash
+            if value > hash
+              max = index
+            else
+              min = index + 1
+            end
+          end
+          return continuum[max][1]
+        end
+
+        def select_range_and_replicas(partition, args, routes)
+          date_range = args["date_range"] || 0..-1
+          partition["partitions"].sort[date_range].each do |time, replicas|
+            case args["replica"]
+            when "top"
+              routes << replicas[0]
+            when "random"
+              routes << replicas[rand(replicas.size)]
+            when "all"
+              routes.concat(replicas)
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/droonga/catalog/version2.rb

@@ -57,26 +57,8 @@ module Droonga
         results
       end
 
-      def get_routes(name, args)
-        routes = []
-        dataset = dataset(name)
-        case args["type"]
-        when "broadcast"
-          volumes = dataset.replicas.select(args["replica"].to_sym)
-          volumes.each do |volume|
-            slices = volume.select_slices
-            slices.each do |slice|
-              routes << slice.volume.address
-            end
-          end
-        when "scatter"
-          volumes = dataset.replicas.select(args["replica"].to_sym)
-          volumes.each do |volume|
-            slice = volume.choose_slice(args["record"])
-            routes << slice.volume.address
-          end
-        end
-        routes
+      def all_nodes
+        @all_nodes ||= collect_all_nodes
       end
 
       private
@@ -91,6 +73,14 @@ module Droonga
           @datasets[name] = Dataset.new(name, dataset)
         end
       end
+
+      def collect_all_nodes
+        nodes = []
+        @datasets.each do |name, dataset|
+          nodes += dataset.all_nodes
+        end
+        nodes.sort.uniq
+      end
     end
   end
 end