droonga-engine 1.0.9 → 1.1.0

data/install/centos/functions.sh

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,7 +20,7 @@ register_service() {
 
   #TODO: we should migrate to systemd in near future...
 
-  curl -o /etc/rc.d/init.d/$NAME $(download_url "install/centos/$NAME")
+  curl -s -o /etc/rc.d/init.d/$NAME $(download_url "install/centos/$NAME")
   if [ $? -ne 0 ]; then
     echo "ERROR: Failed to download service script!"
     exit 1

data/install/debian/functions.sh

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,7 +18,7 @@ register_service() {
   local USER=$2
   local GROUP=$3
 
-  curl -o /etc/init.d/$NAME $(download_url "install/debian/$NAME")
+  curl -s -o /etc/init.d/$NAME $(download_url "install/debian/$NAME")
   if [ $? -ne 0 ]; then
     echo "ERROR: Failed to download service script!"
     exit 1

data/lib/droonga/address.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,7 +13,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "socket"
+require "droonga/node_name"
 
 module Droonga
   class Address
@@ -24,49 +24,51 @@ module Droonga
             :host => $1,
             :port => $2.to_i,
             :tag  => $3,
-            :name => $4
+            :local_name => $4
           }
           new(components)
         else
-          format = "${host_name}:${port_number}/${tag}.${name}"
+          format = "${host_name}:${port_number}/${tag}.${local_name}"
           message = "volume address must be <#{format}> format: <#{string}>"
           raise ArgumentError, message
         end
       end
     end
 
-    DEFAULT_HOST = Socket.gethostname
-    DEFAULT_HOST.force_encoding("US-ASCII") if DEFAULT_HOST.ascii_only?
-    DEFAULT_PORT = 10031
-    DEFAULT_TAG  = "droonga"
-
-    attr_reader :host
-    attr_reader :port
-    attr_reader :tag
-    attr_reader :name
+    attr_reader :local_name
     def initialize(components={})
-      @host = components[:host] || DEFAULT_HOST
-      @port = components[:port] || DEFAULT_PORT
-      @tag  = components[:tag]  || DEFAULT_TAG
-      @name = components[:name]
+      @node_name  = NodeName.new(components)
+      @local_name = components[:local_name]
+    end
+
+    def host
+      @node_name.host
+    end
+
+    def port
+      @node_name.port
+    end
+
+    def tag
+      @node_name.tag
+    end
+
+    def node
+      @node_name.node
     end
 
     def to_s
-      string = node
-      string << ".#{@name}" if @name
+      string = @node_name.node
+      string << ".#{@local_name}" if @local_name
       string
     end
 
     def to_a
-      [@host, @port, @tag, @name]
+      @node_name.to_a + [@local_name]
     end
 
     def ==(other)
       other.is_a?(self.class) and to_a == other.to_a
     end
-
-    def node
-      "#{@host}:#{@port}/#{@tag}"
-    end
   end
 end

data/lib/droonga/buffered_tcp_socket.rb

@@ -18,11 +18,15 @@
 require "coolio"
 
 require "droonga/loggable"
+require "droonga/timestamp"
 
 module Droonga
   class BufferedTCPSocket < Coolio::TCPSocket
     include Loggable
 
+    class AlreadyInWritingByOthers < StandardError
+    end
+
     def initialize(socket, data_directory)
       super(socket)
       @data_directory = data_directory
@@ -34,7 +38,8 @@ module Droonga
     end
 
     def write(data)
-      chunk = Chunk.new(@data_directory, data, Time.now, 0)
+      chunk = Chunk.new(:directory => @data_directory,
+                        :data      => data)
       chunk.buffering
       @_write_buffer << chunk
       schedule_write
@@ -45,19 +50,35 @@ module Droonga
       until @_write_buffer.empty?
         chunk = @_write_buffer.shift
         begin
+          chunk.writing
+          logger.trace("Sending...", :data => chunk.data)
           written_size = @_io.write_nonblock(chunk.data)
           if written_size == chunk.data.bytesize
             chunk.written
+            logger.trace("Completely sent.")
           else
             chunk.written_partial(written_size)
+            logger.trace("Partially sent. Retry later.",
+                         :written => written_size,
+                         :rest    => chunk.data.bytesize)
             @_write_buffer.unshift(chunk)
             break
           end
+        rescue AlreadyInWritingByOthers
+          logger.trace("Chunk is already in sending by another process.")
         rescue Errno::EINTR
           @_write_buffer.unshift(chunk)
+          chunk.failed
+          logger.trace("Failed to send chunk. Retry later.",
+                       :chunk => chunk,
+                       :errpr => "Errno::EINTR")
           return
-        rescue SystemCallError, IOError, SocketError
+        rescue SystemCallError, IOError, SocketError => exception
           @_write_buffer.unshift(chunk)
+          chunk.failed
+          logger.trace("Failed to send chunk. Retry later.",
+                       :chunk => chunk,
+                       :exception => exception)
           return close
         end
       end
@@ -108,24 +129,30 @@ module Droonga
 
     class Chunk
       SUFFIX = ".chunk"
+      WRITING_SUFFIX = ".writing"
 
       class << self
         def load(path)
           data_directory = path.dirname
-          time_stamp1, time_stamp2, revision, = path.basename.to_s.split(".", 4)
+          time_stamp1, time_stamp2, uniqueness, revision, = path.basename.to_s.split(".", 5)
           data = path.open("rb") {|file| file.read}
           time_stamp = Time.iso8601("#{time_stamp1}.#{time_stamp2}")
           revision = Integer(revision)
-          new(data_directory, data, time_stamp, revision)
+          new(:directory  => data_directory,
+              :data       => data,
+              :time_stamp => time_stamp,
+              :uniqueness => uniqueness,
+              :revision   => revision)
         end
       end
 
       attr_reader :data, :time_stamp
-      def initialize(data_directory, data, time_stamp, revision)
-        @data_directory = data_directory
-        @data = data
-        @time_stamp = time_stamp.utc
-        @revision = revision
+      def initialize(params)
+        @data_directory = params[:directory]
+        @data = params[:data]
+        @time_stamp = params[:time_stamp] || Time.now
+        @uniqueness = params[:uniqueness]
+        @revision = params[:revision] || 0
       end
 
       def buffering
@@ -134,8 +161,23 @@ module Droonga
         end
       end
 
+      def writing
+        raise AlreadyInWritingByOthers.new if writing?
+        FileUtils.mv(path.to_s, writing_path.to_s)
+      end
+
+      def writing?
+        not path.exist?
+      end
+
+      def failed
+        return unless writing?
+        FileUtils.mv(writing_path.to_s, path.to_s)
+      end
+
       def written
         FileUtils.rm_f(path.to_s)
+        FileUtils.rm_f(writing_path.to_s)
       end
 
       def written_partial(size)
@@ -147,7 +189,20 @@ module Droonga
 
       private
       def path
-        @data_directory + "#{@time_stamp.iso8601(6)}.#{@revision}.#{SUFFIX}"
+        @path ||= create_chunk_file_path
+      end
+
+      def writing_path
+        @writing_path ||= Pathname("#{path.to_s}#{WRITING_SUFFIX}")
+      end
+
+      def create_chunk_file_path
+        basename = Timestamp.stringify(@time_stamp)
+        if @uniqueness
+          @data_directory + "#{basename}.#{@uniqueness}.#{@revision}#{SUFFIX}"
+        else
+          Path.unique_file_path(@data_directory, basename, "#{@revision}#{SUFFIX}")
+        end
       end
     end
   end

data/lib/droonga/catalog/base.rb

@@ -24,10 +24,10 @@ module Droonga
   module Catalog
     class Base
       attr_reader :path, :base_path
-      def initialize(data, path)
-        @data = data
-        @path = path
-        @base_path = File.dirname(path)
+      def initialize(raw, path)
+        @raw = raw
+        @path = path || "/tmp/temporary-catalog.json"
+        @base_path = File.dirname(@path)
       end
 
       def have_dataset?(name)
@@ -44,8 +44,11 @@ module Droonga
 
       private
       def calculate_cluster_id
-        raw_id = all_nodes.sort.join(",")
-        Digest::SHA1.hexdigest(raw_id)
+        raw_id = []
+        datasets.each do |name, dataset|
+          raw_id << "#{name}-#{dataset.all_nodes.sort.join(",")}"
+        end
+        Digest::SHA1.hexdigest(raw_id.sort.join("|"))
       end
 
       def migrate_database_location(current_db_path, device, name)

data/lib/droonga/catalog/dataset.rb

@@ -15,7 +15,6 @@
 
 require "droonga/catalog/schema"
 require "droonga/catalog/volume"
-require "droonga/catalog/volume_collection"
 
 module Droonga
   module Catalog
@@ -24,80 +23,57 @@ module Droonga
 
       attr_reader :name
 
-      def initialize(name, data)
+      def initialize(name, raw)
         @name = name
-        @data = data
+        @raw = raw
         @schema = nil
       end
 
       # provided for compatibility
       def [](key)
-        @data[key]
+        @raw[key]
       end
 
       # provided for compatibility
       def []=(key, value)
-        @data[key] = value
+        @raw[key] = value
       end
 
       def schema
-        @schema ||= Schema.new(@name, @data["schema"])
+        @schema ||= Schema.new(@name, @raw["schema"])
       end
 
       def plugins
-        @data["plugins"] || []
+        @raw["plugins"] || []
       end
 
       def fact
-        @data["fact"]
+        @raw["fact"]
       end
 
       def n_workers
-        @data["nWorkers"] || 0
+        @raw["nWorkers"] || 0
       end
 
+      #XXX Currently, dataset has a property named "replicas" so
+      #    can be parsed as a ReplicasVolume.
+      #    We must introduce a new property "volume" to provide
+      #    ReplicasVolume safely.
       def replicas
-        @replicas ||= VolumeCollection.new(create_volumes(@data["replicas"]))
+        @replicas ||= ReplicasVolume.new(self, @raw)
       end
 
       def all_nodes
         @all_nodes ||= replicas.all_nodes
       end
 
-      def compute_routes(message, live_nodes)
-        routes = []
-        case message["type"]
-        when "broadcast"
-          volumes = replicas.select(message["replica"].to_sym, live_nodes)
-          volumes.each do |volume|
-            slices = volume.select_slices
-            slices.each do |slice|
-              routes << slice.volume.address.to_s
-            end
-          end
-        when "scatter"
-          volumes = replicas.select(message["replica"].to_sym, live_nodes)
-          volumes.each do |volume|
-            slice = volume.choose_slice(message["record"])
-            routes << slice.volume.address.to_s
-          end
-        end
-        routes
+      def compute_routes(message, active_nodes)
+        @replicas.compute_routes(message, active_nodes)
       end
 
-      def single_slice?
+      def sliced?
         # TODO: Support slice key
-        replicas.all? do |volume|
-          volume.is_a?(SingleVolume) or
-            volume.slices.size == 1
-        end
-      end
-
-      private
-      def create_volumes(raw_volumes)
-        raw_volumes.collect do |raw_volume|
-          Volume.create(self, raw_volume)
-        end
+        @replicas.sliced?
       end
     end
   end

data/lib/droonga/catalog/fetcher.rb

@@ -0,0 +1,64 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "socket"
+
+require "droonga/client"
+
+require "droonga/node_name"
+require "droonga/catalog/dataset"
+
+module Droonga
+  module Catalog
+    class Fetcher
+      class EmptyResponse < StandardError
+      end
+
+      class EmptyCatalog < StandardError
+      end
+
+      def initialize(client_options)
+        @client_options = default_options.merge(client_options)
+      end
+
+      def fetch(options={})
+        message = {
+          "dataset" => options[:dataset] || Catalog::Dataset::DEFAULT_NAME,
+          "type"    => "catalog.fetch"
+        }
+        response = nil
+        Droonga::Client.open(@client_options) do |client|
+          response = client.request(message)
+        end
+        raise EmptyResponse.new unless response
+        raise EmptyCatalog.new unless response["body"]
+        response["body"]
+      end
+
+      private
+      def default_options
+        {
+          :host          => "127.0.0.1",
+          :port          => NodeName::DEFAULT_PORT,
+          :tag           => NodeName::DEFAULT_TAG,
+          :protocol      => :droonga,
+          :timeout       => 1,
+          :receiver_host => Socket.gethostname,
+          :receiver_port => 0,
+        }
+      end
+    end
+  end
+end