droonga-engine 1.0.4 → 1.0.5

data/lib/droonga/buffered_tcp_socket.rb

@@ -13,9 +13,9 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "cool.io"
+require "coolio"
 
 require "droonga/loggable"
 
@@ -34,32 +34,32 @@ module Droonga
     end
 
     def write(data)
-      reserve_write(data)
-      schedule_write
-      data.bytesize
-    end
-
-    def reserve_write(data)
       chunk = Chunk.new(@data_directory, data, Time.now, 0)
       chunk.buffering
       @_write_buffer << chunk
+      schedule_write
       data.bytesize
     end
 
     def on_writable
-      begin
+      until @_write_buffer.empty?
         chunk = @_write_buffer.shift
-        written_size = @_io.write(chunk.data)
-        if written_size == chunk.data.bytesize
-          chunk.written
-        else
-          chunk.written_partial(written_size)
+        begin
+          written_size = @_io.write_nonblock(chunk.data)
+          if written_size == chunk.data.bytesize
+            chunk.written
+          else
+            chunk.written_partial(written_size)
+            @_write_buffer.unshift(chunk)
+            break
+          end
+        rescue Errno::EINTR
           @_write_buffer.unshift(chunk)
+          return
+        rescue SystemCallError, IOError, SocketError
+          @_write_buffer.unshift(chunk)
+          return close
         end
-      rescue Errno::EINTR
-        return
-      rescue SystemCallError, IOError, SocketError
-        return close
       end
 
       if @_write_buffer.empty?
@@ -77,16 +77,38 @@ module Droonga
     private
     def load_chunks
       FileUtils.mkdir_p(@data_directory.to_s)
-      Pathname.glob("#{@data_directory}/*.chunk").collect do |chunk_path|
-        Chunk.load(chunk_path)
-      end
+      chunk_loader = ChunkLoader.new(@data_directory)
+      chunk_loader.load
     end
 
     def log_tag
-      "[#{Process.ppid}][#{Process.pid}] buffered-tcp-socket"
+      "[#{Process.ppid}] buffered-tcp-socket"
+    end
+
+    class ChunkLoader
+      def initialize(path)
+        @path = path
+      end
+
+      def have_any_chunk?
+        @path.opendir do |dir|
+          dir.each do |entry|
+            return true if entry.end_with?(Chunk::SUFFIX)
+          end
+        end
+        false
+      end
+
+      def load
+        Pathname.glob("#{@path}/*#{Chunk::SUFFIX}").collect do |chunk_path|
+          Chunk.load(chunk_path)
+        end
+      end
     end
 
     class Chunk
+      SUFFIX = ".chunk"
+
       class << self
         def load(path)
           data_directory = path.dirname

data/lib/droonga/catalog/base.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "digest/sha1"
 require "zlib"

data/lib/droonga/catalog/collection_volume.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "digest/sha1"
 require "zlib"

data/lib/droonga/catalog/dataset.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/catalog/schema"
 require "droonga/catalog/volume"
@@ -62,22 +62,22 @@ module Droonga
         @all_nodes ||= replicas.all_nodes
       end
 
-      def get_routes(args, live_nodes=nil)
+      def compute_routes(message, live_nodes)
         routes = []
-        case args["type"]
+        case message["type"]
         when "broadcast"
-          volumes = replicas.select(args["replica"].to_sym, live_nodes)
+          volumes = replicas.select(message["replica"].to_sym, live_nodes)
           volumes.each do |volume|
             slices = volume.select_slices
             slices.each do |slice|
-              routes << slice.volume.address
+              routes << slice.volume.address.to_s
             end
           end
         when "scatter"
-          volumes = replicas.select(args["replica"].to_sym, live_nodes)
+          volumes = replicas.select(message["replica"].to_sym, live_nodes)
           volumes.each do |volume|
-            slice = volume.choose_slice(args["record"])
-            routes << slice.volume.address
+            slice = volume.choose_slice(message["record"])
+            routes << slice.volume.address.to_s
           end
         end
         routes

data/lib/droonga/catalog/errors.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/error"
 

data/lib/droonga/catalog/schema.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "tsort"
 

data/lib/droonga/catalog/single_volume.rb

@@ -11,23 +11,21 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "droonga/address"
 
 module Droonga
   module Catalog
     class SingleVolume
+      attr_reader :address
       def initialize(data)
         @data = data
-      end
-
-      def address
-        @data["address"]
+        @address = Address.parse(@data["address"])
       end
 
       def node
-        ip_address_and_port, path = address.split("/")
-        tag = path.split(".").first
-        "#{ip_address_and_port}/#{tag}"
+        @address.node
       end
 
       def all_nodes

data/lib/droonga/catalog/slice.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Catalog

data/lib/droonga/catalog/version1.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "English"
 
@@ -385,7 +385,7 @@ module Droonga
       end
 
       class Dataset < Catalog::Dataset
-        def get_routes(args, live_nodes=nil)
+        def compute_routes(args, live_nodes=nil)
           routes = []
           case args["type"]
           when "broadcast"

data/lib/droonga/catalog/version2.rb

@@ -11,8 +11,9 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "droonga/address"
 require "droonga/catalog/base"
 require "droonga/catalog/dataset"
 require "droonga/catalog/version2_validator"
@@ -30,9 +31,8 @@ module Droonga
         @datasets
       end
 
-      def slices(name)
+      def slices(node)
         device = "."
-        pattern = Regexp.new("^#{name}\.")
         results = {}
         @datasets.each do |dataset_name, dataset|
           n_workers = dataset.n_workers
@@ -40,8 +40,8 @@ module Droonga
           dataset.replicas.each do |volume|
             volume.slices.each do |slice|
               volume_address = slice.volume.address
-              if pattern =~ volume_address
-                path = File.join([device, $POSTMATCH, "db"])
+              if volume_address.node == node
+                path = File.join([device, volume_address.name, "db"])
                 path = File.expand_path(path, base_path)
                 options = {
                   :dataset => dataset_name,
@@ -49,7 +49,7 @@ module Droonga
                   :n_workers => n_workers,
                   :plugins => plugins
                 }
-                results[volume_address] = options
+                results[volume_address.to_s] = options
               end
             end
           end

data/lib/droonga/catalog/version2_validator.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/catalog/errors"
 

data/lib/droonga/catalog/volume.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/catalog/single_volume"
 require "droonga/catalog/collection_volume"

data/lib/droonga/catalog/volume_collection.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Catalog
@@ -47,7 +47,7 @@ module Droonga
         when :random
           [volumes.sample]
         when :all
-          volumes
+          @volumes
         else
           super
         end

data/lib/droonga/catalog_generator.rb

@@ -11,10 +11,12 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "time"
 
+require "droonga/catalog/dataset"
+
 module Droonga
   class CatalogGenerator
     DEFAULT_DATASET = "Default"
@@ -56,8 +58,8 @@ module Droonga
     end
 
     def load(catalog)
-      catalog["datasets"].each do |name, dataset|
-        add_dataset(name, dataset_to_params(dataset))
+      catalog["datasets"].each do |name, catalog_dataset|
+        load_dataset(name, catalog_dataset)
       end
       self
     end
@@ -102,6 +104,16 @@ module Droonga
       catalog_datasets
     end
 
+    def load_dataset(name, catalog_dataset)
+      options = {}
+      options[:n_workers] = catalog_dataset["nWorkers"]
+      options[:plugins]   = catalog_dataset["plugins"]
+      options[:schema]    = catalog_dataset["schema"]
+      options[:fact]      = catalog_dataset["fact"]
+      options[:replicas]  = catalog_dataset["replicas"]
+      add_dataset(name, options)
+    end
+
     class Dataset
       attr_reader :name
 
@@ -127,8 +139,7 @@ module Droonga
       end
 
       def replicas
-        return @options[:replicas] if @options[:replicas]
-        @generated_replicas ||= Replicas.new(@options)
+        @replicas ||= create_replicas
       end
 
       def to_catalog
@@ -136,13 +147,23 @@ module Droonga
           "nWorkers" => n_workers,
           "plugins"  => plugins,
           "schema"   => schema,
-          "replicas" => replicas.to_json,
+          "replicas" => replicas.to_catalog,
         }
         catalog["fact"] = fact if fact
         catalog
       end
 
       private
+      def create_replicas
+        catalog_replicas = @options[:replicas]
+        if catalog_replicas
+          replicas = Replicas.new
+          replicas.load(catalog_replicas)
+          replicas
+        else
+          Replicas.new(@options)
+        end
+      end
     end
 
     class Replicas
@@ -156,20 +177,29 @@ module Droonga
         @n_slices   = options[:n_slices]
       end
 
-      def to_json
-        @json ||= generate_json
+      def load(catalog_replicas)
+        dataset = Catalog::Dataset.new("temporary",
+                                       "replicas" => catalog_replicas)
+        @hosts = dataset.replicas.collect do |replica|
+          replica.slices.first.volume.address.host
+        end
+        collection_volume = dataset.replicas.first
+        slices = collection_volume.slices
+        @n_slices = slices.size
+        single_volume_address = slices.first.volume.address
+        @port = single_volume_address.port
+        @tag = single_volume_address.tag
       end
 
-      private
-      def generate_json
-        replicas = []
+      def to_catalog
+        catalog_replicas = []
         @hosts.each do |host|
           replica = Replica.new(host, :port => @port,
                                       :tag => @tag,
                                       :n_slices => @n_slices)
-          replicas << replica.to_json
+          catalog_replicas << replica.to_catalog
         end
-        replicas
+        catalog_replicas
       end
     end
 
@@ -179,19 +209,12 @@ module Droonga
         @port       = options[:port]     || DEFAULT_PORT
         @tag        = options[:tag]      || DEFAULT_TAG
         @n_slices   = options[:n_slices] || DEFAULT_N_SLICES
-
-        @n_volumes = 0
-      end
-
-      def to_json
-        @json ||= generate_json
       end
 
-      private
-      def generate_json
+      def to_catalog
         slices = []
-        @n_slices.times do |index|
-          slices << generate_slice
+        @n_slices.times do |i|
+          slices << catalog_slice(i)
         end
         {
           "dimension" => "_key",
@@ -200,9 +223,9 @@ module Droonga
         }
       end
 
-      def generate_slice
-        name = sprintf('%03d', @n_volumes)
-        @n_volumes += 1
+      private
+      def catalog_slice(nth_slice)
+        name = "%03d" % nth_slice
         {
           "weight" => weight,
           "volume" => {
@@ -215,32 +238,5 @@ module Droonga
         @weight ||= 100 / @n_slices
       end
     end
-
-    ADDRESS_MATCHER = /\A(.*):(\d+)\/([^\.]+)\.(.+)\z/
-
-    def dataset_to_params(dataset)
-      params = {}
-      params[:n_workers] = dataset["nWorkers"]
-      params[:n_slices]  = dataset["replicas"].first["slices"].size
-      params[:plugins]   = dataset["plugins"]
-      params[:schema]    = dataset["schema"] if dataset["schema"]
-      params[:fact]      = dataset["fact"] if dataset["fact"]
-
-      nodes = dataset["replicas"].collect do |replica|
-        ADDRESS_MATCHER =~ replica["slices"].first["volume"]["address"]
-        {
-          :host => $1,
-          :port => $2.to_i,
-          :tag  => $3,
-          :path => $4,
-        }
-      end
-      params[:tag]   = nodes.first[:tag]
-      params[:port]  = nodes.first[:port].to_i
-      params[:hosts] = nodes.collect do |node|
-        node[:host]
-      end
-      params
-    end
   end
 end

data/lib/droonga/catalog_loader.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "json"
 

data/lib/droonga/collector.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/pluggable"
 require "droonga/plugin/metadata/collector_message"

data/lib/droonga/collector_message.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class CollectorMessage

data/lib/droonga/collector_runner.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/loggable"
 require "droonga/message_matcher"

data/lib/droonga/collectors/and.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Collectors

data/lib/droonga/collectors/or.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Collectors

data/lib/droonga/collectors/sum.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   module Collectors

data/lib/droonga/collectors.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/collectors/and"
 require "droonga/collectors/or"