droonga-engine 1.0.1

data/lib/droonga/catalog/version2.rb

@@ -0,0 +1,96 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/base"
+require "droonga/catalog/dataset"
+require "droonga/catalog/version2_validator"
+
+module Droonga
+  module Catalog
+    class Version2 < Base
+      def initialize(data, path)
+        super
+        validate
+        prepare_data
+      end
+
+      def datasets
+        @datasets
+      end
+
+      def slices(name)
+        device = "."
+        pattern = Regexp.new("^#{name}\.")
+        results = {}
+        @datasets.each do |dataset_name, dataset|
+          n_workers = dataset.n_workers
+          plugins = dataset.plugins
+          dataset.replicas.each do |volume|
+            volume.slices.each do |slice|
+              volume_address = slice.volume.address
+              if pattern =~ volume_address
+                path = File.join([device, $POSTMATCH, "db"])
+                path = File.expand_path(path, base_path)
+                options = {
+                  :dataset => dataset_name,
+                  :database => path,
+                  :n_workers => n_workers,
+                  :plugins => plugins
+                }
+                results[volume_address] = options
+              end
+            end
+          end
+        end
+        results
+      end
+
+      def get_routes(name, args)
+        routes = []
+        dataset = dataset(name)
+        case args["type"]
+        when "broadcast"
+          volumes = dataset.replicas.select(args["replica"].to_sym)
+          volumes.each do |volume|
+            slices = volume.select_slices
+            slices.each do |slice|
+              routes << slice.volume.address
+            end
+          end
+        when "scatter"
+          volumes = dataset.replicas.select(args["replica"].to_sym)
+          volumes.each do |volume|
+            slice = volume.choose_slice(args["record"])
+            routes << slice.volume.address
+          end
+        end
+        routes
+      end
+
+      private
+      def validate
+        validator = Version2Validator.new(@data, @path)
+        validator.validate
+      end
+
+      def prepare_data
+        @datasets = {}
+        @data["datasets"].each do |name, dataset|
+          @datasets[name] = Dataset.new(name, dataset)
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog/version2_validator.rb

@@ -0,0 +1,63 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/errors"
+
+module Droonga
+  module Catalog
+    class Version2Validator
+      def initialize(data, path)
+        @data = data
+        @path = path
+      end
+
+      def validate
+        @details = []
+
+        validate_datasets
+
+        unless @details.empty?
+          raise ValidationError.new(@path, @details)
+        end
+      end
+
+      private
+      def validate_datasets
+        unless @data.key?("datasets")
+          required_parameter_is_missing("datasets")
+          return
+        end
+        @data["datasets"].each do |name, dataset|
+          validate_dataset(name, dataset)
+        end
+      end
+
+      def validate_dataset(name, dataset)
+        unless dataset.key?("replicas")
+          required_parameter_is_missing("datasets.#{name}.replicas")
+          return
+        end
+      end
+
+      def add_detail(value_path, message)
+        @details << ValidationError::Detail.new(value_path, message)
+      end
+
+      def required_parameter_is_missing(value_path)
+        add_detail(value_path, "required parameter is missing")
+      end
+    end
+  end
+end

data/lib/droonga/catalog/volume.rb

@@ -0,0 +1,33 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/single_volume"
+require "droonga/catalog/collection_volume"
+
+module Droonga
+  module Catalog
+    module Volume
+      class << self
+        def create(dataset, raw_volume)
+          if raw_volume.key?("address")
+            SingleVolume.new(raw_volume)
+          else
+            CollectionVolume.new(dataset, raw_volume)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog/volume_collection.rb

@@ -0,0 +1,56 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Catalog
+    class VolumeCollection
+      include Enumerable
+
+      def initialize(volumes)
+        @volumes = volumes
+      end
+
+      def each(&block)
+        @volumes.each(&block)
+      end
+
+      def ==(other)
+        other.is_a?(self.class) and
+          to_a == other.to_a
+      end
+
+      def eql?(other)
+        self == other
+      end
+
+      def hash
+        to_a.hash
+      end
+
+      def select(how=nil)
+        case how
+        when :top
+          [@volumes.first]
+        when :random
+          [@volumes.sample]
+        when :all
+          @volumes
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog_generator.rb

@@ -0,0 +1,156 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "time"
+
+module Droonga
+  class CatalogGenerator
+    def initialize
+      @version = 2
+      @effective_date = Time.now
+      @datasets = {}
+    end
+
+    def add_dataset(name, options)
+      @datasets[name] = Dataset.new(name, options)
+    end
+
+    def generate
+      {
+        "version"       => @version,
+        "effectiveDate" => @effective_date.iso8601,
+        "datasets"      => catalog_datasets,
+      }
+    end
+
+    private
+    def catalog_datasets
+      catalog_datasets = {}
+      @datasets.each do |name, dataset|
+        catalog_datasets[name] = dataset.to_catalog
+      end
+      catalog_datasets
+    end
+
+    class Dataset
+      def initialize(name, options)
+        @name = name
+        @options = options
+      end
+
+      def n_workers
+        @options[:n_workers] || 4
+      end
+
+      def plugins
+        @options[:plugins] || ["groonga", "search", "crud"]
+      end
+
+      def schema
+        @options[:schema] || {}
+      end
+
+      def fact
+        @options[:fact]
+      end
+
+      def replicas
+        return @options[:replicas] if @options[:replicas]
+        @generated_replicas ||= Replicas.new(@options).to_json
+      end
+
+      def to_catalog
+        catalog = {
+          "nWorkers" => n_workers,
+          "plugins"  => plugins,
+          "schema"   => schema,
+          "replicas" => replicas,
+        }
+        catalog["fact"] = fact if fact
+        catalog
+      end
+
+      private
+    end
+
+    class Replicas
+      def initialize(options={})
+        @hosts      = options[:hosts] || ["127.0.0.1"]
+        @port       = options[:port]
+        @tag        = options[:tag]
+        @n_slices   = options[:n_slices]
+      end
+
+      def to_json
+        @json ||= generate_json
+      end
+
+      private
+      def generate_json
+        replicas = []
+        @hosts.each do |host|
+          replica = Replica.new(host, :port => @port,
+                                      :tag => @tag,
+                                      :n_slices => @n_slices)
+          replicas << replica.to_json
+        end
+        replicas
+      end
+    end
+
+    class Replica
+      def initialize(host, options={})
+        @host       = host
+        @port       = options[:port]     || 10031
+        @tag        = options[:tag]      || "droonga"
+        @n_slices   = options[:n_slices] || 1
+
+        @n_volumes = 0
+      end
+
+      def to_json
+        @json ||= generate_json
+      end
+
+      private
+      def generate_json
+        slices = []
+        @n_slices.times do |index|
+          slices << generate_slice
+        end
+        {
+          "dimension" => "_key",
+          "slicer" => "hash",
+          "slices" => slices,
+        }
+      end
+
+      def generate_slice
+        name = sprintf('%03d', @n_volumes)
+        @n_volumes += 1
+        {
+          "weight" => weight,
+          "volume" => {
+            "address" => "#{@host}:#{@port}/#{@tag}.#{name}",
+          },
+        }
+      end
+
+      def weight
+        @weight ||= 100 / @n_slices
+      end
+    end
+  end
+end

data/lib/droonga/catalog_loader.rb

@@ -0,0 +1,56 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "json"
+
+require "droonga/catalog/version1"
+require "droonga/catalog/version2"
+
+module Droonga
+  class CatalogLoader
+    def initialize(path)
+      @path = path
+    end
+
+    def load
+      data = nil
+      begin
+        data = File.open(@path) do |file|
+          JSON.parse(file.read)
+        end
+      rescue Errno::ENOENT => error
+        raise Error.new("Missing catalog file #{@path}")
+      rescue JSON::ParserError => error
+        raise Error.new("Syntax error in #{@path}:\n#{error.to_s}")
+      end
+
+      unless data.is_a?(Hash)
+        raise Error.new("Root element of catalog must be an object in #{@path}")
+      end
+
+      version = data["version"]
+      case version
+      when 1
+        Catalog::Version1.new(data, @path)
+      when 2
+        Catalog::Version2.new(data, @path)
+      when nil
+        raise Error.new("Catalog version must be specified in #{@path}")
+      else
+        raise Error.new("Unsupported catalog version <#{version}> is specified in #{@path}")
+      end
+    end
+  end
+end

data/lib/droonga/catalog_observer.rb

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+require "droonga/catalog_loader"
+
+module Droonga
+  class CatalogObserver
+    include Loggable
+
+    DEFAULT_CATALOG_PATH = "catalog.json"
+    CHECK_INTERVAL = 1
+
+    attr_reader :catalog
+    attr_accessor :on_reload
+
+    def initialize(loop)
+      @loop = loop
+      @catalog_path = catalog_path
+      load_catalog!
+    end
+
+    def start
+      @watcher = Cool.io::TimerWatcher.new(CHECK_INTERVAL, true)
+      observer = self
+      @watcher.on_timer do
+        observer.ensure_latest_catalog_loaded
+      end
+      @loop.attach(@watcher)
+    end
+
+    def stop
+      @watcher.detach
+    end
+
+    def ensure_latest_catalog_loaded
+      if catalog_updated?
+        begin
+          load_catalog!
+          on_reload.call(catalog) if on_reload
+        rescue Droonga::Error => error
+          logger.warn("reload: fail", :path => @catalog_path, :error => error)
+        end
+      end
+    end
+
+    def catalog_path
+      path = ENV["DROONGA_CATALOG"] || DEFAULT_CATALOG_PATH
+      File.expand_path(path)
+    end
+
+    def catalog_updated?
+      File.mtime(catalog_path) > @catalog_mtime
+    end
+
+    def load_catalog!
+      loader = CatalogLoader.new(@catalog_path)
+      @catalog = loader.load
+      logger.info("loaded", :path => @catalog_path, :mtime => @catalog_mtime)
+    ensure
+      @catalog_mtime = File.mtime(@catalog_path)
+    end
+
+    private
+    def log_tag
+      "catalog-observer"
+    end
+  end
+end

data/lib/droonga/collector.rb

@@ -0,0 +1,38 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/pluggable"
+require "droonga/plugin/metadata/collector_message"
+require "droonga/error_messages"
+
+module Droonga
+  class Collector
+    extend Pluggable
+    include ErrorMessages
+
+    class << self
+      def message
+        Plugin::Metadata::CollectorMessage.new(self)
+      end
+    end
+
+    def initialize
+    end
+
+    def collect(message)
+      raise NotImplemented, "#{self.class.name}\##{__method__} must implement."
+    end
+  end
+end

data/lib/droonga/collector_message.rb

@@ -0,0 +1,71 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class CollectorMessage
+    attr_reader :raw
+    def initialize(raw)
+      @raw = raw
+    end
+
+    def valid?
+      task and step and values
+    end
+
+    def [](key)
+      @raw[key]
+    end
+
+    def task
+      @raw["task"]
+    end
+
+    def step
+      task["step"]
+    end
+
+    def type
+      step["type"]
+    end
+
+    def values
+      task["values"]
+    end
+
+    def body
+      step["body"]
+    end
+
+    def input
+      if body
+        body[name]
+      else
+        nil
+      end
+    end
+
+    def outputs
+      step["outputs"]
+    end
+
+    def name
+      @raw["name"]
+    end
+
+    def value
+      @raw["value"]
+    end
+  end
+end