fluent-plugin-droonga 0.9.9 → 1.0.0

data/lib/droonga/catalog/version2.rb

@@ -0,0 +1,160 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/base"
+require "droonga/catalog/dataset"
+
+module Droonga
+  module Catalog
+    class Version2 < Base
+      def initialize(data, path)
+        super
+        prepare_data
+      end
+
+      def datasets
+        @datasets
+      end
+
+      def slices(name)
+        device = "."
+        pattern = Regexp.new("^#{name}\.")
+        results = {}
+        @datasets.each do |dataset_name, dataset|
+          n_workers = dataset["nWorkers"]
+          plugins = dataset["plugins"]
+          dataset["replicas"].each do |replica|
+            replica["slices"].each do |slice|
+              volume_address = slice["volume"]["address"]
+              if pattern =~ volume_address
+                path = File.join([device, $POSTMATCH, "db"])
+                path = File.expand_path(path, base_path)
+                options = {
+                  :dataset => dataset_name,
+                  :database => path,
+                  :n_workers => n_workers,
+                  :plugins => plugins
+                }
+                results[volume_address] = options
+              end
+            end
+          end
+        end
+        results
+      end
+
+      def get_routes(name, args)
+        routes = []
+        dataset = dataset(name)
+        case args["type"]
+        when "broadcast"
+          replicas = select_replicas(dataset["replicas"], args["replica"])
+          replicas.each do |replica|
+            slices = select_slices(replica)
+            slices.each do |slice|
+              routes << slice["volume"]["address"]
+            end
+          end
+        when "scatter"
+          replicas = select_replicas(dataset["replicas"], args["replica"])
+          replicas.each do |replica|
+            slice = select_slice(replica, args["key"])
+            routes << slice["volume"]["address"]
+          end
+        end
+        routes
+      end
+
+      private
+      def validate
+        # TODO: Implement me.
+      end
+
+      def prepare_data
+        @datasets = {}
+        @data["datasets"].each do |name, dataset|
+          replicas = dataset["replicas"]
+          replicas.each do |replica|
+            total_weight = compute_total_weight(replica)
+            continuum = []
+            slices = replica["slices"]
+            n_slices = slices.size
+            slices.each do |slice|
+              weight = slice["weight"] || default_weight
+              points = n_slices * 160 * weight / total_weight
+              points.times do |point|
+                hash = Digest::SHA1.hexdigest("#{name}:#{point}")
+                continuum << [hash[0..7].to_i(16), slice]
+              end
+            end
+            replica["continuum"] = continuum.sort do |a, b|
+              a[0] - b[0]
+            end
+          end
+          @datasets[name] = Dataset.new(name, dataset)
+        end
+      end
+
+      def default_weight
+        1
+      end
+
+      def compute_total_weight(replica)
+        slices = replica["slices"]
+        slices.reduce(0) do |result, slice|
+          result + (slice["weight"] || default_weight)
+        end
+      end
+
+      def select_replicas(replicas, how)
+        case how
+        when "top"
+          [replicas.first]
+        when "random"
+          [replicas.sample]
+        when "all"
+          replicas
+        end
+      end
+
+      def select_slices(replica, range=0..-1)
+        sorted_slices = replica["slices"].sort_by do |slice|
+          slice["label"]
+        end
+        sorted_slices[range]
+      end
+
+      def select_slice(replica, key)
+        continuum = replica["continuum"]
+        return replica["slices"].first unless continuum
+
+        hash = Zlib.crc32(key)
+        min = 0
+        max = continuum.size - 1
+        while (min < max) do
+          index = (min + max) / 2
+          value, key = continuum[index]
+          return key if value == hash
+          if value > hash
+            max = index
+          else
+            min = index + 1
+          end
+        end
+        continuum[max][1]
+      end
+    end
+  end
+end

data/lib/droonga/catalog_loader.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,6 +16,7 @@
 require "json"
 
 require "droonga/catalog/version1"
+require "droonga/catalog/version2"
 
 module Droonga
   class CatalogLoader
@@ -24,10 +25,32 @@ module Droonga
     end
 
     def load
-      data = File.open(@path) do |file|
-        JSON.parse(file.read)
+      data = nil
+      begin
+        data = File.open(@path) do |file|
+          JSON.parse(file.read)
+        end
+      rescue Errno::ENOENT => error
+        raise Error.new("Missing catalog file #{@path}")
+      rescue JSON::ParserError => error
+        raise Error.new("Syntax error in #{@path}:\n#{error.to_s}")
+      end
+
+      unless data.is_a?(Hash)
+        raise Error.new("Root element of catalog must be an object in #{@path}")
+      end
+
+      version = data["version"]
+      case version
+      when 1
+        Catalog::Version1.new(data, @path)
+      when 2
+        Catalog::Version2.new(data, @path)
+      when nil
+        raise Error.new("Catalog version must be specified in #{@path}")
+      else
+        raise Error.new("Unsupported catalog version <#{version}> is specified in #{@path}")
       end
-      Catalog::Version1.new(data, File.dirname(@path))
     end
   end
 end

data/lib/droonga/catalog_observer.rb

@@ -15,15 +15,42 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "droonga/loggable"
+require "droonga/catalog_loader"
+
 module Droonga
   class CatalogObserver
+    include Loggable
+
     DEFAULT_CATALOG_PATH = "catalog.json"
     CHECK_INTERVAL = 1
 
-    def initialize(loop)
+    attr_reader :catalog
+    attr_accessor :on_reload
+
+    def initialize
       @catalog_path = catalog_path
-      load_catalog
+      load_catalog!
+    end
 
+    def start
+      @loop = Cool.io::Loop.new
+      attach(@loop)
+      @thread = Thread.new do
+        begin
+          @loop.run
+        rescue => error
+          logger.exception("error in catalog observing thread", error)
+        end
+      end
+    end
+
+    def stop
+      @loop.stop
+      @thread.join
+    end
+
+    def attach(loop)
       watcher = Cool.io::TimerWatcher.new(CHECK_INTERVAL, true)
       observer = self
       watcher.on_timer do
@@ -34,7 +61,12 @@ module Droonga
 
     def ensure_latest_catalog_loaded
       if catalog_updated?
-        load_catalog
+        begin
+          load_catalog!
+          on_reload.call(catalog) if on_reload
+        rescue Droonga::Error => error
+          logger.warn("reload: fail", :path => @catalog_path, :error => error)
+        end
       end
     end
 
@@ -47,11 +79,17 @@ module Droonga
       File.mtime(catalog_path) > @catalog_mtime
     end
 
-    def load_catalog
+    def load_catalog!
       loader = CatalogLoader.new(@catalog_path)
-      Droonga.catalog = loader.load
+      @catalog = loader.load
+      logger.info("loaded", :path => @catalog_path, :mtime => @catalog_mtime)
+    ensure
       @catalog_mtime = File.mtime(@catalog_path)
-      $log.info "catalog loaded", path: @catalog_path, mtime: @catalog_mtime
+    end
+
+    private
+    def log_tag
+      "catalog-observer"
     end
   end
 end

data/lib/droonga/collector.rb

@@ -13,24 +13,26 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/legacy_pluggable"
-require "droonga/collector_plugin"
+require "droonga/pluggable"
+require "droonga/plugin/metadata/collector_message"
+require "droonga/error_messages"
 
 module Droonga
   class Collector
-    include LegacyPluggable
+    extend Pluggable
+    include ErrorMessages
 
-    def initialize(plugins)
-      load_plugins(plugins)
+    class << self
+      def message
+        Plugin::Metadata::CollectorMessage.new(self)
+      end
     end
 
-    private
-    def instantiate_plugin(name)
-      CollectorPlugin.repository.instantiate(name)
+    def initialize
     end
 
-    def log_tag
-      "collector"
+    def collect(message)
+      raise NotImplemented, "#{self.class.name}\##{__method__} must implement."
     end
   end
 end

data/lib/droonga/{handler_plugin.rb → collector_message.rb}

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,30 +13,59 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/legacy_plugin"
-require "droonga/message_processing_error"
-
 module Droonga
-  class HandlerPlugin < LegacyPlugin
-    extend PluginRegisterable
+  class CollectorMessage
+    attr_reader :raw
+    def initialize(raw)
+      @raw = raw
+    end
+
+    def valid?
+      task and step and values
+    end
 
-    def initialize(handler)
-      super()
-      @handler = handler
-      @context = @handler.context
+    def [](key)
+      @raw[key]
     end
 
-    def prefer_synchronous?(command)
-      false
+    def task
+      @raw["task"]
     end
 
-    private
-    def run_command(command, message, messenger)
-      begin
-        super
-      rescue MessageProcessingError => error
-        messenger.error(error.status_code, error.response_body)
+    def step
+      task["step"]
+    end
+
+    def type
+      step["type"]
+    end
+
+    def values
+      task["values"]
+    end
+
+    def body
+      step["body"]
+    end
+
+    def input
+      if body
+        body[name]
+      else
+        nil
       end
     end
+
+    def outputs
+      step["outputs"]
+    end
+
+    def name
+      @raw["name"]
+    end
+
+    def value
+      @raw["value"]
+    end
   end
 end

data/lib/droonga/collector_runner.rb

@@ -0,0 +1,64 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+require "droonga/message_matcher"
+require "droonga/collector"
+require "droonga/collector_message"
+
+module Droonga
+  class CollectorRunner
+    include Loggable
+
+    def initialize(plugins)
+      default_plugins = ["basic"]
+      plugins += (default_plugins - plugins)
+      @collector_classes = Collector.find_sub_classes(plugins)
+    end
+
+    def shutdown
+    end
+
+    def collect(message)
+      collector_message = CollectorMessage.new(message)
+      logger.trace("collect: start",
+                   :type => collector_message.type)
+      collector_class = find_collector_class(message)
+      if collector_class.nil?
+        raise UnsupportedMessageError.new(:collector, message)
+      end
+      collector = collector_class.new
+      collector.collect(collector_message)
+      logger.trace("collector: done")
+    end
+
+    private
+    def find_collector_class(message)
+      @collector_classes.find do |collector_class|
+        pattern = collector_class.message.pattern
+        if pattern
+          matcher = MessageMatcher.new(pattern)
+          matcher.match?(message)
+        else
+          false
+        end
+      end
+    end
+
+    def log_tag
+      "collector-runner"
+    end
+  end
+end