fluent-plugin-droonga 0.9.9 → 1.0.0

data/lib/droonga/single_step.rb

@@ -0,0 +1,53 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/planner"
+require "droonga/collectors"
+
+module Droonga
+  class SingleStep
+    def initialize(definition)
+      @definition = definition
+    end
+
+    def plan(message)
+      if message["type"] == "search"
+        # XXX: workaround
+        planner = Plugins::Search::Planner.new
+        return planner.plan(message)
+      end
+
+      # XXX: Re-implement me.
+      planner = Planner.new
+      options = {}
+      options[:write] = @definition.write?
+      collector_class = @definition.collector_class
+      if collector_class
+        reduce_key = "result"
+        options[:reduce] = {
+          reduce_key => collector_class.operator,
+        }
+      end
+      inputs = @definition.inputs
+      if inputs.empty?
+        planner.send(:broadcast, message, options)
+      else
+        input = inputs.values.first
+        options[:key] = message["body"][input[:filter]]["key"]
+        planner.send(:scatter, message, options)
+      end
+    end
+  end
+end

data/lib/droonga/{plugin/planner/watch.rb → single_step_definition.rb}

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,39 +13,42 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/planner_plugin"
-
 module Droonga
-  class WatchPlanner < Droonga::PlannerPlugin
-    repository.register("watch", self)
-
-    command "watch.feed" => :feed
-    def feed(message)
-      broadcast(message)
+  class SingleStepDefinition
+    attr_accessor :name
+    attr_accessor :handler
+    attr_accessor :collector
+    attr_writer :write
+    attr_accessor :inputs
+    attr_accessor :output
+    def initialize(plugin_module)
+      @plugin_module = plugin_module
+      @name = nil
+      @handler = nil
+      @collector = nil
+      @write = false
+      @inputs = []
+      @output = {}
+      yield(self)
     end
 
-    command "watch.subscribe" => :subscribe
-    def subscribe(message)
-      broadcast(message)
+    def write?
+      @write
     end
 
-    command "watch.unsubscribe" => :unsubscribe
-    def unsubscribe(message)
-      broadcast(message)
+    def handler_class
+      resolve_class(@handler)
     end
 
-    command "watch.sweep" => :sweep
-    def sweep(message)
-      broadcast(message)
+    def collector_class
+      resolve_class(@collector)
     end
 
     private
-    def broadcast(message)
-      super(message,
-            :write => true,
-            :reduce => {
-              "success" => "and"
-            })
+    def resolve_class(target)
+      return nil if target.nil?
+      return target if target.is_a?(Class)
+      @plugin_module.const_get(target)
     end
   end
 end

data/lib/droonga/{partition.rb → slice.rb}

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,6 +15,7 @@
 
 require "serverengine"
 
+require "droonga/loggable"
 require "droonga/server"
 require "droonga/worker"
 require "droonga/event_loop"
@@ -24,7 +23,9 @@ require "droonga/message_pusher"
 require "droonga/processor"
 
 module Droonga
-  class Partition
+  class Slice
+    include Loggable
+
     def initialize(loop, options={})
       @options = options
       @n_workers = @options[:n_workers] || 0
@@ -43,21 +44,22 @@ module Droonga
     end
 
     def shutdown
-      $log.trace("partition: shutdown: start")
+      logger.trace("shutdown: start")
       shutdown_supervisor if @supervisor
       @message_pusher.shutdown
       @processor.shutdown
-      $log.trace("partition: shutdown: done")
+      logger.trace("shutdown: done")
     end
 
     def process(message)
-      $log.trace("partition: process: start")
+      logger.trace("process: start")
       @processor.process(message)
-      $log.trace("partition: process: done")
+      logger.trace("process: done")
     end
 
     private
     def ensure_database
+      enforce_umask
       database_path = @options[:database]
       return if File.exist?(database_path)
       FileUtils.mkdir_p(File.dirname(database_path))
@@ -70,12 +72,16 @@ module Droonga
       end
     end
 
+    def enforce_umask
+      File.umask(022)
+    end
+
     def start_supervisor
       @supervisor = ServerEngine::Supervisor.new(Server, Worker) do
         force_options = {
           :worker_type   => "process",
           :workers       => @options[:n_workers],
-          :log_level     => $log.level,
+          :log_level     => logger.level,
           :server_process_name => "Server[#{@options[:database]}] #$0",
           :worker_process_name => "Worker[#{@options[:database]}] #$0",
           :message_receiver => @message_pusher.raw_receiver,
@@ -88,11 +94,16 @@ module Droonga
     end
 
     def shutdown_supervisor
-      $log.trace("supervisor: shutdown: start")
+      logger.trace("supervisor: shutdown: start")
       @supervisor.stop(true)
-      $log.trace("supervisor: shutdown: stopped")
+      logger.trace("supervisor: shutdown: stopped")
       @supervisor_thread.join
-      $log.trace("supervisor: shutdown: done")
+      logger.trace("supervisor: shutdown: done")
+    end
+
+    private
+    def log_tag
+      "slice"
     end
   end
 end

data/lib/droonga/status_code.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module StatusCode
+    OK                    = 200
+    BAD_REQUEST           = 400
+    NOT_FOUND             = 404
+    INTERNAL_SERVER_ERROR = 500
+  end
+end

data/lib/droonga/step_runner.rb

@@ -0,0 +1,63 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+require "droonga/plugin"
+require "droonga/single_step"
+
+module Droonga
+  class StepRunner
+    include Loggable
+
+    def initialize(plugins)
+      @definitions = {}
+      plugins.each do |name|
+        plugin = Plugin.registry[name]
+        plugin.single_step_definitions.each do |definition|
+          @definitions[definition.name] = definition
+        end
+      end
+    end
+
+    def shutdown
+    end
+
+    def plan(message)
+      type = message["type"]
+      logger.trace("plan: start",
+                   :dataset => message["dataset"],
+                   :type => type)
+      definition = find(type)
+      if definition.nil?
+        raise UnsupportedMessageError.new(:planner, message)
+      end
+      step = SingleStep.new(definition)
+      plan = step.plan(message)
+      logger.trace("plan: done",
+                   :dataset => message["dataset"],
+                   :type => type)
+      plan
+    end
+
+    def find(type)
+      @definitions[type]
+    end
+
+    private
+    def log_tag
+      "step-runner"
+    end
+  end
+end

data/lib/droonga/watch_schema.rb

@@ -17,20 +17,24 @@
 
 require "groonga"
 
+require "droonga/loggable"
+
 module Droonga
   class WatchSchema
+    include Loggable
+
     def initialize(context)
       @context = context
     end
 
     def ensure_created
       if @context["Keyword"]
-        $log.trace "#{log_tag} skip table creation"
+        logger.trace("skip table creation")
         return
       end
-      $log.trace "#{log_tag} ensure_tables: start"
+      logger.trace("ensure_tables: start")
       ensure_tables
-      $log.trace "#{log_tag} ensure_tables: done"
+      logger.trace("ensure_tables: done")
     end
 
     private

data/lib/droonga/watcher.rb

@@ -34,8 +34,8 @@ module Droonga
       route      = request[:route]
 
       # XXX better validation and error class must be written!!
-      if subscriber.nil? || subscriber.empty? || condition.nil? ||
-         query.nil? || route.nil?
+      if subscriber.nil? or subscriber.empty? or condition.nil? or
+         query.nil? or route.nil?
         raise "invalid request"
       end
       raise "too long query" if query.size > 4095
@@ -65,14 +65,14 @@ module Droonga
       subscriber = request[:subscriber]
       query      = request[:query]
 
-      if subscriber.nil? || subscriber.empty?
+      if subscriber.nil? or subscriber.empty?
         raise "invalid request"
       end
 
       subscriber_record = @subscriber_table[subscriber]
       return unless subscriber_record
 
-      if query.nil? || query.empty?
+      if query.nil? or query.empty?
         delete_subscriber(subscriber_record)
       else
         query_record = @query_table[query]

data/lib/droonga/worker.rb

@@ -30,26 +30,26 @@ module Droonga
     end
 
     def run
-      $log.trace("#{log_tag}: run: start")
+      Droonga.logger.trace("#{log_tag}: run: start")
       @handler_runner.start
       @message_receiver.start
       @loop.run
       @handler_runner.shutdown
-      $log.trace("#{log_tag}: run: done")
+      Droonga.logger.trace("#{log_tag}: run: done")
     end
 
     def stop
-      $log.trace("#{log_tag}: stop: start")
+      Droonga.logger.trace("#{log_tag}: stop: start")
       @message_receiver.shutdown
       @loop.stop
-      $log.trace("#{log_tag}: stop: done")
+      Droonga.logger.trace("#{log_tag}: stop: done")
     end
 
     private
     def process(message)
-      $log.trace("#{log_tag}: process: start")
+      Droonga.logger.trace("#{log_tag}: process: start")
       @handler_runner.process(message)
-      $log.trace("#{log_tag}: process: done")
+      Droonga.logger.trace("#{log_tag}: process: done")
     end
 
     def log_tag

data/lib/fluent/plugin/out_droonga.rb

@@ -17,6 +17,7 @@
 
 require "droonga/engine"
 require "droonga/plugin_loader"
+require "droonga/catalog_observer"
 
 module Fluent
   class DroongaOutput < Output
@@ -27,12 +28,20 @@ module Fluent
     def start
       super
       Droonga::PluginLoader.load_all
-      @engine = Droonga::Engine.new(:name => @name)
+      @catalog_observer = Droonga::CatalogObserver.new
+      @catalog_observer.on_reload = lambda do |catalog|
+        graceful_engine_restart(catalog)
+        $log.info("engine restarted")
+      end
+      @catalog_observer.start
+      catalog = @catalog_observer.catalog
+      @engine = create_engine(catalog)
       @engine.start
     end
 
     def shutdown
       @engine.shutdown
+      @catalog_observer.stop
       super
     end
 
@@ -44,6 +53,22 @@ module Fluent
     end
 
     private
+    def create_engine(catalog)
+      Droonga::Engine.new(catalog, :name => @name)
+    end
+
+    def graceful_engine_restart(catalog)
+      $log.trace("out_droonga: start: graceful_engine_restart")
+      old_engine = @engine
+      $log.trace("out_droonga: creating new engine")
+      new_engine = create_engine(catalog)
+      new_engine.start
+      @engine = new_engine
+      $log.trace("out_droonga: shutdown old engine")
+      old_engine.shutdown
+      $log.trace("out_droonga: done: graceful_engine_restart")
+    end
+
     def process_event(tag, record)
       $log.trace("out_droonga: tag: <#{tag}>")
       @engine.process(parse_record(tag, record))
@@ -51,7 +76,7 @@ module Fluent
 
     def parse_record(tag, record)
       prefix, type, *arguments = tag.split(/\./)
-      if type.nil? || type.empty? || type == "message"
+      if type.nil? or type.empty? or type == "message"
         message = record
       else
         message = {