fluent-plugin-droonga 0.7.0 → 0.8.0

data/lib/droonga/server.rb

@@ -15,8 +15,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/job_queue"
-
 module Droonga
   module Server
     def before_run
@@ -39,70 +37,5 @@ module Droonga
     def log_tag
       "[#{Process.ppid}][#{Process.pid}] server"
     end
-
-    def start_worker(wid)
-      worker = super(wid)
-      worker.extend(WorkerStopper)
-      worker
-    end
-
-    module WorkerStopper
-      def send_stop(stop_graceful)
-        in_signal_sending do
-          open_queue do |queue|
-            $log.trace("#{log_tag}: stop: start")
-
-            $log.trace("#{log_tag}: stop: queue: unblock: start")
-            max_n_retries = 20
-            max_n_retries.times do |i|
-              $log.trace("#{log_tag}: stop: queue: unblock: #{i}: start")
-              super(stop_graceful)
-              queue.unblock
-              alive_p = alive?
-              $log.trace("#{log_tag}: stop: queue: unblock: #{i}: done: " +
-                         "#{alive_p}")
-              break unless alive_p
-              sleep(i * 0.1)
-            end
-            $log.trace("#{log_tag}: stop: queue: unblock: done")
-
-            $log.trace("#{log_tag}: stop: done")
-          end
-        end
-      end
-
-      def send_reload
-        in_signal_sending do
-          open_queue do |queue|
-            $log.trace("#{log_tag}: reload: start")
-            super
-            queue.unblock
-            $log.trace("#{log_tag}: reload: done")
-          end
-        end
-      end
-
-      private
-      def log_tag
-        "[#{Process.ppid}][#{Process.pid}][#{@wid}] server: worker-stopper"
-      end
-
-      def in_signal_sending
-        Thread.new do
-          yield
-        end
-      end
-
-      def open_queue
-        config = @worker.config
-        queue = JobQueue.open(config[:database],
-                              config[:queue_name] || "DroongaQueue")
-        begin
-          yield(queue)
-        ensure
-          queue.close
-        end
-      end
-    end
   end
 end

data/lib/droonga/session.rb

@@ -0,0 +1,85 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Session
+    def initialize(id, dispatcher, collector, tasks, inputs)
+      @id = id
+      @dispatcher = dispatcher
+      @collector = collector
+      @tasks = tasks
+      @n_dones = 0
+      @inputs = inputs
+    end
+
+    def done?
+      @n_dones == @tasks.size
+    end
+
+    def start
+      tasks = @inputs[nil]
+      tasks.each do |task|
+        local_message = {
+          "id"   => @id,
+          "task" => task,
+        }
+        @dispatcher.process_local_message(local_message)
+        @n_dones += 1
+      end
+    end
+
+    def receive(name, value)
+      tasks = @inputs[name]
+      unless tasks
+        #TODO: result arrived before its query
+        return
+      end
+      tasks.each do |task|
+        task["n_of_inputs"] += 1
+        component = task["component"]
+        type = component["type"]
+        command = "collector_" + type
+        n_of_expects = component["n_of_expects"]
+        message = {
+          "task"=>task,
+          "name"=>name,
+          "value"=>value
+        }
+        @collector.process(command, message)
+        return if task["n_of_inputs"] < n_of_expects
+        #the task is done
+        result = task["values"]
+        post = component["post"]
+        @dispatcher.reply("body" => result) if post
+        component["descendants"].each do |name, routes|
+          message = {
+            "id" => @id,
+            "input" => name,
+            "value" => result[name]
+          }
+          routes.each do |route|
+            @dispatcher.dispatch(message, route)
+          end
+        end
+        @n_dones += 1
+      end
+    end
+
+    private
+    def log_tag
+      "session"
+    end
+  end
+end

data/lib/droonga/test.rb

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/test/stub_distributor"
+require "droonga/test/stub_handler"
+require "droonga/test/stub_handler_message"
+require "droonga/test/stub_handler_messenger"

data/lib/droonga/test/stub_distributor.rb

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Test
+    class StubDistributor
+      attr_reader :messages
+      def initialize
+        @messages = []
+      end
+
+      def distribute(message)
+        @messages << message
+      end
+    end
+  end
+end

data/lib/droonga/test/stub_handler.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Test
+    class StubHandler
+      attr_reader :context, :envelope, :messages
+      def initialize
+        @context = Groonga::Context.default
+        @envelope = {}
+        @messages = []
+      end
+
+      def emit(message)
+        @messages << message
+      end
+
+      def forward(message, destination)
+        @messages << [message, destination]
+      end
+    end
+  end
+end

data/lib/droonga/test/stub_handler_message.rb

@@ -0,0 +1,35 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/handler_message"
+
+module Droonga
+  module Test
+    class StubHandlerMessage < HandlerMessage
+      def initialize(request, headers={})
+        raw = {
+          "body" => {
+            "task" => {
+              "component" => {
+                "body" => request,
+              },
+            },
+          },
+        }
+        super(headers.merge(raw))
+      end
+    end
+  end
+end

data/lib/droonga/test/stub_handler_messenger.rb

@@ -0,0 +1,34 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Test
+    class StubHandlerMessenger
+      attr_reader :values, :messages
+      def initialize
+        @values = []
+        @messages = []
+      end
+
+      def emit(value)
+        @values << value
+      end
+
+      def forward(message, destination)
+        @messages << [message, destination]
+      end
+    end
+  end
+end

data/lib/droonga/time_formatter.rb

@@ -0,0 +1,37 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "time"
+
+module Droonga
+  class TimeFormatter
+    class << self
+      def format(object)
+        formatter = new(object)
+        formatter.format
+      end
+    end
+
+    MICRO_SECONDS_DECIMAL_PLACE = 6
+
+    def initialize(time)
+      @time = time
+    end
+
+    def format
+      @time.utc.iso8601(MICRO_SECONDS_DECIMAL_PLACE)
+    end
+  end
+end

data/lib/droonga/watcher.rb

@@ -99,6 +99,7 @@ module Droonga
       targets.each do |key, target|
         scan_body(hits, target)
       end
+      hits.uniq! # hits may be duplicated if multiple targets are matched
 
       publish(hits, request, &block)
     end

data/lib/droonga/worker.rb

@@ -15,44 +15,41 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/job_queue"
+require "droonga/event_loop"
 require "droonga/handler"
+require "droonga/message_receiver"
 
 module Droonga
   module Worker
     def initialize
-      @database_name = config[:database]
-      @queue_name = config[:queue_name] || "DroongaQueue"
+      @loop = EventLoop.new
+      @handler = Handler.new(@loop, config.merge(:dispatcher => nil))
+      receiver_socket = config[:message_receiver]
+      @message_receiver = MessageReceiver.new(@loop, receiver_socket) do |message|
+        process(message)
+      end
     end
 
     def run
       $log.trace("#{log_tag}: run: start")
-      handler = Handler.new(config)
-      job_queue = JobQueue.open(@database_name, @queue_name)
-      @running = true
-      while @running
-        process(handler, job_queue)
-      end
-      handler.shutdown
-      job_queue.close
+      @handler.start
+      @message_receiver.start
+      @loop.run
+      @handler.shutdown
       $log.trace("#{log_tag}: run: done")
     end
 
     def stop
       $log.trace("#{log_tag}: stop: start")
-      @running = false
+      @message_receiver.shutdown
+      @loop.stop
       $log.trace("#{log_tag}: stop: done")
     end
 
     private
-    def process(handler, job_queue)
+    def process(message)
       $log.trace("#{log_tag}: process: start")
-      envelope = job_queue.pull_message
-      unless envelope
-        $log.trace("#{log_tag}: process: abort: no message")
-        return
-      end
-      handler.process(envelope)
+      @handler.process(message)
       $log.trace("#{log_tag}: process: done")
     end
 

data/lib/fluent/plugin/out_droonga.rb

@@ -45,30 +45,30 @@ module Fluent
 
     private
     def process_event(tag, record)
-      $log.trace("out_droonga: tag: <#{tag}> caller: <#{caller.first}>")
+      $log.trace("out_droonga: tag: <#{tag}>")
       @engine.process(parse_record(tag, record))
     end
 
     def parse_record(tag, record)
       prefix, type, *arguments = tag.split(/\./)
-      if type.nil? || type.empty? || type == 'message'
-        envelope = record
+      if type.nil? || type.empty? || type == "message"
+        message = record
       else
-        envelope = {
+        message = {
           "type" => type,
           "arguments" => arguments,
           "body" => record
         }
       end
-      envelope["via"] ||= []
-      reply_to = envelope["replyTo"]
+      message["via"] ||= []
+      reply_to = message["replyTo"]
       if reply_to.is_a? String
-        envelope["replyTo"] = {
-          "type" => envelope["type"] + ".result",
+        message["replyTo"] = {
+          "type" => "#{message["type"]}.result",
           "to" => reply_to
         }
       end
-      envelope
+      message
     end
   end
 end