fluent-plugin-droonga 0.0.2 → 0.7.0

data/lib/droonga/executor.rb

@@ -1,289 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 droonga project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-require "fluent-logger"
-require "groonga"
-
-require "droonga/job_queue"
-require "droonga/handler_plugin"
-require "droonga/plugin"
-require "droonga/proxy"
-
-module Droonga
-  class Executor
-    attr_reader :context, :envelope, :name
-
-    def initialize(options={})
-      @handlers = []
-      @outputs = {}
-      @options = options
-      @name = options[:name]
-      @database_name = options[:database]
-      @queue_name = options[:queue_name]
-      @handler_names = options[:handlers]
-      @pool_size = options[:n_workers]
-#     load_handlers
-      Droonga::Plugin.load_all
-      prepare
-    end
-
-    def shutdown
-      $log.trace("#{log_tag}: shutdown: start")
-      @handlers.each do |handler|
-        handler.shutdown
-      end
-      @outputs.each do |dest, output|
-        output[:logger].close if output[:logger]
-      end
-      if @database
-        @database.close
-        @context.close
-        @database = @context = nil
-      end
-      if @job_queue
-        @job_queue.close
-        @job_queue = nil
-      end
-      $log.trace("#{log_tag}: shutdown: done")
-    end
-
-    def add_handler(name)
-      plugin = HandlerPlugin.new(name)
-      @handlers << plugin.instantiate(self)
-    end
-
-    def add_route(route)
-      envelope["via"].push(route)
-    end
-
-    def dispatch(tag, time, record, synchronous=nil)
-      $log.trace("#{log_tag}: dispatch: start")
-      message = [tag, time, record]
-      body, type, arguments = parse_message([tag, time, record])
-      reply_to = envelope["replyTo"]
-      if reply_to.is_a? String
-        envelope["replyTo"] = {
-          "type" => type + ".result",
-          "to" => reply_to
-        }
-      end
-      post_or_push(message, body,
-                   "type" => type,
-                   "arguments" => arguments,
-                   "synchronous" => synchronous)
-      $log.trace("#{log_tag}: dispatch: done")
-    end
-
-    def execute_one
-      $log.trace("#{log_tag}: execute_one: start")
-      message = @job_queue.pull_message
-      unless message
-        $log.trace("#{log_tag}: execute_one: abort: no message")
-        return
-      end
-      body, command, arguments = parse_message(message)
-      handler = find_handler(command)
-      if handler
-        $log.trace("#{log_tag}: execute_one: handle: start",
-                   :hander => handler.class)
-        handler.handle(command, body, *arguments)
-        $log.trace("#{log_tag}: execute_one: handle: done",
-                   :hander => handler.class)
-      end
-      $log.trace("#{log_tag}: execute_one: done")
-    end
-
-    def post(body, destination=nil)
-      $log.trace("#{log_tag}: post: start")
-      post_or_push(nil, body, destination)
-      $log.trace("#{log_tag}: post: done")
-    end
-
-    private
-    def post_or_push(message, body, destination)
-      route = nil
-      unless is_route?(destination)
-        route = envelope["via"].pop
-        destination = route
-      end
-      unless is_route?(destination)
-        destination = envelope["replyTo"]
-      end
-      command = nil
-      receiver = nil
-      arguments = nil
-      synchronous = nil
-      case destination
-      when String
-        command = destination
-      when Hash
-        command = destination["type"]
-        receiver = destination["to"]
-        arguments = destination["arguments"]
-        synchronous = destination["synchronous"]
-      end
-      if receiver
-        output(receiver, body, command, arguments)
-      else
-        handler = find_handler(command)
-        if handler
-          if synchronous.nil?
-            synchronous = handler.prefer_synchronous?(command)
-          end
-          if route || @pool_size.zero? || synchronous
-            $log.trace("#{log_tag}: post_or_push: handle: start")
-            handler.handle(command, body, *arguments)
-            $log.trace("#{log_tag}: post_or_push: handle: done")
-          else
-            unless message
-              envelope["body"] = body
-              envelope["type"] = command
-              envelope["arguments"] = arguments
-              message = ['', Time.now.to_f, envelope]
-            end
-            @job_queue.push_message(message)
-          end
-        end
-      end
-      add_route(route) if route
-    end
-
-    def is_route?(route)
-      route.is_a?(String) || route.is_a?(Hash)
-    end
-
-    def output(receiver, body, command, arguments)
-      $log.trace("#{log_tag}: output: start")
-      unless receiver.is_a?(String) && command.is_a?(String)
-        $log.trace("#{log_tag}: output: abort: invalid argument",
-                   :receiver => receiver,
-                   :command  => command)
-        return
-      end
-      unless receiver =~ /\A(.*):(\d+)\/(.*?)(\?.+)?\z/
-        raise "format: hostname:port/tag(?params)"
-      end
-      host = $1
-      port = $2
-      tag  = $3
-      params = $4
-      output = get_output(host, port, params)
-      unless output
-        $log.trace("#{log_tag}: output: abort: no output",
-                   :host   => host,
-                   :port   => port,
-                   :params => params)
-        return
-      end
-      if command =~ /\.result$/
-        message = {
-          inReplyTo: envelope["id"],
-          statusCode: 200,
-          type: command,
-          body: body
-        }
-      else
-        message = envelope.merge(
-          body: body,
-          type: command,
-          arguments: arguments
-        )
-      end
-      output_tag = "#{tag}.message"
-      $log.trace("#{log_tag}: output: post: start: <#{output_tag}>")
-      output.post(output_tag, message)
-      $log.trace("#{log_tag}: output: post: done: <#{output_tag}>")
-      $log.trace("#{log_tag}: output: done")
-    end
-
-    def parse_message(message)
-      tag, time, record = message
-      prefix, type, *arguments = tag.split(/\./)
-      if type.nil? || type.empty? || type == 'message'
-        @envelope = record
-      else
-        @envelope = {
-          "type" => type,
-          "arguments" => arguments,
-          "body" => record
-        }
-      end
-      envelope["via"] ||= []
-      [envelope["body"], envelope["type"], envelope["arguments"]]
-    end
-
-    def load_handlers
-      @handler_names.each do |handler_name|
-        plugin = Droonga::Plugin.new("handler", handler_name)
-        plugin.load
-      end
-    end
-
-    def prepare
-      if @database_name && !@database_name.empty?
-        @context = Groonga::Context.new
-        @database = @context.open_database(@database_name)
-        @job_queue = JobQueue.open(@database_name, @queue_name)
-      end
-      @handler_names.each do |handler_name|
-        add_handler(handler_name)
-      end
-      add_handler("proxy_message") if @options[:proxy]
-    end
-
-    def find_handler(command)
-      @handlers.find do |handler|
-        handler.handlable?(command)
-      end
-    end
-
-    def get_output(host, port, params)
-      host_port = "#{host}:#{port}"
-      @outputs[host_port] ||= {}
-      output = @outputs[host_port]
-
-      has_connection_id = (not params.nil? \
-                           and params =~ /[\?&;]connection_id=([^&;]+)/)
-      if output[:logger].nil? or has_connection_id
-        connection_id = $1
-        if not has_connection_id or output[:connection_id] != connection_id
-          output[:connection_id] = connection_id
-          logger = create_logger(:host => host, :port => port.to_i)
-          # output[:logger] should be closed if it exists beforehand?
-          output[:logger] = logger
-        end
-      end
-
-      has_client_session_id = (not params.nil? \
-                               and params =~ /[\?&;]client_session_id=([^&;]+)/)
-      if has_client_session_id
-        client_session_id = $1
-        # some generic way to handle client_session_id is expected
-      end
-
-      output[:logger]
-    end
-
-    def create_logger(options)
-      Fluent::Logger::FluentLogger.new(nil, options)
-    end
-
-    def log_tag
-      "[#{Process.ppid}][#{Process.pid}] executor"
-    end
-  end
-end

data/lib/droonga/plugin/handler_proxy.rb

@@ -1,82 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 droonga project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-require "droonga/handler"
-require "droonga/searcher"
-
-module Droonga
-  class BasicProxyHandler < Droonga::ProxyHandler
-    Droonga::HandlerPlugin.register("proxy", self)
-
-    command :proxy_gather
-    def proxy_gather(request)
-      output = body ? body[input_name] : input_name
-      emit(request, output)
-    end
-
-    command :proxy_reduce
-    def proxy_reduce(request)
-      return unless request
-      body[input_name].each do |output, elements|
-        value = request
-        old_value = output_values[output]
-        value = reduce(elements, old_value, request) if old_value
-        emit(value, output)
-      end
-    end
-
-    def reduce(elements, *values)
-      result = {}
-      elements.each do |key, deal|
-        func, *args = deal
-        case func
-        when "sum"
-          result[key] = values[0][key] + values[1][key]
-        when "sort"
-          result[key] = merge(values[0][key], values[1][key], args)
-        end
-      end
-      return result
-    end
-
-    def merge(x, y, order)
-      index = 0
-      y.each do |_y|
-        loop do
-          _x = x[index]
-          break unless _x
-          break if compare(_y, _x, order)
-          index += 1
-        end
-        x.insert(index, _y)
-        index += 1
-      end
-      return x
-    end
-
-    def compare(x, y, operators)
-      for index in 0..x.size-1 do
-        _x = x[index]
-        _y = y[index]
-        operator = operators[index]
-        break unless operator
-        return true if _x.__send__(operator, _y)
-      end
-      return false
-    end
-  end
-end

data/test/plugin/handler/test_handler_search.rb

@@ -1,512 +0,0 @@
-# Copyright (C) 2013 droonga project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-require "droonga/plugin/handler_search"
-
-class SearchHandlerTest < Test::Unit::TestCase
-  def setup
-    setup_database
-    setup_handler
-  end
-
-  def teardown
-    teardown_handler
-    teardown_database
-  end
-
-  private
-  def setup_database
-    restore(fixture_data("document.grn"))
-    @database = Groonga::Database.open(@database_path.to_s)
-  end
-
-  def teardown_database
-    @database.close
-    @database = nil
-  end
-
-  def setup_handler
-    @worker = StubWorker.new
-    @handler = Droonga::SearchHandler.new(@worker)
-  end
-
-  def teardown_handler
-    @handler = nil
-  end
-
-  def search(request)
-    @handler.search(request)
-    normalize_result_set(@worker.body)
-  end
-
-  def normalize_result_set(result_set)
-    result_set.each do |name, result|
-      result["startTime"] = start_time if result["startTime"]
-      result["elapsedTime"] = elapsed_time if result["elapsedTime"]
-    end
-    result_set
-  end
-
-  def start_time
-    "2013-01-31T14:34:47+09:00"
-  end
-
-  def elapsed_time
-    0.01
-  end
-
-  def assert_search(expected, request)
-    assert_equal(expected, search(request))
-  end
-
-  class NoParameterTest < self
-    def test_empty
-      assert_search({}, {})
-    end
-  end
-
-  class QueriesTest < self
-    def test_empty
-      assert_search({}, {"queries" => {}})
-    end
-  end
-
-  class HashQueryTest < self
-    def test_string_matchTo
-      request = base_request
-      request["queries"]["sections-result"]["condition"] = {
-        "query" => "Groonga",
-        "matchTo" => "title"
-      }
-      assert_search({
-                      "sections-result" => {
-                        "records" => [
-                          { "title" => "Groonga overview" },
-                        ],
-                      },
-                    },
-                    request)
-    end
-
-    def test_array_matchTo
-      request = base_request
-      request["queries"]["sections-result"]["condition"] = {
-        "query" => "Groonga",
-        "matchTo" => ["title"]
-      }
-      assert_search({
-                      "sections-result" => {
-                        "records" => [
-                          { "title" => "Groonga overview" },
-                        ],
-                      },
-                    },
-                    request)
-    end
-
-    def base_request
-      {
-        "queries" => {
-          "sections-result" => {
-            "source" => "Sections",
-            "output" => {
-              "elements" => [
-                "records",
-              ],
-              "format" => "complex",
-              "limit" => 1,
-              "attributes" => ["title"],
-            },
-          },
-        },
-      }
-    end
-  end
-
-  class SourceTest < self
-    def test_non_existent
-      assert_raise(Droonga::Searcher::UndefinedSourceError) do
-        search({
-                 "queries" => {
-                   "non-existent-result" => {
-                     "source" => "non-existent",
-                   },
-                 },
-               })
-      end
-    end
-
-    def test_existent
-      assert_search({
-                      "sections-result" => {},
-                    },
-                    {
-                      "queries" => {
-                        "sections-result" => {
-                          "source" => "Sections",
-                          "output" => {},
-                        },
-                      },
-                    })
-    end
-  end
-
-  class OutputTest < self
-    def test_count
-      assert_search({
-                      "sections-result" => {
-                        "count" => 9,
-                      },
-                    },
-                    {
-                      "queries" => {
-                        "sections-result" => {
-                          "source" => "Sections",
-                          "output" => {
-                            "elements" => [
-                              "count",
-                            ],
-                          },
-                        },
-                      },
-                    })
-    end
-
-    def test_elapsed_time
-      assert_search({
-                      "sections-result" => {
-                        "startTime" => start_time,
-                        "elapsedTime" => elapsed_time,
-                      },
-                    },
-                    {
-                      "queries" => {
-                        "sections-result" => {
-                          "source" => "Sections",
-                          "output" => {
-                            "elements" => [
-                              "startTime",
-                              "elapsedTime",
-                            ],
-                          },
-                        },
-                      },
-                    })
-    end
-
-    class AttributesTest < self
-      def test_source_only
-        expected = {
-          "sections-result" => {
-            "records" => [
-              {
-                "_key" => "1.1",
-                "title" => "Groonga overview",
-              },
-              {
-                "_key" => "1.2",
-                "title" => "Full text search and Instant update",
-              },
-              {
-                "_key" => "1.3",
-                "title" => "Column store and aggregate query",
-              },
-            ],
-          },
-        }
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "format" => "complex",
-                "limit" => 3,
-                "attributes" => ["_key", "title"],
-              },
-            },
-          },
-        }
-        assert_search(expected, request)
-      end
-
-      def test_label
-        expected = {
-          "sections-result" => {
-            "records" => [
-              {
-                "key" => "1.1",
-                "title" => "Groonga overview",
-              },
-              {
-                "key" => "1.2",
-                "title" => "Full text search and Instant update",
-              },
-              {
-                "key" => "1.3",
-                "title" => "Column store and aggregate query",
-              },
-            ],
-          },
-        }
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "format" => "complex",
-                "limit" => 3,
-                "attributes" => [
-                  {
-                    "label" => "key",
-                    "source" => "_key",
-                  },
-                  "title",
-                ],
-              },
-            },
-          },
-        }
-        assert_search(expected, request)
-      end
-
-      def test_static_value
-        expected = {
-          "sections-result" => {
-            "records" => [
-              {
-                "single_quote_string" => "string value",
-                "double_quote_string" => "string value",
-                "integer" => 29,
-                "complex_negative_number" => -29.29,
-              },
-              {
-                "single_quote_string" => "string value",
-                "double_quote_string" => "string value",
-                "integer" => 29,
-                "complex_negative_number" => -29.29,
-              },
-            ],
-          },
-        }
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "format" => "complex",
-                "limit" => 2,
-                "attributes" => [
-                  {
-                    "label" => "single_quote_string",
-                    "source" => "'string value'",
-                  },
-                  {
-                    "label" => "double_quote_string",
-                    "source" => '"string value"',
-                  },
-                  {
-                    "label" => "integer",
-                    "source" => "29",
-                  },
-                  {
-                    "label" => "complex_negative_number",
-                    "source" => "-29.29",
-                  },
-                ],
-              },
-            },
-          },
-        }
-        assert_search(expected, request)
-      end
-
-      def test_expression
-        expected = {
-          "sections-result" => {
-            "records" => [
-              {
-                "formatted title" => "<Groonga overview>",
-                "title" => "Groonga overview",
-              },
-            ],
-          },
-        }
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "format" => "complex",
-                "limit" => 1,
-                "attributes" => [
-                  "title",
-                  {
-                    "label" => "formatted title",
-                    "source" => "'<' + title + '>'",
-                  },
-                ],
-              },
-            },
-          },
-        }
-        assert_search(expected, request)
-      end
-
-      def test_snippet_html
-        expected = {
-          "sections-result" => {
-            "records" => [
-              {
-                "title" => "Groonga overview",
-                "snippet" => [
-                  "<span class=\"keyword\">Groonga</span> overview",
-                ],
-              },
-            ],
-          },
-        }
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "condition" => {
-                "query" => "Groonga",
-                "matchTo" => ["title"],
-              },
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "format" => "complex",
-                "limit" => 1,
-                "attributes" => [
-                  "title",
-                  {
-                    "label" => "snippet",
-                    "source" => "snippet_html(title)",
-                  },
-                ],
-              },
-            },
-          },
-        }
-        assert_search(expected, request)
-      end
-    end
-
-    class FormatTest < self
-      def test_complex
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "format" => "complex",
-                "limit" => 3,
-                "attributes" => ["_key", "title"],
-              },
-            },
-          },
-        }
-        assert_search(complex_result, request)
-      end
-
-      def test_simple
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "format" => "simple",
-                "limit" => 3,
-                "attributes" => ["_key", "title"],
-              },
-            },
-          },
-        }
-        assert_search(simple_result, request)
-      end
-
-      def test_default
-        request = {
-          "queries" => {
-            "sections-result" => {
-              "source" => "Sections",
-              "output" => {
-                "elements" => [
-                  "records",
-                ],
-                "limit" => 3,
-                "attributes" => ["_key", "title"],
-              },
-            },
-          },
-        }
-        assert_search(simple_result, request)
-      end
-
-      def complex_result
-        {
-          "sections-result" => {
-            "records" => [
-              {
-                "_key" => "1.1",
-                "title" => "Groonga overview",
-              },
-              {
-                "_key" => "1.2",
-                "title" => "Full text search and Instant update",
-              },
-              {
-                "_key" => "1.3",
-                "title" => "Column store and aggregate query",
-              },
-            ],
-          },
-        }
-      end
-
-      def simple_result
-        {
-          "sections-result" => {
-            "records" => [
-              ["1.1", "Groonga overview"],
-              ["1.2", "Full text search and Instant update"],
-              ["1.3", "Column store and aggregate query"],
-            ],
-          },
-        }
-      end
-    end
-  end
-end