droonga-engine 1.0.1

data/lib/droonga/plugin_loader.rb

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "pathname"
+
+require "droonga/loggable"
+
+module Droonga
+  class PluginLoader
+    include Loggable
+
+    class << self
+      def load_all
+        loader = new
+        loader.load_all
+      end
+    end
+
+    def initialize
+    end
+
+    def load(name)
+      logger.debug("loading...: <#{name}>")
+      path = "droonga/plugins/#{name}"
+      begin
+        require path
+      rescue StandardError, SyntaxError => error
+        logger.exception("failed to load: <#{path}>", error)
+        raise
+      end
+    end
+
+    def load_all
+      $LOAD_PATH.each do |load_path|
+        search_pattern = "#{load_path}/droonga/plugins/*.rb"
+        logger.debug("searching...: <#{search_pattern}>")
+        Pathname.glob(search_pattern) do |plugin_path|
+          name = Pathname(plugin_path).basename(".rb").to_s
+          load(name)
+        end
+      end
+    end
+
+    private
+    def log_tag
+      "plugin-loader"
+    end
+  end
+end

data/lib/droonga/plugin_registry.rb

@@ -0,0 +1,66 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class PluginRegistry
+    include Enumerable
+
+    class UnknownPlugin < Error
+      def initialize(name, klass)
+        super("[#{klass}] Plugin not found: <#{name}>")
+      end
+    end
+
+    def initialize
+      @plugins = {}
+    end
+
+    def each(&block)
+      @plugins.each(&block)
+    end
+
+    def register(name, plugin_module)
+      @plugins[name] = plugin_module
+    end
+
+    def [](name)
+      @plugins[name]
+    end
+
+    def clear
+      @plugins.clear
+    end
+
+    def find_sub_classes(name, klass)
+      plugin_module = self[name]
+      if plugin_module.nil?
+        raise UnknownPlugin.new(name, klass)
+      end
+      sub_classes = []
+      collect_sub_classes_recursive(plugin_module, klass, sub_classes)
+      sub_classes
+    end
+
+    private
+    def collect_sub_classes_recursive(base, klass, sub_classes)
+      base.constants.each do |constant_name|
+        constant = base.const_get(constant_name)
+        next unless constant.is_a?(Module)
+        sub_classes << constant if constant < klass
+        collect_sub_classes_recursive(constant, klass, sub_classes)
+      end
+    end
+  end
+end

data/lib/droonga/plugins/basic.rb

@@ -0,0 +1,54 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+require "droonga/reducer"
+
+module Droonga
+  module Plugins
+    module Basic
+      extend Plugin
+      register("basic")
+
+      class GatherCollector < Droonga::Collector
+        message.pattern = ["task.step.type", :equal, "gather"]
+
+        def collect(message)
+          output = message.input || message.name
+          if output.is_a?(Hash)
+            output_name = output["output"]
+          else
+            output_name = output
+          end
+          message.values[output_name] = message.value
+        end
+      end
+
+      class ReduceCollector < Droonga::Collector
+        message.pattern = ["task.step.type", :equal, "reduce"]
+
+        def collect(message)
+          message.input.each do |output_name, deal|
+            left_value = message.values[output_name]
+            right_value = message.value
+            reducer = Reducer.new(deal)
+            value = reducer.reduce(left_value, right_value)
+            message.values[output_name] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/plugins/crud.rb

@@ -0,0 +1,145 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga"
+
+require "droonga/plugin"
+require "droonga/error_messages"
+
+module Droonga
+  module Plugins
+    module CRUD
+      extend Plugin
+      register("crud")
+
+      class Adapter < Droonga::Adapter
+        input_message.pattern  = ["type", :equal, "add"]
+        output_message.pattern = ["body.success", :exist]
+
+        def adapt_input(input_message)
+          request = input_message.body
+          key = request["key"] || rand.to_s
+          values = request["values"] || {}
+          request["filter"] = values.merge("_key" => key)
+        end
+
+        def adapt_output(output_message)
+          if output_message.errors
+            detail = output_message.body["detail"]
+            return if detail.nil?
+            detail.delete("filter")
+            output_message.errors.each do |path, error|
+              error["body"]["detail"].delete("filter")
+            end
+          else
+            output_message.body.delete("filter")
+          end
+        end
+      end
+
+      class Handler < Droonga::Handler
+        class MissingTableParameter < BadRequest
+          def initialize
+            super("\"table\" must be specified.")
+          end
+        end
+
+        class MissingPrimaryKeyParameter < BadRequest
+          def initialize(table_name)
+            super("\"key\" must be specified. " +
+                    "The table #{table_name.inspect} requires a primary key for a new record.")
+          end
+        end
+
+        class UnknownTable < NotFound
+          def initialize(table_name)
+            super("The table #{table_name.inspect} does not exist in the dataset.")
+          end
+        end
+
+        class InvalidValue < BadRequest
+          def initialize(column, value, request)
+            super("The column #{column.inspect} cannot store the value #{value.inspect}.",
+                  request)
+          end
+        end
+
+        class UnknownColumn < NotFound
+          def initialize(column, table, request)
+            super("The column #{column.inspect} does not exist in the table #{table.inspect}.",
+                  request)
+          end
+        end
+
+        def handle(message)
+          process_add(message.request)
+        end
+
+        private
+        def process_add(request)
+          raise MissingTableParameter.new unless request.include?("table")
+
+          table = @context[request["table"]]
+          raise UnknownTable.new(request["table"]) unless table
+
+          if table.support_key?
+            unless request.include?("key")
+              raise MissingPrimaryKeyParameter.new(request["table"])
+            end
+          end
+
+          add_record(table, request)
+          true
+        end
+
+        def add_record(table, request)
+          record = nil
+          if table.support_key?
+            record = table.add(request["key"])
+          else
+            record = table.add
+          end
+          (request["values"] || []).each do |column, value|
+            begin
+              record[column] = value
+            rescue ::Groonga::InvalidArgument => error
+              record.delete if record.added?
+              raise InvalidValue.new(column, value, request)
+            rescue ArgumentError => error
+              record.delete if record.added?
+              raise InvalidValue.new(column, value, request)
+            rescue ::Groonga::NoSuchColumn => error
+              record.delete if record.added?
+              raise UnknownColumn.new(column, request["table"], request)
+            end
+          end
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "add"
+        step.inputs = {
+          "table" => {
+            :type => :table,
+            :filter => "filter",
+          },
+        }
+        step.write = true
+        step.handler = Handler
+        step.collector = Collectors::And
+      end
+    end
+  end
+end

data/lib/droonga/plugins/dump.rb

@@ -0,0 +1,97 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga"
+
+require "droonga/plugin"
+require "droonga/error_messages"
+
+module Droonga
+  module Plugins
+    module Dump
+      extend Plugin
+      register("dump")
+
+      class Handler < Droonga::Handler
+        def handle(message)
+          replyTo = (message.raw["replyTo"] || {})["to"]
+          return false unless replyTo
+
+          request = message.request || {}
+          messages_per_seconds = request["messagesPerSecond"] || 10000
+          messages_per_seconds = [10, messages_per_seconds.to_i].max
+          messages_per_100msec = messages_per_seconds / 10
+          dumper = Enumerator.new do |yielder|
+            n = 0
+            each_table do |table|
+              table.each do |record|
+                values = {}
+                record.attributes.each do |key, value|
+                  values[key] = value unless key.start_with?("_")
+                end
+                dump_message = {
+                  "dataset" => message.raw["dataset"],
+                  "body" => {
+                    "table" => table.name,
+                    "key" => record.key,
+                    "values" => values,
+                  },
+                }
+                messenger.forward(dump_message,
+                                  "to" => replyTo,
+                                  "type" => "dump.record")
+                n = (n + 1) % messages_per_100msec
+                yielder << nil if n.zero?
+              end
+            end
+          end
+
+          timer = Coolio::TimerWatcher.new(0.1, true)
+          timer.on_timer do
+            begin
+              dumper.next
+            rescue StopIteration
+              timer.detach
+            end
+          end
+          loop.attach(timer)
+
+          true
+        end
+
+        private
+        def each_table
+          @context.database.each(:ignore_missing_object => true) do |object|
+            next unless object.is_a?(::Groonga::Table)
+            next if index_only_table?(object)
+            yield(object)
+          end
+        end
+
+        def index_only_table?(table)
+          table.columns.all? do |column|
+            column.is_a?(::Groonga::IndexColumn)
+          end
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "dump"
+        step.handler = Handler
+        step.collector = Collectors::And
+      end
+    end
+  end
+end

data/lib/droonga/plugins/error.rb

@@ -0,0 +1,51 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+
+module Droonga
+  module Plugins
+    module Error
+      extend Plugin
+      register("error")
+
+      class Adapter < Droonga::Adapter
+        output_message.pattern = ["body.errors", :exist]
+
+        def adapt_output(output_message)
+          errors = output_message.body["errors"]
+          if errors and !errors.empty?
+            output_message.errors = errors
+
+            status_codes = []
+            errors.values.each do |error|
+              status_codes << error["statusCode"]
+            end
+            status_codes = status_codes.uniq
+            if status_codes.size == 1
+              output_message.status_code = status_codes.first
+            else
+              output_message.status_code = ErrorMessages::InternalServerError::STATUS_CODE
+            end
+
+            output_message.body = errors.values.first["body"]
+          else
+            output_message.body.delete("errors")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/plugins/groonga/column_create.rb

@@ -0,0 +1,123 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga/command/column-create"
+
+require "droonga/plugin"
+require "droonga/plugins/groonga/generic_command"
+
+module Droonga
+  module Plugins
+    module Groonga
+      module ColumnCreate
+        class Command < GenericCommand
+          def process_request(request)
+            command_class = ::Groonga::Command.find("column_create")
+            @command = command_class.new("column_create", request)
+
+            table_name = valid_table_name("table")
+
+            if @command.column_index?
+              define_index(table_name)
+            else
+              define_column(table_name)
+            end
+          end
+
+          private
+          def define_column(table_name)
+            column_name = @command["name"]
+            column_type = @command["type"]
+
+            options = create_column_options
+            ::Groonga::Schema.define(:context => @context) do |schema|
+              schema.change_table(table_name) do |table|
+                table.column(column_name, column_type, options)
+              end
+            end
+            true
+          end
+
+          def create_column_options
+            options = {}
+            create_column_options_flags(options)
+            options
+          end
+
+          def create_column_options_flags(options)
+            options[:type] = :scalar
+            if @command.column_scalar?
+              options[:type] = :scalar
+            elsif @command.column_vector?
+              options[:type] = :vector
+              if @command.with_weight?
+                options[:with_weight] = true
+              end
+            end
+            options
+          end
+
+          def define_index(table_name)
+            target_table = @command["type"]
+            target_columns = (@command["source"] || "").split(/\s*,\s*/)
+
+            options = create_index_options
+            ::Groonga::Schema.define(:context => @context) do |schema|
+              schema.change_table(table_name) do |table|
+                arguments = [target_table, *target_columns]
+                arguments << options
+                table.index(*arguments)
+              end
+            end
+            true
+          end
+
+          def create_index_options
+            options = {}
+            create_index_options_name(options)
+            create_index_options_flags(options)
+            options
+          end
+
+          def create_index_options_name(options)
+            options[:name] = @command["name"]
+          end
+
+          def create_index_options_flags(options)
+            options[:with_section] = true if @command.with_section?
+            options[:with_weight] = true if @command.with_weight?
+            options[:with_position] = true if @command.with_position?
+          end
+        end
+
+        class Handler < Droonga::Handler
+          action.synchronous = true
+
+          def handle(message)
+            command = Command.new(@context)
+            command.execute(message.request)
+          end
+        end
+
+        Groonga.define_single_step do |step|
+          step.name = "column_create"
+          step.write = true
+          step.handler = Handler
+          step.collector = Collectors::Or
+        end
+      end
+    end
+  end
+end