droonga-engine 1.0.1

data/lib/droonga/plugins/watch.rb

@@ -0,0 +1,178 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+require "droonga/watcher"
+require "droonga/sweeper"
+require "droonga/watch_schema"
+
+module Droonga
+  module Plugins
+    module Watch
+      extend Plugin
+      register("watch")
+
+      module SchemaCreatable
+        private
+        def ensure_schema_created
+          # XXX just workaround. This must be re-written.
+          # When secondary and later processes opens the database,
+          # creation processes of tables by the first process is
+          # not finished yet. Then secondary and others tries to
+          # create tables and raises errors. To avoid such a problem,
+          # the creation processes of tables is disabled on workers.
+          if $0 !~ /\AServer/
+            schema = WatchSchema.new(@context)
+            schema.ensure_created
+          else
+            until @context["Keyword"]
+              sleep 0.1
+            end
+            sleep 1
+          end
+        end
+      end
+
+      module MessageParsable
+        private
+        def parse_message(message)
+          request = message.request
+          subscriber = request["subscriber"]
+          condition = request["condition"]
+          route = request["route"] || message["from"]
+          if condition
+            query = condition.to_json
+          else
+            query = nilondition
+          end
+          [subscriber, condition, query, route]
+        end
+      end
+
+      class SubscribeHandler < Droonga::Handler
+        include SchemaCreatable
+        include MessageParsable
+
+        def initialize(*args)
+          super
+          ensure_schema_created # TODO: REMOVE ME
+        end
+
+        def handle(message)
+          subscriber, condition, query, route = parse_message(message)
+          normalized_request = {
+            :subscriber => subscriber,
+            :condition  => condition,
+            :query      => query,
+            :route      => route,
+          }
+          watcher = Watcher.new(@context)
+          watcher.subscribe(normalized_request)
+          true
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "watch.subscribe"
+        step.write = true
+        step.handler = SubscribeHandler
+        step.collector = Collectors::And
+      end
+
+      class UnsubscribeHandler < Droonga::Handler
+        include SchemaCreatable
+        include MessageParsable
+
+        def initialize(*args)
+          super
+          ensure_schema_created # TODO: REMOVE ME
+        end
+
+        def handle(message)
+          subscriber, condition, query, route = parse_message(message)
+          _ = route # suppress a warning
+          normalized_request = {
+            :subscriber => subscriber,
+            :condition  => condition,
+            :query      => query,
+          }
+          watcher = Watcher.new(@context)
+          watcher.unsubscribe(normalized_request)
+          true
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "watch.unsubscribe"
+        step.write = true
+        step.handler = UnsubscribeHandler
+        step.collector = Collectors::And
+      end
+
+      class FeedHandler < Droonga::Handler
+        include SchemaCreatable
+
+        def initialize(*args)
+          super
+          ensure_schema_created # TODO: REMOVE ME
+        end
+
+        def handle(message)
+          request = message.request
+          watcher = Watcher.new(@context)
+          watcher.feed(:targets => request["targets"]) do |route, subscribers|
+            published_message = {
+              "to"   => subscribers,
+              "body" => request,
+            }
+            published_message = message.raw.merge(published_message)
+            messenger.forward(published_message,
+                              "to" => route, "type" => "watch.publish")
+          end
+          nil
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "watch.feed"
+        step.write = true
+        step.handler = FeedHandler
+      end
+
+      class SweepHandler < Droonga::Handler
+        include SchemaCreatable
+
+        message.type = "watch.sweep"
+
+        def initialize(*args)
+          super
+          ensure_schema_created # TODO: REMOVE ME
+        end
+
+        def handle(message)
+          sweeper = Sweeper.new(@context)
+          sweeper.sweep_expired_subscribers
+          nil
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "watch.sweep"
+        step.write = true
+        step.handler = SweepHandler
+      end
+    end
+  end
+end

data/lib/droonga/processor.rb

@@ -0,0 +1,63 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+require "droonga/handler_runner"
+
+module Droonga
+  class Processor
+    include Loggable
+
+    def initialize(loop, job_pusher, options={})
+      @loop = loop
+      @job_pusher = job_pusher
+      @options = options
+      @n_workers = @options[:n_workers] || 0
+    end
+
+    def start
+      @handler_runner = HandlerRunner.new(@loop, @options)
+      @handler_runner.start
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      @handler_runner.shutdown
+      logger.trace("shutdown: done")
+    end
+
+    def process(message)
+      logger.trace("process: start")
+      type = message["type"]
+      if @handler_runner.processable?(type)
+        logger.trace("process: handlable: #{type}")
+        synchronous = @handler_runner.prefer_synchronous?(type)
+        if @n_workers.zero? or synchronous
+          @handler_runner.process(message)
+        else
+          @job_pusher.push(message)
+        end
+      else
+        logger.trace("process: ignore #{type}")
+      end
+      logger.trace("process: done")
+    end
+
+    private
+    def log_tag
+      "processor"
+    end
+  end
+end

data/lib/droonga/reducer.rb

@@ -0,0 +1,169 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Reducer
+    class << self
+      # TODO: This is right location?
+      def apply_range(items, range)
+        if items and items.is_a?(Array)
+          offset = range["offset"] || 0
+          unless offset.zero?
+            items = items[offset..-1] || []
+          end
+
+          limit = range["limit"] || 0
+          unless limit == UNLIMITED
+            items = items[0...limit]
+          end
+        end
+        items
+      end
+    end
+
+    # XXX: We has ULIMITED defined
+    # lib/droonga/plugins/search/distributed_search_planner.rb. We
+    # should unify it.
+    UNLIMITED = -1
+
+    def initialize(deal)
+      @deal = deal # TODO: deal is good name?
+    end
+
+    def reduce(left_value, right_value)
+      if left_value.nil? or right_value.nil?
+        return right_value || left_value
+      end
+
+      reduced_value = nil
+
+      case @deal["type"]
+      when "and"
+        reduced_value = (left_value and right_value)
+      when "or"
+        reduced_value = (left_value or right_value)
+      when "sum"
+        reduced_value = sum(left_value, right_value)
+        reduced_value = self.class.apply_range(reduced_value,
+                                               "limit" => @deal["limit"])
+      when "average"
+        reduced_value = (left_value.to_f + right_value.to_f) / 2
+      when "sort"
+        reduced_value = merge(left_value,
+                              right_value,
+                              :operators => @deal["operators"],
+                              :key_column => @deal["key_column"])
+        reduced_value = self.class.apply_range(reduced_value,
+                                               "limit" => @deal["limit"])
+      end
+
+      reduced_value
+    end
+
+    private
+    def sum(x, y)
+      return x || y if x.nil? or y.nil?
+
+      if x.is_a?(Hash) and y.is_a?(Hash)
+        x.merge(y)
+      else
+        x + y
+      end
+    end
+
+    def merge(x, y, options={})
+      operators = options[:operators] = normalize_operators(options[:operators])
+
+      unify_by_key!(x, y, options)
+
+      index = 0
+      y.each do |_y|
+        loop do
+          _x = x[index]
+          break unless _x
+          break if compare(_y, _x, operators)
+          index += 1
+        end
+        x.insert(index, _y)
+        index += 1
+      end
+      return x
+    end
+
+    def normalize_operators(operators)
+      operators ||= []
+      operators.collect do |operator|
+        if operator.is_a?(String)
+          { "operator" => operator }
+        else
+          operator
+        end
+      end
+    end
+
+    def compare(x, y, operators)
+      operators.each_with_index do |operator, index|
+        column = operator["column"] || index
+        operator = operator["operator"]
+        _x = x[column]
+        _y = y[column]
+        return true if _x.__send__(operator, _y)
+      end
+      return false
+    end
+
+    def unify_by_key!(base_items, unified_items, options={})
+      key_column_index = options[:key_column]
+      return unless key_column_index
+
+      # The unified records must be smaller than the base, because
+      # I sort unified records at last. I want to sort only smaller array.
+      if base_items.size < unified_items.size
+        base_items, unified_items = unified_items, base_items
+      end
+
+      unified_key_map = {}
+      unified_items.each do |unified_item|
+        unified_key_map[unified_item[key_column_index]] = unified_item
+      end
+
+      unified = false
+      base_items.reject! do |base_item|
+        key = base_item[key_column_index]
+        unified_item = unified_key_map[key]
+        if unified_item
+          base_item.each_with_index do |value, column_index|
+            next if column_index == key_column_index
+            unified_item[column_index] += value
+          end
+          unified = true
+          true
+        else
+          false
+        end
+      end
+
+      if unified
+        unified_items.sort! do |a, b|
+          if compare(a, b, options[:operators])
+            -1
+          else
+            1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/replier.rb

@@ -0,0 +1,49 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+require "droonga/status_code"
+
+module Droonga
+  class Replier
+    include Loggable
+
+    def initialize(forwarder)
+      @forwarder = forwarder
+    end
+
+    def reply(message)
+      logger.trace("reply: start")
+      destination = message["replyTo"]
+      reply_message = {
+        "inReplyTo"  => message["id"],
+        "statusCode" => message["statusCode"] || StatusCode::OK,
+        "type"       => destination["type"],
+        "body"       => message["body"],
+      }
+      if message.include?("errors")
+        errors = message["errors"]
+        reply_message["errors"] = errors unless errors.empty?
+      end
+      @forwarder.forward(reply_message, destination)
+      logger.trace("reply: done")
+    end
+
+    private
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] replier"
+    end
+  end
+end

data/lib/droonga/schema_applier.rb

@@ -0,0 +1,167 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+
+module Droonga
+  class SchemaApplier
+    include Loggable
+
+    def initialize(context, schema)
+      @context = context
+      @schema = schema
+    end
+
+    def apply
+      # TODO: Support migration
+      Groonga::Schema.define(:context => @context) do |schema|
+        create_tables(schema)
+        create_reference_columns(schema)
+        create_index_columns(schema)
+      end
+    end
+
+    private
+    def each_table
+      reference_tables = []
+      @schema.tables.each_value do |table|
+        if reference_table?(table)
+          reference_tables << table
+        else
+          yield(table)
+        end
+      end
+      reference_tables.each do |table|
+        yield(table)
+      end
+    end
+
+    def reference_table?(table)
+      table.type != "Array" and !type_names.include?(table.key_type_groonga)
+    end
+
+    def each_column(table)
+      table.columns.each_value do |column|
+        yield(column)
+      end
+    end
+
+    def normal_column?(column)
+      column.type != "Index" and type_names.include?(column.value_type_groonga)
+    end
+
+    def reference_column?(column)
+      column.type != "Index" and !type_names.include?(column.value_type_groonga)
+    end
+
+    def index_column?(column)
+      column.type == "Index"
+    end
+
+    def types
+      @types ||= collect_available_types
+    end
+
+    def type_names
+      @type_names ||= types.collect(&:name)
+    end
+
+    def collect_available_types
+      each_options = {
+        :ignore_missing_object => true
+      }
+      @context.database.each(each_options).find_all do |object|
+        object.is_a?(Groonga::Type)
+      end
+    end
+
+    def create_tables(schema)
+      each_table do |table|
+        create_table(schema, table)
+      end
+    end
+
+    def create_table(schema, table)
+      options = {
+        :type => table.type_symbol,
+        :key_type => table.key_type_groonga,
+        :default_tokenizer => table.tokenizer,
+        :normalizer => table.normalizer,
+      }
+      schema.create_table(table.name, options) do |table_definition|
+        each_column(table) do |column|
+          next unless normal_column?(column)
+          create_data_column(table_definition, column)
+        end
+      end
+    end
+
+    def create_data_column(table_definition, column)
+      options = {
+        :type => column.type_symbol,
+      }
+      if options[:type] == :vector
+        options[:with_weight] = true if column.vector_options.weight
+      end
+      table_definition.column(column.name, column.value_type_groonga, options)
+    end
+
+    def create_reference_columns(schema)
+      each_table do |table|
+        reference_columns = []
+        each_column(table) do |column|
+          reference_columns << column if reference_column?(column)
+        end
+        next if reference_columns.empty?
+
+        schema.change_table(table.name) do |table_definition|
+          reference_columns.each do |column|
+            create_data_column(table_definition, column)
+          end
+        end
+      end
+    end
+
+    def create_index_columns(schema)
+      each_table do |table|
+        index_columns = []
+        each_column(table) do |column|
+          index_columns << column if index_column?(column)
+        end
+        next if index_columns.empty?
+
+        schema.change_table(table.name) do |table_definition|
+          index_columns.each do |column|
+            create_index_column(table_definition, column)
+          end
+        end
+      end
+    end
+
+    def create_index_column(table_definition, column)
+      sources = column.index_options.sources || []
+      options = {
+        :with_section  => column.index_options.section,
+        :with_weight   => column.index_options.weight,
+        :with_position => column.index_options.position,
+      }
+      table_definition.index(column.value_type_groonga, *sources, options)
+    end
+
+    def log_tag
+      "schema_applier"
+    end
+  end
+end

data/lib/droonga/searcher/mecab_filter.rb

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "MeCab"
+
+module Droonga
+  class Searcher
+    class QuerySearcher
+      class MeCabTokenizer
+        def initialize
+          @mecab = MeCab::Tagger.new("-Owakati")
+        end
+
+        def tokenize(text)
+          tokens = @mecab.parse(text).force_encoding("utf-8").split(/\s+/)
+          tokens.reject do |token|
+            token.empty?
+          end
+        end
+      end
+
+      def apply_mecab_filter(condition)
+        return unless condition.is_a?(Hash)
+        return unless condition["useMeCabFilter"]
+        query = condition["query"]
+        return if query.nil?
+        match_columns = condition["matchTo"]
+        return unless match_columns.is_a?(Array)
+        return if match_columns.size != 1
+        match_column = match_columns.first
+
+        tokenizer = MeCabTokenizer.new
+
+        @records.open_cursor do |cursor|
+          count = 0
+          cursor.each do |record|
+            match_target = record[match_column]
+            body_terms = tokenizer.tokenize(match_target)
+            unless body_terms.include?(query)
+              record.delete
+            end
+          end
+        end
+      end
+
+      alias_method :original_apply_condition!, :apply_condition!
+      def apply_condition!(condition)
+        original_apply_condition!(condition)
+        apply_mecab_filter(condition)
+      end
+    end
+  end
+end