fluent-plugin-droonga 0.9.0 → 0.9.9

data/lib/droonga/processor.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,7 +13,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/handler"
+require "droonga/handler_runner"
 
 module Droonga
   class Processor
@@ -27,24 +25,24 @@ module Droonga
     end
 
     def start
-      @handler = Handler.new(@loop, @options)
-      @handler.start
+      @handler_runner = HandlerRunner.new(@loop, @options)
+      @handler_runner.start
     end
 
     def shutdown
       $log.trace("#{log_tag}: shutdown: start")
-      @handler.shutdown
+      @handler_runner.shutdown
       $log.trace("#{log_tag}: shutdown: done")
     end
 
     def process(message)
       $log.trace("#{log_tag}: process: start")
       command = message["type"]
-      if @handler.processable?(command)
+      if @handler_runner.processable?(command)
         $log.trace("#{log_tag}: process: handlable: #{command}")
-        synchronous = @handler.prefer_synchronous?(command)
+        synchronous = @handler_runner.prefer_synchronous?(command)
         if @n_workers.zero? or synchronous
-          @handler.process(message)
+          @handler_runner.process(message)
         else
           @message_pusher.push(message)
         end

data/lib/droonga/searcher.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,9 +20,16 @@ require "tsort"
 require "groonga"
 
 require "droonga/time_formatter"
+require "droonga/message_processing_error"
 
 module Droonga
   class Searcher
+    class NoQuery < BadRequest
+      def initialize
+        super("You must specify one or more query.")
+      end
+    end
+
     class MissingSourceParameter < BadRequest
       def initialize(query, queries)
         super("The query #{query.inspect} has no source. " +
@@ -63,9 +70,8 @@ module Droonga
     private
     def process_queries(queries)
       $log.trace("#{log_tag}: process_queries: start")
-      unless queries
-        $log.trace("#{log_tag}: process_queries: done")
-        return {}
+      if queries.nil? || queries.empty?
+        raise NoQuery.new
       end
       $log.trace("#{log_tag}: process_queries: sort: start")
       sorted_queries = QuerySorter.sort(queries)
@@ -640,3 +646,7 @@ module Droonga
     end
   end
 end
+
+if ENV["DROONGA_ENABLE_SEARCH_MECAB_FILTER"] == "yes"
+  require "droonga/searcher/mecab_filter"
+end

data/lib/droonga/searcher/mecab_filter.rb

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "MeCab"
+
+module Droonga
+  class Searcher
+    class QuerySearcher
+      class MeCabTokenizer
+        def initialize
+          @mecab = MeCab::Tagger.new("-Owakati")
+        end
+
+        def tokenize(text)
+          tokens = @mecab.parse(text).force_encoding("utf-8").split(/\s+/)
+          tokens.reject do |token|
+            token.empty?
+          end
+        end
+      end
+
+      def apply_mecab_filter(condition)
+        return unless condition.is_a?(Hash)
+        return unless condition["useMeCabFilter"]
+        query = condition["query"]
+        return if query.nil?
+        match_columns = condition["matchTo"]
+        return unless match_columns.is_a?(Array)
+        return if match_columns.size != 1
+        match_column = match_columns.first
+
+        tokenizer = MeCabTokenizer.new
+
+        @records.open_cursor do |cursor|
+          count = 0
+          cursor.each do |record|
+            match_target = record[match_column]
+            body_terms = tokenizer.tokenize(match_target)
+            unless body_terms.include?(query)
+              record.delete
+            end
+          end
+        end
+      end
+
+      alias_method :original_apply_condition!, :apply_condition!
+      def apply_condition!(condition)
+        original_apply_condition!(condition)
+        apply_mecab_filter(condition)
+      end
+    end
+  end
+end

data/lib/droonga/session.rb

@@ -48,10 +48,10 @@ module Droonga
       end
       tasks.each do |task|
         task["n_of_inputs"] += 1
-        component = task["component"]
-        type = component["type"]
+        step = task["step"]
+        type = step["type"]
         command = "collector_" + type
-        n_of_expects = component["n_of_expects"]
+        n_of_expects = step["n_of_expects"]
         message = {
           "task"=>task,
           "name"=>name,
@@ -61,9 +61,9 @@ module Droonga
         return if task["n_of_inputs"] < n_of_expects
         #the task is done
         result = task["values"]
-        post = component["post"]
+        post = step["post"]
         @dispatcher.reply("body" => result) if post
-        component["descendants"].each do |name, routes|
+        step["descendants"].each do |name, routes|
           message = {
             "id" => @id,
             "input" => name,

data/lib/droonga/test.rb

@@ -15,7 +15,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/test/stub_distributor"
+require "droonga/test/stub_planner"
 require "droonga/test/stub_handler"
 require "droonga/test/stub_handler_message"
 require "droonga/test/stub_handler_messenger"

data/lib/droonga/test/stub_handler_message.rb

@@ -22,7 +22,7 @@ module Droonga
         raw = {
           "body" => {
             "task" => {
-              "component" => {
+              "step" => {
                 "body" => request,
               },
             },

data/lib/droonga/test/{stub_distributor.rb → stub_planner.rb}

@@ -17,7 +17,7 @@
 
 module Droonga
   module Test
-    class StubDistributor
+    class StubPlanner
       attr_reader :messages
       def initialize
         @messages = []

data/lib/droonga/worker.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,14 +14,15 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 require "droonga/event_loop"
-require "droonga/handler"
+require "droonga/handler_runner"
 require "droonga/message_receiver"
 
 module Droonga
   module Worker
     def initialize
       @loop = EventLoop.new
-      @handler = Handler.new(@loop, config.merge(:dispatcher => nil))
+      @handler_runner = HandlerRunner.new(@loop,
+                                          config.merge(:dispatcher => nil))
       receiver_socket = config[:message_receiver]
       @message_receiver = MessageReceiver.new(@loop, receiver_socket) do |message|
         process(message)
@@ -32,10 +31,10 @@ module Droonga
 
     def run
       $log.trace("#{log_tag}: run: start")
-      @handler.start
+      @handler_runner.start
       @message_receiver.start
       @loop.run
-      @handler.shutdown
+      @handler_runner.shutdown
       $log.trace("#{log_tag}: run: done")
     end
 
@@ -49,7 +48,7 @@ module Droonga
     private
     def process(message)
       $log.trace("#{log_tag}: process: start")
-      @handler.process(message)
+      @handler_runner.process(message)
       $log.trace("#{log_tag}: process: done")
     end
 

data/lib/fluent/plugin/out_droonga.rb

@@ -60,7 +60,6 @@ module Fluent
           "body" => record
         }
       end
-      message["via"] ||= []
       reply_to = message["replyTo"]
       if reply_to.is_a? String
         message["replyTo"] = {

data/sample/cluster/catalog.json

@@ -1,4 +1,5 @@
 {
+  "version": 1,
   "effective_date": "2013-09-01T00:00:00Z",
   "zones": ["localhost:23003/droonga"],
   "farms": {
@@ -10,7 +11,7 @@
   "datasets": {
     "Droonga": {
       "workers": 2,
-      "plugins": ["search", "groonga", "add"],
+      "plugins": ["groonga", "search", "crud"],
       "number_of_replicas": 2,
       "number_of_partitions": 2,
       "partition_key": "_key",
@@ -36,8 +37,5 @@
         }
       }
     }
-  },
-  "options": {
-    "plugins": ["groonga"]
   }
 }

data/sample/mecab_filter/data.grn

@@ -0,0 +1,7 @@
+load --table Memo
+[
+{"_key": "1", "content": "アンパサンド"},
+{"_key": "2", "content": "ジャパン"},
+{"_key": "3", "content": "食パン"},
+{"_key": "4", "content": "フランスのパン"}
+]

data/sample/mecab_filter/ddl.grn

@@ -0,0 +1,7 @@
+table_create Memo TABLE_HASH_KEY ShortText
+column_create Memo content COLUMN_SCALAR ShortText
+
+table_create Terms TABLE_PAT_KEY ShortText \
+  --default_tokenizer TokenBigram \
+  --normalizer NormalizerAuto
+column_create Terms memo_context COLUMN_INDEX|WITH_POSITION Memo content

data/sample/mecab_filter/search_with_mecab_filter.json

@@ -0,0 +1,21 @@
+{
+    "type": "search",
+    "dataset": "Droonga",
+    "body": {
+	"queries": {
+	    "search": {
+		"source": "Memo",
+		"condition": {
+		    "matchTo": ["content"],
+		    "query": "パン",
+		    "useMeCabFilter": true
+		},
+		"output": {
+		    "elements": ["count", "records"],
+		    "attributes": ["_key", "content"],
+		    "limit": 10
+		}
+	    }
+	}
+    }
+}

data/sample/mecab_filter/search_without_mecab_filter.json

@@ -0,0 +1,21 @@
+{
+    "type": "search",
+    "dataset": "Droonga",
+    "body": {
+	"queries": {
+	    "search": {
+		"source": "Memo",
+		"condition": {
+		    "matchTo": ["content"],
+		    "query": "パン",
+		    "useMeCabFilter": false
+		},
+		"output": {
+		    "elements": ["count", "records"],
+		    "attributes": ["_key", "content"],
+		    "limit": 10
+		}
+	    }
+	}
+    }
+}

data/test/command/config/default/catalog.json

@@ -10,7 +10,7 @@
   "datasets": {
     "Droonga": {
       "workers": 2,
-      "plugins": ["search", "groonga", "add"],
+      "plugins": ["groonga", "crud", "search"],
       "number_of_replicas": 2,
       "number_of_partitions": 2,
       "partition_key": "_key",
@@ -38,7 +38,7 @@
     },
     "Watch": {
       "workers": 2,
-      "plugins": ["search", "groonga", "add", "watch"],
+      "plugins": ["groonga", "watch", "search", "crud"],
       "number_of_replicas": 1,
       "number_of_partitions": 1,
       "partition_key": "_key",
@@ -54,8 +54,5 @@
         }
       }
     }
-  },
-  "options": {
-    "plugins": ["crud", "groonga"]
   }
 }

data/test/command/suite/search/error/no-query.expected

@@ -0,0 +1,13 @@
+[
+  "droonga.message",
+  0,
+  {
+    "inReplyTo": "request-id",
+    "statusCode": 400,
+    "type": "search.result",
+    "body": {
+      "name": "NoQuery",
+      "message": "You must specify one or more query."
+    }
+  }
+]

data/test/command/suite/search/error/no-query.test

@@ -0,0 +1,7 @@
+#@include fixture/user-table.jsons
+{
+  "type": "search",
+  "dataset": "Droonga",
+  "body": {
+  }
+}

data/test/command/suite/search/error/unknown-source.expected

@@ -13,6 +13,32 @@
           "source": "unknown"
         }
       }
+    },
+    "errors": {
+      "sources0": {
+        "statusCode": 404,
+        "body": {
+          "name": "UnknownSource",
+          "message": "The source \"unknown\" does not exist. It must be a name of an existing table or another query.",
+          "detail": {
+            "unknown-source": {
+              "source": "unknown"
+            }
+          }
+        }
+      },
+      "sources1": {
+        "statusCode": 404,
+        "body": {
+          "name": "UnknownSource",
+          "message": "The source \"unknown\" does not exist. It must be a name of an existing table or another query.",
+          "detail": {
+            "unknown-source": {
+              "source": "unknown"
+            }
+          }
+        }
+      }
     }
   }
 ]