droonga-engine 1.0.1

data/lib/droonga/watch_schema.rb

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga"
+
+require "droonga/loggable"
+
+module Droonga
+  class WatchSchema
+    include Loggable
+
+    def initialize(context)
+      @context = context
+    end
+
+    def ensure_created
+      if @context["Keyword"]
+        logger.trace("skip table creation")
+        return
+      end
+      logger.trace("ensure_tables: start")
+      ensure_tables
+      logger.trace("ensure_tables: done")
+    end
+
+    private
+    def ensure_tables
+      Groonga::Schema.define(:context => @context) do |schema|
+        schema.create_table("Keyword",
+                            :type => :patricia_trie,
+                            :key_type => "ShortText",
+                            :key_normalize => true,
+                            :force => true) do |table|
+                            end
+
+        schema.create_table("Query",
+                            :type => :hash,
+                            :key_type => "ShortText",
+                            :force => true) do |table|
+                            end
+
+        schema.create_table("Route",
+                            :type => :hash,
+                            :key_type => "ShortText",
+                            :force => true) do |table|
+                            end
+
+        schema.create_table("Subscriber",
+                            :type => :hash,
+                            :key_type => "ShortText",
+                            :force => true) do |table|
+          table.time("last_modified")
+                            end
+
+        schema.change_table("Query") do |table|
+          table.reference("keywords", "Keyword", :type => :vector)
+        end
+
+        schema.change_table("Subscriber") do |table|
+          table.reference("route", "Route")
+          table.reference("subscriptions", "Query", :type => :vector)
+        end
+
+        schema.change_table("Keyword") do |table|
+          table.index("Query", "keywords", :name => "queries")
+        end
+
+        schema.change_table("Query") do |table|
+          table.index("Subscriber", "subscriptions", :name => "subscribers")
+        end
+      end
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] watch_schema"
+    end
+  end
+end

data/lib/droonga/watcher.rb

@@ -0,0 +1,257 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Watcher
+    EXACT_MATCH = false
+
+    def initialize(context)
+      @context = context
+
+      @subscriber_table = @context["Subscriber"]
+      @query_table      = @context["Query"]
+      @keyword_table    = @context["Keyword"]
+    end
+
+    def subscribe(request)
+      subscriber = request[:subscriber]
+      condition  = request[:condition]
+      query      = request[:query]
+      route      = request[:route]
+
+      # XXX better validation and error class must be written!!
+      if subscriber.nil? or subscriber.empty? or condition.nil? or
+         query.nil? or route.nil?
+        raise "invalid request"
+      end
+      raise "too long query" if query.size > 4095
+
+      query_record = @query_table[query]
+      unless query_record
+        keywords = pick_keywords([], condition)
+        query_record = @query_table.add(query, :keywords => keywords)
+      end
+      subscriber_record = @subscriber_table[subscriber]
+      if subscriber_record
+        subscriptions = subscriber_record.subscriptions
+        unless subscriptions.include?(query_record)
+          subscriptions << query_record
+          subscriber_record.subscriptions = subscriptions
+        end
+        subscriber_record.last_modified = Time.now
+      else
+        @subscriber_table.add(subscriber,
+                              :subscriptions => [query_record],
+                              :route => route,
+                              :last_modified => Time.now)
+      end
+    end
+
+    def unsubscribe(request)
+      subscriber = request[:subscriber]
+      query      = request[:query]
+
+      if subscriber.nil? or subscriber.empty?
+        raise "invalid request"
+      end
+
+      subscriber_record = @subscriber_table[subscriber]
+      return unless subscriber_record
+
+      if query.nil? or query.empty?
+        delete_subscriber(subscriber_record)
+      else
+        query_record = @query_table[query]
+        return unless query_record
+
+        subscriptions = subscriber_record.subscriptions
+        new_subscriptions = subscriptions.select do |subscription|
+          subscription != query_record
+        end
+
+        if new_subscriptions.empty?
+          delete_subscriber(subscriber_record)
+        else
+          subscriber_record.subscriptions = new_subscriptions
+          sweep_orphan_queries(subscriptions)
+        end
+      end
+    end
+
+    def feed(request, &block)
+      targets = request[:targets]
+
+      hits = []
+      targets.each do |key, target|
+        scan_body(hits, target)
+      end
+      hits.uniq! # hits may be duplicated if multiple targets are matched
+
+      publish(hits, request, &block)
+    end
+
+    def pick_keywords(memo, condition)
+      case condition
+      when Hash
+        memo << condition["query"]
+      when String
+        memo << condition
+      when Array
+        condition[1..-1].each do |element|
+          pick_keywords(memo, element)
+        end
+      end
+      memo
+    end
+
+    def scan_body(hits, body)
+      trimmed = body.strip
+      candidates = {}
+      # FIXME scan reports the longest keyword matched only
+      @keyword_table.scan(trimmed).each do |keyword, word, start, length|
+        @query_table.select do |query|
+          query.keywords =~ keyword
+        end.each do |record|
+          candidates[record.key] ||= []
+          candidates[record.key] << keyword
+        end
+      end
+      candidates.each do |query, keywords|
+        hits << query if query_match(query, keywords)
+      end
+    end
+
+    def query_match(query, keywords)
+      return true unless EXACT_MATCH
+      @conditions = {} unless @conditions
+      condition = @conditions[query.id]
+      unless condition
+        condition = JSON.parse(query.key)
+        @conditions[query.id] = condition
+        # CAUTION: @conditions can be huge.
+      end
+      words = {}
+      keywords.each do |keyword|
+        words[keyword.key] = true
+      end
+      eval_condition(condition, words)
+    end
+
+    def eval_condition(condition, words)
+      case condition
+      when Hash
+        # todo
+      when String
+        words[condition]
+      when Array
+        case condition.first
+        when "||"
+          condition[1..-1].each do |element|
+            return true if eval_condition(element, words)
+          end
+          false
+        when "&&"
+          condition[1..-1].each do |element|
+            return false unless eval_condition(element, words)
+          end
+          true
+        when "-"
+          return false unless eval_condition(condition[1], words)
+          condition[2..-1].each do |element|
+            return false if eval_condition(element, words)
+          end
+          true
+        end
+      end
+    end
+
+    def publish(hits, request)
+      routes = {}
+      hits.each do |query|
+        subscribers = @subscriber_table.select do |subscriber|
+          subscriber.subscriptions =~ query
+        end
+        subscribers.each do |subscriber|
+          route = subscriber.route.key
+          routes[route] ||= []
+          routes[route] << subscriber.key.key
+        end
+=begin
+# "group" version. This is slower than above...
+        route_records = subscribers.group("route",
+                                          :max_n_sub_records => subscribers.size)
+        route_records.each do |route_record|
+          route = route_record._key
+          routes[route] ||= []
+          route_record.sub_records.each do |subscriber|
+            routes[route] << subscriber.key.key
+          end
+        end
+=end
+      end
+      routes.each do |route, subscribers|
+        yield(route, subscribers)
+      end
+    end
+
+    private
+    def delete_subscriber(subscriber)
+      queries = subscriber.subscriptions
+      route = subscriber.route
+      subscriber.delete
+      sweep_orphan_queries(queries)
+      sweep_orphan_route(route)
+    end
+
+    def delete_query(query)
+      keywords = query.keywords
+      query.delete
+      sweep_orphan_keywords(keywords)
+    end
+
+    def sweep_orphan_queries(queries)
+      queries.each do |query|
+        related_subscribers = @subscriber_table.select do |subscriber|
+          subscriber.subscriptions =~ query
+        end
+        if related_subscribers.empty?
+          delete_query(query)
+        end
+      end
+    end
+
+    def sweep_orphan_keywords(keywords)
+      keywords.each do |keyword|
+        related_queries = @query_table.select do |query|
+          query.keywords =~ keyword
+        end
+        if related_queries.empty?
+          keyword.delete
+        end
+      end
+    end
+
+    def sweep_orphan_route(route)
+      related_subscribers = @subscriber_table.select do |subscriber|
+        subscriber.route == route
+      end
+      if related_subscribers.empty?
+        route.delete
+      end
+    end
+  end
+end

data/lib/droonga/worker.rb

@@ -0,0 +1,61 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/event_loop"
+require "droonga/handler_runner"
+require "droonga/job_receiver"
+
+module Droonga
+  module Worker
+    def initialize
+      @raw_loop = Coolio::Loop.new
+      @loop = EventLoop.new(@raw_loop)
+      @handler_runner = HandlerRunner.new(@loop,
+                                          config.merge(:dispatcher => nil))
+      receive_socket_path = config[:job_receive_socket_path]
+      @job_receiver = JobReceiver.new(@loop, receive_socket_path) do |message|
+        process(message)
+      end
+    end
+
+    def run
+      Droonga.logger.trace("#{log_tag}: run: start")
+      @handler_runner.start
+      @job_receiver.start
+      @raw_loop.run
+      @handler_runner.shutdown
+      Droonga.logger.trace("#{log_tag}: run: done")
+    end
+
+    def stop
+      Droonga.logger.trace("#{log_tag}: stop: start")
+      @job_receiver.shutdown
+      @raw_loop.stop
+      @loop.break_current_loop
+      Droonga.logger.trace("#{log_tag}: stop: done")
+    end
+
+    private
+    def process(message)
+      Droonga.logger.trace("#{log_tag}: process: start")
+      @handler_runner.process(message)
+      Droonga.logger.trace("#{log_tag}: process: done")
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] worker"
+    end
+  end
+end

data/sample/cluster/catalog.json

@@ -0,0 +1,42 @@
+{
+  "version": 2,
+  "effectiveDate": "2013-09-01T00:00:00Z",
+  "datasets": {
+    "Droonga": {
+      "nWorkers": 2,
+      "plugins": ["groonga", "search", "crud"],
+      "schema": {
+      },
+      "replicas": [
+        {
+          "slices": [
+            {
+              "volume": {
+                "address": "127.0.0.1:10031/droonga.000"
+              }
+            },
+            {
+              "volume": {
+                "address": "127.0.0.1:10031/droonga.001"
+              }
+            }
+          ]
+        },
+        {
+          "slices": [
+            {
+              "volume": {
+                "address": "127.0.0.1:10031/droonga.010"
+              }
+            },
+            {
+              "volume": {
+                "address": "127.0.0.1:10031/droonga.011"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  }
+}

data/sample/mecab_filter/data.grn

@@ -0,0 +1,7 @@
+load --table Memo
+[
+{"_key": "1", "content": "アンパサンド"},
+{"_key": "2", "content": "ジャパン"},
+{"_key": "3", "content": "食パン"},
+{"_key": "4", "content": "フランスのパン"}
+]

data/sample/mecab_filter/ddl.grn

@@ -0,0 +1,7 @@
+table_create Memo TABLE_HASH_KEY ShortText
+column_create Memo content COLUMN_SCALAR ShortText
+
+table_create Terms TABLE_PAT_KEY ShortText \
+  --default_tokenizer TokenBigram \
+  --normalizer NormalizerAuto
+column_create Terms memo_context COLUMN_INDEX|WITH_POSITION Memo content

data/sample/mecab_filter/search_with_mecab_filter.json

@@ -0,0 +1,21 @@
+{
+    "type": "search",
+    "dataset": "Droonga",
+    "body": {
+	"queries": {
+	    "search": {
+		"source": "Memo",
+		"condition": {
+		    "matchTo": ["content"],
+		    "query": "パン",
+		    "useMeCabFilter": true
+		},
+		"output": {
+		    "elements": ["count", "records"],
+		    "attributes": ["_key", "content"],
+		    "limit": 10
+		}
+	    }
+	}
+    }
+}

data/sample/mecab_filter/search_without_mecab_filter.json

@@ -0,0 +1,21 @@
+{
+    "type": "search",
+    "dataset": "Droonga",
+    "body": {
+	"queries": {
+	    "search": {
+		"source": "Memo",
+		"condition": {
+		    "matchTo": ["content"],
+		    "query": "パン",
+		    "useMeCabFilter": false
+		},
+		"output": {
+		    "elements": ["count", "records"],
+		    "attributes": ["_key", "content"],
+		    "limit": 10
+		}
+	    }
+	}
+    }
+}

data/test/command/config/default/catalog.json

@@ -0,0 +1,85 @@
+{
+  "version": 2,
+  "effectiveDate": "2014-02-28T00:00:00Z",
+  "datasets": {
+    "Droonga": {
+      "nWorkers": 4,
+      "plugins": ["groonga", "crud", "search"],
+      "replicas": [
+        {
+          "dimension": "_key",
+          "slicer": "hash",
+          "slices": [
+            {
+              "label": "slice000",
+              "weight": 50,
+              "volume": {
+                "address": "127.0.0.1:23003/droonga.000"
+              }
+            },
+            {
+              "label": "slice001",
+              "weight": 50,
+              "volume": {
+                "address": "127.0.0.1:23003/droonga.001"
+              }
+            },
+            {
+              "label": "slice002",
+              "weight": 50,
+              "volume": {
+                "address": "127.0.0.1:23003/droonga.002"
+              }
+            }
+          ]
+        },
+        {
+          "dimension": "_key",
+          "slicer": "hash",
+          "slices": [
+            {
+              "label": "slice010",
+              "weight": 50,
+              "volume": {
+                "address": "127.0.0.1:23003/droonga.010"
+              }
+            },
+            {
+              "label": "slice011",
+              "weight": 50,
+              "volume": {
+                "address": "127.0.0.1:23003/droonga.011"
+              }
+            },
+            {
+              "label": "slice012",
+              "weight": 50,
+              "volume": {
+                "address": "127.0.0.1:23003/droonga.012"
+              }
+            }
+          ]
+        }
+      ]
+    },
+    "Watch": {
+      "nWorkers": 4,
+      "plugins": ["groonga", "watch", "search", "crud"],
+      "replicas": [
+        {
+          "dimension": "_key",
+          "slicer": "hash",
+          "slices": [
+            {
+              "label": "slice100",
+              "weight": 50,
+              "volume": {
+                "address": "127.0.0.1:23003/droonga.watch"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  }
+}

data/test/command/config/default/fluentd.conf

@@ -0,0 +1,11 @@
+<source>
+  type forward
+  port 23003
+</source>
+<match droonga.message>
+  name 127.0.0.1:23003/droonga
+  type droonga
+</match>
+<match output.message>
+  type stdout
+</match>

data/test/command/config/version1/catalog.json

@@ -0,0 +1,68 @@
+{
+  "version": 1,
+  "effective_date": "2013-09-01T00:00:00Z",
+  "zones": ["127.0.0.1:23003/droonga"],
+  "farms": {
+    "127.0.0.1:23003/droonga": {
+      "device": ".",
+      "capacity": 10
+    }
+  },
+  "datasets": {
+    "Droonga": {
+      "workers": 4,
+      "plugins": ["groonga", "crud", "search"],
+      "number_of_replicas": 2,
+      "number_of_partitions": 3,
+      "partition_key": "_key",
+      "date_range": "infinity",
+      "ring": {
+        "127.0.0.1:23041": {
+          "weight": 50,
+          "partitions": {
+            "2013-09-01": [
+              "127.0.0.1:23003/droonga.000",
+              "127.0.0.1:23003/droonga.001"
+            ]
+          }
+        },
+        "127.0.0.1:23042": {
+          "weight": 50,
+          "partitions": {
+            "2013-09-01": [
+              "127.0.0.1:23003/droonga.010",
+              "127.0.0.1:23003/droonga.011"
+            ]
+          }
+        },
+        "127.0.0.1:23043": {
+          "weight": 50,
+          "partitions": {
+            "2013-09-01": [
+              "127.0.0.1:23003/droonga.020",
+              "127.0.0.1:23003/droonga.021"
+            ]
+          }
+        }
+      }
+    },
+    "Watch": {
+      "workers": 4,
+      "plugins": ["groonga", "watch", "search", "crud"],
+      "number_of_replicas": 1,
+      "number_of_partitions": 1,
+      "partition_key": "_key",
+      "date_range": "infinity",
+      "ring": {
+        "127.0.0.1:23041": {
+          "weight": 50,
+          "partitions": {
+            "2013-09-01": [
+              "127.0.0.1:23003/droonga.watch"
+            ]
+          }
+        }
+      }
+    }
+  }
+}

data/test/command/config/version1/fluentd.conf

@@ -0,0 +1,11 @@
+<source>
+  type forward
+  port 23003
+</source>
+<match droonga.message>
+  name 127.0.0.1:23003/droonga
+  type droonga
+</match>
+<match output.message>
+  type stdout
+</match>