fluent-plugin-droonga 0.0.2

data/lib/droonga/server.rb

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga"
+
+module Droonga
+  module Server
+    def before_run
+      $log.trace("#{log_tag}: before_run: start")
+      $log.trace("#{log_tag}: before_run: done")
+    end
+
+    def after_run
+      $log.trace("#{log_tag}: after_run: start")
+      $log.trace("#{log_tag}: after_run: done")
+    end
+
+    def stop(stop_graceful)
+      $log.trace("#{log_tag}: stop: start")
+      super(stop_graceful)
+      $log.trace("#{log_tag}: stop: done")
+    end
+
+    private
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] server"
+    end
+
+    def start_worker(wid)
+      worker = super(wid)
+      worker.extend(WorkerStopper)
+      worker
+    end
+
+    module WorkerStopper
+      def send_stop(stop_graceful)
+        in_signal_sending do
+          open_queue do |queue|
+            $log.trace("#{log_tag}: stop: start")
+
+            $log.trace("#{log_tag}: stop: queue: unblock: start")
+            max_n_retries = 10
+            max_n_retries.times do |i|
+              $log.trace("#{log_tag}: stop: queue: unblock: #{i}: start")
+              super(stop_graceful)
+              queue.unblock
+              alive_p = alive?
+              $log.trace("#{log_tag}: stop: queue: unblock: #{i}: done: " +
+                         "#{alive_p}")
+              break unless alive_p
+              sleep(i * 0.1)
+            end
+            $log.trace("#{log_tag}: stop: queue: unblock: done")
+
+            $log.trace("#{log_tag}: stop: done")
+          end
+        end
+      end
+
+      def send_reload
+        in_signal_sending do
+          open_queue do |queue|
+            $log.trace("#{log_tag}: reload: start")
+            super
+            queue.unblock
+            $log.trace("#{log_tag}: reload: done")
+          end
+        end
+      end
+
+      private
+      def log_tag
+        "[#{Process.ppid}][#{Process.pid}][#{@wid}] server: worker-stopper"
+      end
+
+      def in_signal_sending
+        Thread.new do
+          yield
+        end
+      end
+
+      def open_queue
+        config = @worker.config
+        # TODO: Use JobQueue object
+        context = Groonga::Context.new
+        database = context.open_database(config[:database])
+        queue = context[config[:queue_name]]
+        begin
+          yield(queue)
+        ensure
+          queue.close
+          database.close
+          context.close
+        end
+      end
+    end
+  end
+end

data/lib/droonga/sweeper.rb

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Sweeper
+    SUBSCRIBER_LIFETIME_SECONDS = 10 * 60 # 10 min
+
+    def initialize(context)
+      @context = context
+    end
+
+    def sweep_expired_subscribers(options={})
+      now = options[:now] || Time.now
+      boundary = now - SUBSCRIBER_LIFETIME_SECONDS
+      expired_subscribers = @context["Subscriber"].select do |subscriber|
+        subscriber.last_modified < boundary
+      end
+      expired_subscribers.each do |subscriber|
+        watcher.unsubscribe(:subscriber => subscriber._key)
+      end
+    end
+
+    private
+    def watcher
+      @watcher ||= Watcher.new(@context)
+    end
+  end
+end

data/lib/droonga/watch_schema.rb

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga"
+
+module Droonga
+  class WatchSchema
+    def initialize(context)
+      @context = context
+    end
+
+    def ensure_created
+      if @context["Keyword"]
+        $log.trace "#{log_tag} skip table creation"
+        return
+      end
+      $log.trace "#{log_tag} ensure_tables: start"
+      ensure_tables
+      $log.trace "#{log_tag} ensure_tables: done"
+    end
+
+    private
+    def ensure_tables
+      Groonga::Schema.define(:context => @context) do |schema|
+        schema.create_table("Keyword",
+                            :type => :patricia_trie,
+                            :key_type => "ShortText",
+                            :key_normalize => true,
+                            :force => true) do |table|
+                            end
+
+        schema.create_table("Query",
+                            :type => :hash,
+                            :key_type => "ShortText",
+                            :force => true) do |table|
+                            end
+
+        schema.create_table("Route",
+                            :type => :hash,
+                            :key_type => "ShortText",
+                            :force => true) do |table|
+                            end
+
+        schema.create_table("Subscriber",
+                            :type => :hash,
+                            :key_type => "ShortText",
+                            :force => true) do |table|
+          table.time("last_modified")
+                            end
+
+        schema.change_table("Query") do |table|
+          table.reference("keywords", "Keyword", :type => :vector)
+        end
+
+        schema.change_table("Subscriber") do |table|
+          table.reference("route", "Route")
+          table.reference("subscriptions", "Query", :type => :vector)
+        end
+
+        schema.change_table("Keyword") do |table|
+          table.index("Query", "keywords", :name => "queries")
+        end
+
+        schema.change_table("Query") do |table|
+          table.index("Subscriber", "subscriptions", :name => "subscribers")
+        end
+      end
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] watch_schema"
+    end
+  end
+end

data/lib/droonga/watcher.rb

@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Watcher
+    EXACT_MATCH = false
+
+    def initialize(context)
+      @context = context
+
+      @subscriber_table = @context["Subscriber"]
+      @query_table      = @context["Query"]
+      @keyword_table    = @context["Keyword"]
+    end
+
+    def subscribe(request)
+      subscriber = request[:subscriber]
+      condition  = request[:condition]
+      query      = request[:query]
+      route      = request[:route]
+
+      # XXX better validation and error class must be written!!
+      if subscriber.nil? || subscriber.empty? || condition.nil? ||
+         query.nil? || route.nil?
+        raise "invalid request"
+      end
+      raise "too long query" if query.size > 4095
+
+      query_record = @query_table[query]
+      unless query_record
+        keywords = pick_keywords([], condition)
+        query_record = @query_table.add(query, :keywords => keywords)
+      end
+      subscriber_record = @subscriber_table[subscriber]
+      if subscriber_record
+        subscriptions = subscriber_record.subscriptions
+        unless subscriptions.include?(query_record)
+          subscriptions << query_record
+          subscriber_record.subscriptions = subscriptions
+        end
+        subscriber_record.last_modified = Time.now
+      else
+        @subscriber_table.add(subscriber,
+                              :subscriptions => [query_record],
+                              :route => route,
+                              :last_modified => Time.now)
+      end
+    end
+
+    def unsubscribe(request)
+      subscriber = request[:subscriber]
+      query      = request[:query]
+
+      if subscriber.nil? || subscriber.empty?
+        raise "invalid request"
+      end
+
+      subscriber_record = @subscriber_table[subscriber]
+      return unless subscriber_record
+
+      if query.nil? || query.empty?
+        delete_subscriber(subscriber_record)
+      else
+        query_record = @query_table[query]
+        return unless query_record
+
+        subscriptions = subscriber_record.subscriptions
+        new_subscriptions = subscriptions.select do |query|
+          query != query_record
+        end
+
+        if new_subscriptions.empty?
+          delete_subscriber(subscriber_record)
+        else
+          subscriber_record.subscriptions = new_subscriptions
+          sweep_orphan_queries(subscriptions)
+        end
+      end
+    end
+
+    def feed(request, &block)
+      targets = request[:targets]
+
+      hits = []
+      targets.each do |key, target|
+        scan_body(hits, target)
+      end
+
+      publish(hits, request, &block)
+    end
+
+    def pick_keywords(memo, condition)
+      case condition
+      when Hash
+        memo << condition["query"]
+      when String
+        memo << condition
+      when Array
+        condition[1..-1].each do |element|
+          pick_keywords(memo, element)
+        end
+      end
+      memo
+    end
+
+    def scan_body(hits, body)
+      trimmed = body.strip
+      candidates = {}
+      # FIXME scan reports the longest keyword matched only
+      @keyword_table.scan(trimmed).each do |keyword, word, start, length|
+        @query_table.select do |query|
+          query.keywords =~ keyword
+        end.each do |record|
+          candidates[record.key] ||= []
+          candidates[record.key] << keyword
+        end
+      end
+      candidates.each do |query, keywords|
+        hits << query if query_match(query, keywords)
+      end
+    end
+
+    def query_match(query, keywords)
+      return true unless EXACT_MATCH
+      @conditions = {} unless @conditions
+      condition = @conditions[query.id]
+      unless condition
+        condition = JSON.parse(query.key)
+        @conditions[query.id] = condition
+        # CAUTION: @conditions can be huge.
+      end
+      words = {}
+      keywords.each do |keyword|
+        words[keyword.key] = true
+      end
+      eval_condition(condition, words)
+    end
+
+    def eval_condition(condition, words)
+      case condition
+      when Hash
+        # todo
+      when String
+        words[condition]
+      when Array
+        case condition.first
+        when "||"
+          condition[1..-1].each do |element|
+            return true if eval_condition(element, words)
+          end
+          false
+        when "&&"
+          condition[1..-1].each do |element|
+            return false unless eval_condition(element, words)
+          end
+          true
+        when "-"
+          return false unless eval_condition(condition[1], words)
+          condition[2..-1].each do |element|
+            return false if eval_condition(element, words)
+          end
+          true
+        end
+      end
+    end
+
+    def publish(hits, request)
+      routes = {}
+      hits.each do |query|
+        subscribers = @subscriber_table.select do |subscriber|
+          subscriber.subscriptions =~ query
+        end
+        subscribers.each do |subscriber|
+          route = subscriber.route.key
+          routes[route] ||= []
+          routes[route] << subscriber.key.key
+        end
+=begin
+# "group" version. This is slower than above...
+        route_records = subscribers.group("route",
+                                          :max_n_sub_records => subscribers.size)
+        route_records.each do |route_record|
+          route = route_record._key
+          routes[route] ||= []
+          route_record.sub_records.each do |subscriber|
+            routes[route] << subscriber.key.key
+          end
+        end
+=end
+      end
+      routes.each do |route, subscribers|
+        yield(route, subscribers)
+      end
+    end
+
+    private
+    def delete_subscriber(subscriber)
+      queries = subscriber.subscriptions
+      route = subscriber.route
+      subscriber.delete
+      sweep_orphan_queries(queries)
+      sweep_orphan_route(route)
+    end
+
+    def delete_query(query)
+      keywords = query.keywords
+      query.delete
+      sweep_orphan_keywords(keywords)
+    end
+
+    def sweep_orphan_queries(queries)
+      queries.each do |query|
+        related_subscribers = @subscriber_table.select do |subscriber|
+          subscriber.subscriptions =~ query
+        end
+        if related_subscribers.empty?
+          delete_query(query)
+        end
+      end
+    end
+
+    def sweep_orphan_keywords(keywords)
+      keywords.each do |keyword|
+        related_queries = @query_table.select do |query|
+          query.keywords =~ keyword
+        end
+        if related_queries.empty?
+          keyword.delete
+        end
+      end
+    end
+
+    def sweep_orphan_route(route)
+      related_subscribers = @subscriber_table.select do |subscriber|
+        subscriber.route == route
+      end
+      if related_subscribers.empty?
+        route.delete
+      end
+    end
+  end
+end