droonga-engine 1.0.1

data/lib/droonga/server.rb

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/logger"
+
+module Droonga
+  module Server
+    def before_run
+      Droonga.logger.trace("#{log_tag}: before_run: start")
+      config[:job_pusher].close
+      config.delete(:job_pusher)
+      Droonga.logger.trace("#{log_tag}: before_run: done")
+    end
+
+    def after_run
+      Droonga.logger.trace("#{log_tag}: after_run: start")
+      Droonga.logger.trace("#{log_tag}: after_run: done")
+    end
+
+    def stop(stop_graceful)
+      Droonga.logger.trace("#{log_tag}: stop: start")
+      super(stop_graceful)
+      Droonga.logger.trace("#{log_tag}: stop: done")
+    end
+
+    private
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] server"
+    end
+  end
+end

data/lib/droonga/session.rb

@@ -0,0 +1,99 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Session
+    def initialize(id, dispatcher, collector_runner, tasks, inputs)
+      @id = id
+      @dispatcher = dispatcher
+      @collector_runner = collector_runner
+      @tasks = tasks
+      @n_dones = 0
+      @inputs = inputs
+    end
+
+    def done?
+      @n_dones == @tasks.size
+    end
+
+    def start
+      tasks = @inputs[nil] || []
+      tasks.each do |task|
+        local_message = {
+          "id"   => @id,
+          "task" => task,
+        }
+        @dispatcher.process_local_message(local_message)
+        @n_dones += 1
+      end
+    end
+
+    def receive(name, value)
+      tasks = @inputs[name]
+      unless tasks
+        #TODO: result arrived before its query
+        return
+      end
+      tasks.each do |task|
+        task["n_of_inputs"] += 1
+        step = task["step"]
+        command = step["type"]
+        n_of_expects = step["n_of_expects"]
+        message = {
+          "task"=>task,
+          "name"=>name,
+          "value"=>value
+        }
+        @collector_runner.collect(message)
+        return if task["n_of_inputs"] < n_of_expects
+        #the task is done
+        result = task["values"]
+        post = step["post"]
+        if post
+          # XXX: It is just a workaround.
+          # Remove me when super step is introduced.
+          if result["errors"]
+            reply_body = result
+          elsif command == "search_gather"
+            reply_body = result
+          else
+            reply_body = result["result"]
+          end
+          @dispatcher.reply("body" => reply_body)
+        end
+        send_to_descendantas(step["descendants"], result)
+        @n_dones += 1
+      end
+    end
+
+    private
+    def send_to_descendantas(descendantas, result)
+      descendantas.each do |name, routes|
+        message = {
+          "id" => @id,
+          "input" => name,
+          "value" => result[name]
+        }
+        routes.each do |route|
+          @dispatcher.dispatch(message, route)
+        end
+      end
+    end
+
+    def log_tag
+      "session"
+    end
+  end
+end

data/lib/droonga/single_step.rb

@@ -0,0 +1,68 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/planner"
+require "droonga/collectors"
+
+module Droonga
+  class SingleStep
+    def initialize(dataset, definition)
+      @dataset = dataset
+      @definition = definition
+    end
+
+    def plan(message)
+      if message["type"] == "search"
+        # XXX: workaround
+        planner = Plugins::Search::Planner.new
+        return planner.plan(message)
+      end
+
+      # XXX: Re-implement me.
+      planner = Planner.new
+      options = {}
+      options[:write] = @definition.write?
+      collector_class = @definition.collector_class
+      if collector_class
+        reduce_key = "result"
+        options[:reduce] = {
+          reduce_key => collector_class.operator,
+        }
+      end
+
+      body = message["body"]
+      fact_input = find_fact_input(@definition.inputs, @dataset.fact, body)
+      if fact_input
+        record = body[fact_input[:filter]]
+        planner.send(:scatter, message, record, options)
+      else
+        planner.send(:broadcast, message, options)
+      end
+    end
+
+    def find_fact_input(inputs, fact, body)
+      inputs.each do |key, input|
+        if input[:type] == :table
+          # for backward compatibility. We can remove the following code
+          # when all our catalog.json specify "fact" parameter.
+          return input if fact.nil?
+
+          return input if body[key] == fact
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/droonga/single_step_definition.rb

@@ -0,0 +1,54 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class SingleStepDefinition
+    attr_accessor :name
+    attr_accessor :handler
+    attr_accessor :collector
+    attr_writer :write
+    attr_accessor :inputs
+    attr_accessor :output
+    def initialize(plugin_module)
+      @plugin_module = plugin_module
+      @name = nil
+      @handler = nil
+      @collector = nil
+      @write = false
+      @inputs = []
+      @output = {}
+      yield(self)
+    end
+
+    def write?
+      @write
+    end
+
+    def handler_class
+      resolve_class(@handler)
+    end
+
+    def collector_class
+      resolve_class(@collector)
+    end
+
+    private
+    def resolve_class(target)
+      return nil if target.nil?
+      return target if target.is_a?(Class)
+      @plugin_module.const_get(target)
+    end
+  end
+end

data/lib/droonga/slice.rb

@@ -0,0 +1,122 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "serverengine"
+
+require "droonga/loggable"
+require "droonga/server"
+require "droonga/worker"
+require "droonga/event_loop"
+require "droonga/job_pusher"
+require "droonga/processor"
+require "droonga/schema_applier"
+
+module Droonga
+  class Slice
+    include Loggable
+
+    def initialize(dataset, loop, options={})
+      @dataset = dataset
+      @loop = loop
+      @options = options
+      @n_workers = @options[:n_workers] || 0
+      @job_pusher = JobPusher.new(@loop, @options[:database])
+      @processor = Processor.new(@loop, @job_pusher, @options)
+      @supervisor = nil
+    end
+
+    def start
+      ensure_database
+      @processor.start
+      @job_pusher.start
+      start_supervisor if @n_workers > 0
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      shutdown_supervisor if @supervisor
+      @job_pusher.shutdown
+      @processor.shutdown
+      logger.trace("shutdown: done")
+    end
+
+    def process(message)
+      logger.trace("process: start")
+      @processor.process(message)
+      logger.trace("process: done")
+    end
+
+    private
+    def ensure_database
+      enforce_umask
+      context = Groonga::Context.new
+      begin
+        database_path = @options[:database]
+        if File.exist?(database_path)
+          context.open_database(database_path) do
+            apply_schema(context)
+          end
+        else
+          FileUtils.mkdir_p(File.dirname(database_path))
+          context.create_database(database_path) do
+            apply_schema(context)
+          end
+        end
+      ensure
+        context.close
+      end
+    end
+
+    def enforce_umask
+      File.umask(022)
+    end
+
+    def apply_schema(context)
+      applier = SchemaApplier.new(context, @dataset.schema)
+      applier.apply
+    end
+
+    def start_supervisor
+      @supervisor = ServerEngine::Supervisor.new(Server, Worker) do
+        force_options = {
+          :worker_type   => "process",
+          :workers       => @options[:n_workers],
+          :log_level     => logger.level,
+          :server_process_name => "Server[#{@options[:database]}] #$0",
+          :worker_process_name => "Worker[#{@options[:database]}] #$0",
+          :job_receive_socket_path => @job_pusher.socket_path,
+          :job_pusher => @job_pusher,
+        }
+        @options.merge(force_options)
+      end
+      @supervisor_thread = Thread.new do
+        @supervisor.main
+      end
+    end
+
+    def shutdown_supervisor
+      logger.trace("supervisor: shutdown: start")
+      @supervisor.stop(true)
+      logger.trace("supervisor: shutdown: stopped")
+      @supervisor_thread.join
+      logger.trace("supervisor: shutdown: done")
+    end
+
+    private
+    def log_tag
+      "slice"
+    end
+  end
+end

data/lib/droonga/status_code.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module StatusCode
+    OK                    = 200
+    BAD_REQUEST           = 400
+    NOT_FOUND             = 404
+    INTERNAL_SERVER_ERROR = 500
+  end
+end

data/lib/droonga/step_runner.rb

@@ -0,0 +1,64 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+require "droonga/plugin"
+require "droonga/single_step"
+
+module Droonga
+  class StepRunner
+    include Loggable
+
+    def initialize(dataset, plugins)
+      @dataset = dataset
+      @definitions = {}
+      plugins.each do |name|
+        plugin = Plugin.registry[name]
+        plugin.single_step_definitions.each do |definition|
+          @definitions[definition.name] = definition
+        end
+      end
+    end
+
+    def shutdown
+    end
+
+    def plan(message)
+      type = message["type"]
+      logger.trace("plan: start",
+                   :dataset => message["dataset"],
+                   :type => type)
+      definition = find(type)
+      if definition.nil?
+        raise UnsupportedMessageError.new(:planner, message)
+      end
+      step = SingleStep.new(@dataset, definition)
+      plan = step.plan(message)
+      logger.trace("plan: done",
+                   :dataset => message["dataset"],
+                   :type => type)
+      plan
+    end
+
+    def find(type)
+      @definitions[type]
+    end
+
+    private
+    def log_tag
+      "step-runner"
+    end
+  end
+end

data/lib/droonga/sweeper.rb

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Sweeper
+    SUBSCRIBER_LIFETIME_SECONDS = 10 * 60 # 10 min
+
+    def initialize(context)
+      @context = context
+    end
+
+    def sweep_expired_subscribers(options={})
+      now = options[:now] || Time.now
+      boundary = now - SUBSCRIBER_LIFETIME_SECONDS
+      expired_subscribers = @context["Subscriber"].select do |subscriber|
+        subscriber.last_modified < boundary
+      end
+      expired_subscribers.each do |subscriber|
+        watcher.unsubscribe(:subscriber => subscriber._key)
+      end
+    end
+
+    private
+    def watcher
+      @watcher ||= Watcher.new(@context)
+    end
+  end
+end

data/lib/droonga/test/stub_handler.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Test
+    class StubHandler
+      attr_reader :context, :envelope, :messages
+      def initialize
+        @context = Groonga::Context.default
+        @envelope = {}
+        @messages = []
+      end
+
+      def emit(message)
+        @messages << message
+      end
+
+      def forward(message, destination)
+        @messages << [message, destination]
+      end
+    end
+  end
+end

data/lib/droonga/test/stub_handler_message.rb

@@ -0,0 +1,35 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/handler_message"
+
+module Droonga
+  module Test
+    class StubHandlerMessage < HandlerMessage
+      def initialize(request, headers={})
+        raw = {
+          "body" => {
+            "task" => {
+              "step" => {
+                "body" => request,
+              },
+            },
+          },
+        }
+        super(headers.merge(raw))
+      end
+    end
+  end
+end

data/lib/droonga/test/stub_handler_messenger.rb

@@ -0,0 +1,34 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Test
+    class StubHandlerMessenger
+      attr_reader :values, :messages
+      def initialize
+        @values = []
+        @messages = []
+      end
+
+      def emit(value)
+        @values << value
+      end
+
+      def forward(message, destination)
+        @messages << [message, destination]
+      end
+    end
+  end
+end

data/lib/droonga/test/stub_planner.rb

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Test
+    class StubPlanner
+      attr_reader :messages
+      def initialize
+        @messages = []
+      end
+
+      def distribute(message)
+        @messages << message
+      end
+    end
+  end
+end

data/lib/droonga/test.rb

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/test/stub_planner"
+require "droonga/test/stub_handler"
+require "droonga/test/stub_handler_message"
+require "droonga/test/stub_handler_messenger"