droonga-engine 1.0.1

data/lib/droonga/error.rb

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class Error < StandardError
+  end
+
+  class MultiplexError < Error
+    attr_reader :errors
+
+    def initialize(errors=[])
+      @errors = errors
+      error_messages = @errors.collect do |error|
+        error.message
+      end
+      message = error_messages.sort.join("\n-----------------------\n")
+      super(message)
+    end
+  end
+
+  # the base class for any error which can be described as a Droonga message
+  class ErrorMessage < Error
+    STATUS_CODE = nil
+
+    attr_reader :detail
+
+    def initialize(message, detail=nil)
+      @detail = detail
+      super(message)
+    end
+
+    def name
+      self.class.name.split("::").last
+    end
+
+    def status_code
+      self.class::STATUS_CODE
+    end
+
+    def response_body
+      body = {
+        "name"    => name,
+        "message" => message,
+      }
+      body["detail"] = @detail unless @detail.nil?
+      body
+    end
+  end
+
+  # TODO: Move to common file for runners
+  class UnsupportedMessageError < Error
+    attr_reader :phase, :message
+    def initialize(phase, message)
+      @phase = phase
+      @message = message
+      super("[#{@phase}] Unsupported message: #{@message.inspect}")
+    end
+  end
+end

data/lib/droonga/error_messages.rb

@@ -0,0 +1,33 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/error"
+require "droonga/status_code"
+
+module Droonga
+  module ErrorMessages
+    class InternalServerError < ErrorMessage
+      STATUS_CODE = StatusCode::INTERNAL_SERVER_ERROR
+    end
+
+    class BadRequest < ErrorMessage
+      STATUS_CODE = StatusCode::BAD_REQUEST
+    end
+
+    class NotFound < ErrorMessage
+      STATUS_CODE = StatusCode::NOT_FOUND
+    end
+  end
+end

data/lib/droonga/event_loop.rb

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "coolio"
+
+module Droonga
+  class EventLoop
+    def initialize(loop)
+      @loop = loop
+      @loop_breaker = Coolio::AsyncWatcher.new
+      @loop_breaker.attach(@loop)
+    end
+
+    def attach(watcher)
+      @loop.attach(watcher)
+      break_current_loop
+    end
+
+    def break_current_loop
+      @loop_breaker.signal
+    end
+
+    def run
+      @loop.run
+    end
+
+    def stop
+      @loop.stop
+      break_current_loop
+    end
+  end
+end

data/lib/droonga/farm.rb

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/slice"
+
+module Droonga
+  class Farm
+    def initialize(name, catalog, loop, options={})
+      @name = name
+      @catalog = catalog
+      @loop = loop
+      @options = options
+      @slices = {}
+      slices = @catalog.slices(name)
+      slices.each do |slice_name, slice_options|
+        dataset = @catalog.datasets[slice_options[:dataset]]
+        slice = Droonga::Slice.new(dataset,
+                                   @loop,
+                                   @options.merge(slice_options))
+        @slices[slice_name] = slice
+      end
+    end
+
+    def start
+      @slices.each_value do |slice|
+        slice.start
+      end
+    end
+
+    def shutdown
+      threads = []
+      @slices.each_value do |slice|
+        threads << Thread.new do
+          slice.shutdown
+        end
+      end
+      threads.each(&:join)
+    end
+
+    def process(slice_name, message)
+      @slices[slice_name].process(message)
+    end
+  end
+end

data/lib/droonga/fluent_message_receiver.rb

@@ -0,0 +1,191 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "socket"
+
+require "msgpack"
+
+require "droonga/loggable"
+require "droonga/event_loop"
+
+module Droonga
+  class FluentMessageReceiver
+    include Loggable
+
+    def initialize(loop, options={}, &on_message)
+      @loop = loop
+      @listen_fd = options[:listen_fd]
+      @heartbeat_fd = options[:heartbeat_fd]
+      @server = nil
+      @clients = []
+      @on_message = on_message
+    end
+
+    def start
+      logger.trace("start: start")
+      start_heartbeat_receiver
+      start_server
+      logger.trace("start: done")
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      shutdown_server
+      shutdown_clients
+      shutdown_heartbeat_receiver
+      logger.trace("shutdown: done")
+    end
+
+    private
+    def start_heartbeat_receiver
+      logger.trace("start_heartbeat_receiver: start")
+      @heartbeat_receiver = HeartbeatReceiver.new(@loop, @heartbeat_fd)
+      @heartbeat_receiver.start
+      logger.trace("start_heartbeat_receiver: done")
+    end
+
+    def shutdown_heartbeat_receiver
+      logger.trace("shutdown_heartbeat_receiver: start")
+      @heartbeat_receiver.shutdown
+      logger.trace("shutdown_heartbeat_receiver: done")
+    end
+
+    def start_server
+      logger.trace("start_server: start")
+
+      @clients = []
+      @server = create_server do |connection|
+        client = Client.new(connection) do |tag, time, record|
+          @on_message.call(tag, time, record)
+        end
+        @clients << client
+      end
+      @loop.attach(@server)
+
+      logger.trace("start_server: done")
+    end
+
+    def create_server(&block)
+      Coolio::Server.new(TCPServer.for_fd(@listen_fd), Coolio::TCPSocket, &block)
+    end
+
+    def shutdown_server
+      @server.close
+    end
+
+    def shutdown_clients
+      @clients.each do |client|
+        client.close
+      end
+    end
+
+    def log_tag
+      "fluent-message-receiver"
+    end
+
+    class HeartbeatReceiver
+      def initialize(loop, fd)
+        @loop = loop
+        @fd = fd
+      end
+
+      def start
+        @socket = UDPSocket.for_fd(@fd)
+
+        @watcher = Coolio::IOWatcher.new(@socket, "r")
+        on_readable = lambda do
+          receive_heartbeat
+        end
+        @watcher.on_readable do
+          on_readable.call
+        end
+        @loop.attach(@watcher)
+      end
+
+      def shutdown
+        @socket.close
+        @watcher.detach
+      end
+
+      private
+      def receive_heartbeat
+        address = nil
+        begin
+          _, address = @socket.recvfrom(1024)
+        rescue SystamCallError
+          return
+        end
+
+        host = address[3]
+        port = address[1]
+        send_back_heartbeat(host, port)
+      end
+
+      def send_back_heartbeat(host, port)
+        data = "\0"
+        flags = 0
+        begin
+          @socket.send(data, flags, host, port)
+        rescue SystemCallError
+        end
+      end
+    end
+
+    class Client
+      include Loggable
+
+      def initialize(io, &on_message)
+        @io = io
+        @on_message = on_message
+        @unpacker = MessagePack::Unpacker.new
+        on_read = lambda do |data|
+          feed(data)
+        end
+        @io.on_read do |data|
+          on_read.call(data)
+        end
+      end
+
+      def close
+        @io.close
+      end
+
+      private
+      def feed(data)
+        @unpacker.feed_each(data) do |object|
+          tag = object[0]
+          case object[1]
+          when String # PackedForward message
+            entries = MessagePack.unpack(object[1])
+          when Array # Forward message
+            entries = object[1]
+          when Integer, Float # Message message
+            entries = [[object[1], object[2]]]
+          else
+            logger.error("unknown message", :message => object)
+            next
+          end
+          entries.each do |time, record|
+            @on_message.call(tag, time || Time.now.to_i, record)
+          end
+        end
+      end
+
+      def log_tag
+        "fluent-message-receiver::client"
+      end
+    end
+  end
+end

data/lib/droonga/fluent_message_sender.rb

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "thread"
+
+require "cool.io"
+
+require "droonga/message-pack-packer"
+
+require "droonga/loggable"
+
+module Droonga
+  class FluentMessageSender
+    include Loggable
+
+    def initialize(loop, host, port)
+      @loop = loop
+      @host = host
+      @port = port
+      @socket = nil
+      @buffer = []
+      @write_mutex = Mutex.new
+    end
+
+    def start
+      logger.trace("start: start")
+      start_writer
+      logger.trace("start: done")
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      shutdown_writer
+      shutdown_socket
+      logger.trace("shutdown: done")
+    end
+
+    def send(tag, data)
+      logger.trace("send: start")
+      fluent_message = [tag, Time.now.to_i, data]
+      packed_fluent_message = MessagePackPacker.pack(fluent_message)
+      @write_mutex.synchronize do
+        @buffer << packed_fluent_message
+        unless @signaling
+          @signaling = true
+          @writer.signal
+        end
+      end
+      logger.trace("send: done")
+    end
+
+    private
+    def connected?
+      not @socket.nil?
+    end
+
+    def connect
+      logger.trace("connect: start")
+
+      log_write_complete = lambda do
+        logger.trace("write completed")
+      end
+      log_connect = lambda do
+        logger.trace("connected to #{@host}:#{@port}")
+      end
+      log_failed = lambda do
+        logger.error("failed to connect to #{@host}:#{@port}")
+        @socket = nil
+      end
+      on_close = lambda do
+        @socket = nil
+      end
+
+      @socket = Coolio::TCPSocket.connect(@host, @port)
+      @socket.on_write_complete do
+        log_write_complete.call
+      end
+      @socket.on_connect do
+        log_connect.call
+      end
+      @socket.on_connect_failed do
+        log_failed.call
+      end
+      @socket.on_close do
+        on_close.call
+      end
+      @loop.attach(@socket)
+
+      logger.trace("connect: done")
+    end
+
+    def shutdown_socket
+      return unless connected?
+      @socket.close unless @socket.closed?
+    end
+
+    def start_writer
+      @writer = Coolio::AsyncWatcher.new
+      @signaling = false
+
+      on_signal = lambda do
+        @write_mutex.synchronize do
+          @signaling = false
+          connect unless connected?
+          @buffer.each do |data|
+            @socket.write(data)
+          end
+          @buffer.clear
+        end
+      end
+      @writer.on_signal do
+        on_signal.call
+      end
+
+      @loop.attach(@writer)
+    end
+
+    def shutdown_writer
+      @writer.detach
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] fluent-message-sender"
+    end
+  end
+end

data/lib/droonga/forwarder.rb

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/loggable"
+require "droonga/event_loop"
+require "droonga/fluent_message_sender"
+
+module Droonga
+  class Forwarder
+    include Loggable
+
+    def initialize(loop)
+      @loop = loop
+      @senders = {}
+    end
+
+    def start
+      logger.trace("start: start")
+      logger.trace("start: done")
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      @senders.each_value do |sender|
+        sender.shutdown
+      end
+      logger.trace("shutdown: done")
+    end
+
+    def forward(message, destination)
+      logger.trace("forward: start")
+      command = destination["type"]
+      receiver = destination["to"]
+      arguments = destination["arguments"]
+      output(receiver, message, command, arguments)
+      logger.trace("forward: done")
+    end
+
+    private
+    def output(receiver, message, command, arguments)
+      logger.trace("output: start")
+      # TODO: IMPROVE ME: Should not use "unless" and "and". It is confused.
+      unless receiver.is_a?(String) and command.is_a?(String)
+        logger.trace("output: abort: invalid argument",
+                     :receiver => receiver,
+                     :command  => command)
+        return
+      end
+      unless receiver =~ /\A(.*):(\d+)\/(.*?)(\?.+)?\z/
+        raise "format: hostname:port/tag(?params)"
+      end
+      host = $1
+      port = $2
+      tag  = $3
+      params = $4
+      sender = find_sender(host, port, params)
+      unless sender
+        logger.trace("output: abort: no sender",
+                     :host   => host,
+                     :port   => port,
+                     :params => params)
+        return
+      end
+      override_message = {
+        "type" => command,
+      }
+      override_message["arguments"] = arguments if arguments
+      message = message.merge(override_message)
+      output_tag = "#{tag}.message"
+      log_info = "<#{receiver}>:<#{output_tag}>"
+      logger.trace("output: post: start: #{log_info}")
+      sender.send(output_tag, message)
+      logger.trace("output: post: done: #{log_info}")
+      logger.trace("output: done")
+    end
+
+    def find_sender(host, port, params)
+      connection_id = extract_connection_id(params)
+      destination = "#{host}:#{port}"
+      destination << "?#{connection_id}" if connection_id
+
+      @senders[destination] ||= create_sender(host, port)
+    end
+
+    def extract_connection_id(params)
+      return nil unless params
+
+      if /[\?&;]connection_id=([^&;]+)/ =~ params
+        $1
+      else
+        nil
+      end
+    end
+
+    def create_sender(host, port)
+      sender = FluentMessageSender.new(@loop, host, port)
+      sender.start
+      sender
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] forwarder"
+    end
+  end
+end

data/lib/droonga/handler.rb

@@ -0,0 +1,49 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/pluggable"
+require "droonga/loggable"
+require "droonga/plugin/metadata/input_message"
+require "droonga/plugin/metadata/handler_action"
+require "droonga/error_messages"
+
+module Droonga
+  class Handler
+    extend Pluggable
+    include Loggable
+    include ErrorMessages
+
+    class << self
+      def message
+        Plugin::Metadata::InputMessage.new(self)
+      end
+
+      def action
+        Plugin::Metadata::HandlerAction.new(self)
+      end
+    end
+
+    attr_reader :messenger, :loop
+    def initialize(name, context, messenger, loop)
+      @name = name
+      @context = context
+      @messenger = messenger
+      @loop = loop
+    end
+
+    def handle(message)
+    end
+  end
+end