fluent-plugin-groonga 1.0.0

data/doc/text/news.md

@@ -0,0 +1,5 @@
+# News
+
+## 1.0.0: 2012-11-29
+
+The first release!!!

data/fluent-plugin-groonga.gemspec

@@ -0,0 +1,42 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+Gem::Specification.new do |spec|
+  spec.name = "fluent-plugin-groonga"
+  spec.version = "1.0.0"
+  spec.authors = ["Kouhei Sutou"]
+  spec.email = ["kou@clear-code.com"]
+  spec.summary = "Fluentd plugin collection for groonga users"
+  spec.description =
+    "Groonga users can replicate their data by fluent-plugin-groonga"
+  spec.homepage = "https://github.com/groonga/fluent-plugin-groonga"
+
+  spec.files = ["README.md", "Gemfile", "#{spec.name}.gemspec"]
+  spec.files += Dir.glob("lib/**/*.rb")
+  spec.files += Dir.glob("sample/**/*")
+  spec.files += Dir.glob("doc/text/**/*")
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency("fluentd")
+  spec.add_runtime_dependency("gqtp")
+  spec.add_runtime_dependency("groonga-command")
+
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("bundler")
+  spec.add_development_dependency("test-unit")
+  spec.add_development_dependency("test-unit-notify")
+end

data/lib/fluent/plugin/in_groonga.rb

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "English"
+require "webrick/httputils"
+
+require "http_parser"
+
+require "gqtp"
+require "groonga/command"
+
+module Fluent
+  class GroongaInput < Input
+    Plugin.register_input("groonga", self)
+
+    def initialize
+      super
+    end
+
+    config_param :protocol, :string, :default => "http"
+
+    def configure(conf)
+      super
+      case @protocol
+      when "http"
+        @input = HTTPInput.new
+      when "gqtp"
+        @input = GQTPInput.new
+      else
+        message = "unknown protocol: <#{@protocol.inspect}>"
+        $log.error message
+        raise ConfigError, message
+      end
+      @input.configure(conf)
+    end
+
+    def start
+      @input.start
+    end
+
+    def shutdown
+      @input.shutdown
+    end
+
+    class Repeater < Coolio::TCPSocket
+      def initialize(socket, handler)
+        super(socket)
+        @handler = handler
+      end
+
+      def on_read(data)
+        @handler.write(data)
+      end
+
+      def on_close
+        @handler.close
+      end
+    end
+
+    class BaseInput
+      include Configurable
+      include DetachMultiProcessMixin
+
+      config_param :bind, :string, :default => "0.0.0.0"
+      config_param :port, :integer, :default => 10041
+      config_param :real_host, :string
+      config_param :real_port, :integer, :default => 10041
+      DEFAULT_EMIT_COMMANDS = [
+        /\Atable_/,
+        /\Acolumn_/,
+        "load",
+      ]
+      config_param :emit_commands, :default => DEFAULT_EMIT_COMMANDS do |value|
+        commands = value.split(/\s*,\s*/)
+        commands.collect do |command|
+          if /\A\/(.*)\/(i)?\z/ =~ command
+            pattern = $1
+            flag_mark = $2
+            flag = 0
+            flag |= Regexp::IGNORECASE if flag_mark == "i"
+            Regexp.new(pattern, flag)
+          else
+            command
+          end
+        end
+      end
+
+      def start
+        listen_socket = TCPServer.new(@bind, @port)
+        detach_multi_process do
+          @loop = Coolio::Loop.new
+
+          @socket = Coolio::TCPServer.new(listen_socket, nil,
+                                          handler_class, self)
+          @loop.attach(@socket)
+
+          @shutdown_notifier = Coolio::AsyncWatcher.new
+          @loop.attach(@shutdown_notifier)
+
+          @thread = Thread.new do
+            run
+          end
+        end
+      end
+
+      def shutdown
+        @loop.stop
+        @socket.close
+        @shutdown_notifier.signal
+        @thread.join
+      end
+
+      def create_repeater(client)
+        repeater = Repeater.connect(@real_host, @real_port, client)
+        repeater.attach(@loop)
+        repeater
+      end
+
+      def emit(command, params)
+        return unless emit_command?(command)
+        Engine.emit("groonga.command.#{command}", Engine.now, params)
+      end
+
+      private
+      def run
+        @loop.run
+      rescue
+        $log.error "unexpected error", :error => $!.to_s
+        $log.error_backtrace
+      end
+
+      def emit_command?(command)
+        return true if @emit_commands.empty?
+        @emit_commands.any? do |pattern|
+          pattern === command
+        end
+      end
+    end
+
+    class HTTPInput < BaseInput
+      private
+      def handler_class
+        Handler
+      end
+
+      class Handler < Coolio::Socket
+        def initialize(socket, input)
+          super(socket)
+          @input = input
+        end
+
+        def on_connect
+          @parser = HTTP::Parser.new(self)
+          @repeater = @input.create_repeater(self)
+        end
+
+        def on_read(data)
+          @parser << data
+          @repeater.write(data)
+        end
+
+        def on_message_begin
+          @body = ""
+        end
+
+        def on_headers_complete(headers)
+        end
+
+        def on_body(chunk)
+          @body << chunk
+        end
+
+        def on_message_complete
+          params = WEBrick::HTTPUtils.parse_query(@parser.query_string)
+          path_info = @parser.request_path
+          case path_info
+          when /\A\/d\//
+            command = $POSTMATCH
+            if command == "load"
+              params["values"] = @body unless @body.empty?
+            end
+            @input.emit(command, params)
+          end
+        end
+      end
+    end
+
+    class GQTPInput < BaseInput
+      private
+      def handler_class
+        Handler
+      end
+
+      class Handler < Coolio::Socket
+        def initialize(socket, input)
+          super(socket)
+          @input = input
+        end
+
+        def on_connect
+          @parser = Parser.new(@input)
+          @repeater = @input.create_repeater(self)
+        end
+
+        def on_read(data)
+          @parser << data
+          @repeater.write(data)
+        end
+
+        def on_close
+          @parser.close
+        end
+      end
+
+      class Parser < GQTP::Parser
+        def initialize(input)
+          super()
+          @input = input
+          initialize_command_parser
+        end
+
+        def on_body(chunk)
+          @command_parser << chunk
+        end
+
+        def on_complete
+          @command_parser << "\n"
+        end
+
+        def close
+          @command_parser.finish
+        end
+
+        private
+        def initialize_command_parser
+          @command_parser = Groonga::Command::Parser.new
+          @command_parser.on_command do |command|
+            @input.emit(command.name, command.arguments)
+          end
+          @command_parser.on_load_value do |command, value|
+            arguments = command.arguments.dup
+            arguments[:columns] = command.columns.join(", ")
+            arguments[:values] = Yajl::Encoder.encode([value])
+            @input.emit(command.name, arguments)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_groonga.rb

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "fileutils"
+require "cgi/util"
+
+module Fluent
+  class GroongaOutput < BufferedOutput
+    Plugin.register_output("groonga", self)
+
+    def initialize
+      super
+    end
+
+    config_param :protocol, :string, :default => "http"
+    config_param :table, :string, :default => nil
+
+    def configure(conf)
+      super
+      case @protocol
+      when "http"
+        @client = HTTPClient.new
+      when "gqtp"
+        @client = GQTPClient.new
+      when "command"
+        @client = CommandClient.new
+      end
+      @client.configure(conf)
+    end
+
+    def start
+      super
+      @client.start
+    end
+
+    def shutdown
+      super
+      @client.shutdown
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      chunk.msgpack_each do |tag, time, arguments|
+        if /\Agroonga\.command\./ =~ tag
+          name = $POSTMATCH
+          send_command(name, arguments)
+        else
+          store_chunk(chunk)
+        end
+      end
+    end
+
+    private
+    def send_command(name, arguments)
+      command_class = Groonga::Command.find(name)
+      command = command_class.new(name, arguments)
+      @client.send(command)
+    end
+
+    def store_chunk(chunk)
+      return if @table.nil?
+
+      values = []
+      chunk.each do |time, value|
+        values << value
+      end
+      arguments = {
+        "table" => @table,
+        "values" => Yajl::Enocder.encode(values),
+      }
+      send_command("load", arguments)
+    end
+
+    class HTTPClient
+      include Configurable
+
+      config_param :host, :string, :default => "localhost"
+      config_param :port, :integer, :default => 10041
+
+      def start
+        @loop = Coolio::Loop.new
+      end
+
+      def shutdown
+      end
+
+      def send(command)
+        client = GroongaHTTPClient.connect(@host, @port)
+        client.request("GET", command.to_uri_format)
+        @loop.attach(client)
+        @loop.run
+      end
+
+      class GroongaHTTPClient < Coolio::HttpClient
+        def on_body_data(data)
+        end
+      end
+    end
+
+    class GQTPClient
+      include Configurable
+
+      config_param :host, :string, :default => "localhost"
+      config_param :port, :integer, :default => 10041
+
+      def start
+        @loop = Coolio::Loop.new
+        @client = nil
+      end
+
+      def shutdown
+        return if @client.nil?
+        @client.send("shutdown") do
+          @loop.stop
+        end
+        @loop.run
+      end
+
+      def send(command)
+        @client ||= GQTP::Client.new(:host => @host,
+                                     :port => @port,
+                                     :connection => :coolio,
+                                     :loop => @loop)
+        @client.send(command.to_command_format) do |header, body|
+          @loop.stop
+        end
+        @loop.run
+      end
+    end
+
+    class CommandClient
+      include Configurable
+
+      config_param :groonga, :string, :default => "groonga"
+      config_param :database, :string
+      config_param :arguments, :default => [] do |value|
+        Shellwords.split(value)
+      end
+
+      def initialize
+        super
+      end
+
+      def configure(conf)
+        super
+      end
+
+      def start
+        run_groonga
+        wrap_io
+      end
+
+      def shutdown
+        @groonga_input.close
+        @groonga_output.close
+        @groonga_error.close
+        Process.waitpid(@pid)
+      end
+
+      def send(command)
+        body = nil
+        if command.name == "load"
+          body = command.arguments.delete(:values)
+        end
+        @groonga_input.write("#{command.to_uri_format}\n")
+        if body
+          body.each_line do |line|
+            @groonga_input.write("#{line}\n")
+          end
+        end
+        @loop.run
+      end
+
+      private
+      def run_groonga
+        env = {}
+        @input = IO.pipe("ASCII-8BIT")
+        @output = IO.pipe("ASCII-8BIT")
+        @error = IO.pipe("ASCII-8BIT")
+        input_fd = @input[0].to_i
+        output_fd = @output[1].to_i
+        options = {
+          input_fd => input_fd,
+          output_fd => output_fd,
+          :err => @error[1],
+        }
+        arguments = @arguments
+        arguments += [
+          "--input-fd", input_fd.to_s,
+          "--output-fd", output_fd.to_s,
+        ]
+        unless File.exist?(@database)
+          FileUtils.mkdir_p(File.dirname(@database))
+          arguments << "-n"
+        end
+        arguments << @database
+        @pid = spawn(env, @groonga, *arguments, options)
+        @input[0].close
+        @output[1].close
+        @error[1].close
+      end
+
+      def wrap_io
+        @loop = Coolio::Loop.new
+
+        @groonga_input = Coolio::IO.new(@input[1])
+        on_write_complete = lambda do
+          @loop.stop
+        end
+        @groonga_input.on_write_complete do
+          on_write_complete.call
+        end
+        @groonga_output = Coolio::IO.new(@output[0])
+        @groonga_error = Coolio::IO.new(@error[0])
+
+        @loop.attach(@groonga_input)
+        @loop.attach(@groonga_output)
+        @loop.attach(@groonga_error)
+      end
+    end
+  end
+end