fluent-plugin-groonga 1.0.1 → 1.0.2

data/README.md

@@ -53,6 +53,11 @@ contributed patches.)
 * English: [groonga-talk](https://lists.sourceforge.net/lists/listinfo/groonga-talk)
 * Japanese: [groonga-dev](http://lists.sourceforge.jp/mailman/listinfo/groonga-dev)
 
+## Source
+
+The repository for fluent-plugin-groonga is on
+[GitHub](https://github.com/groonga/fluent-plugin-groonga/).
+
 ## Thanks
 
 * ...

data/doc/text/news.md

@@ -2,6 +2,13 @@
 
 # News
 
+## 1.0.2: 2013-08-08
+
+### Improvements
+
+* Supported non-buffer mode.
+* Required gqtp gem >= 1.0.3.
+
 ## 1.0.1: 2012-12-29
 
 ### Improvements

data/fluent-plugin-groonga.gemspec

@@ -1,6 +1,6 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-groonga"
-  spec.version = "1.0.1"
+  spec.version = "1.0.2"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
   spec.summary = "Fluentd plugin collection for groonga users"
@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency("fluentd")
-  spec.add_runtime_dependency("gqtp")
+  spec.add_runtime_dependency("gqtp", ">= 1.0.3")
   spec.add_runtime_dependency("groonga-command")
 
   spec.add_development_dependency("rake")

data/lib/fluent/plugin/out_groonga.rb

@@ -18,73 +18,132 @@
 require "fileutils"
 
 module Fluent
-  class GroongaOutput < BufferedOutput
+  class GroongaOutput < Output
     Plugin.register_output("groonga", self)
 
     def initialize
       super
     end
 
+    BufferedOutput.config_params.each do |name, (block, options)|
+      if options[:type]
+        config_param(name, options[:type], options)
+      else
+        config_param(name, options, &block)
+      end
+    end
+
     config_param :protocol, :string, :default => "http"
     config_param :table, :string, :default => nil
 
     def configure(conf)
       super
-      case @protocol
-      when "http"
-        @client = HTTPClient.new
-      when "gqtp"
-        @client = GQTPClient.new
-      when "command"
-        @client = CommandClient.new
-      end
+      @client = create_client(@protocol)
       @client.configure(conf)
+
+      @emitter = Emitter.new(@client, @table)
+      @output = create_output(@buffer_type, @emitter)
+      @output.configure(conf)
     end
 
     def start
       super
       @client.start
+      @output.start
     end
 
     def shutdown
       super
+      @output.shutdown
       @client.shutdown
     end
 
-    def format(tag, time, record)
-      [tag, time, record].to_msgpack
+    def emit(tag, event_stream, chain)
+      @output.emit(tag, event_stream, chain)
+    end
+
+    def create_client(protocol)
+      case protocol
+      when "http"
+        HTTPClient.new
+      when "gqtp"
+        GQTPClient.new
+      when "command"
+        CommandClient.new
+      end
+    end
+
+    def create_output(buffer_type, emitter)
+      if buffer_type == "none"
+        RawGroongaOutput.new(emitter)
+      else
+        BufferedGroongaOutput.new(emitter)
+      end
     end
 
-    def write(chunk)
-      chunk.msgpack_each do |tag, time, arguments|
+    class Emitter
+      def initialize(client, table)
+        @client = client
+        @table = table
+      end
+
+      def emit(tag, record)
         if /\Agroonga\.command\./ =~ tag
           name = $POSTMATCH
-          send_command(name, arguments)
+          send_command(name, record)
         else
-          store_chunk(chunk)
+          store_chunk(data)
         end
       end
+
+      private
+      def send_command(name, arguments)
+        command_class = Groonga::Command.find(name)
+        command = command_class.new(name, arguments)
+        @client.send(command)
+      end
+
+      def store_chunk(value)
+        return if @table.nil?
+
+        values = [value]
+        arguments = {
+          "table" => @table,
+          "values" => Yajl::Enocder.encode(values),
+        }
+        send_command("load", arguments)
+      end
     end
 
-    private
-    def send_command(name, arguments)
-      command_class = Groonga::Command.find(name)
-      command = command_class.new(name, arguments)
-      @client.send(command)
+    class RawGroongaOutput < Output
+      def initialize(emitter)
+        @emitter = emitter
+        super()
+      end
+
+      def emit(tag, event_stream, chain)
+        event_stream.each do |time, record|
+          @emitter.emit(tag, record)
+        end
+        chain.next
+      end
     end
 
-    def store_chunk(chunk)
-      return if @table.nil?
+    class BufferedGroongaOutput < BufferedOutput
+      def initialize(emitter)
+        @emitter = emitter
+        super()
+      end
+
+      def format(tag, time, record)
+        [tag, time, record].to_msgpack
+      end
 
-      values = []
-      chunk.each do |time, value|
-        values << value
+      def write(chunk)
+        chunk.msgpack_each do |tag, time, record|
+          @emitter.emit(tag, record)
+        end
       end
-      arguments = {
-        "table" => @table,
-        "values" => Yajl::Enocder.encode(values),
-      }
-      send_command("load", arguments)
     end
 
     class HTTPClient
@@ -133,7 +192,7 @@ module Fluent
       end
 
       def send(command)
-        @client ||= GQTP::Client.new(:host => @host,
+        @client ||= GQTP::Client.new(:address => @host,
                                      :port => @port,
                                      :connection => :coolio,
                                      :loop => @loop)

data/test/test_output.rb

@@ -34,8 +34,7 @@ class GroongaOutputTest < Test::Unit::TestCase
 
   private
   def create_driver(tag)
-    driver = Fluent::Test::BufferedOutputTestDriver.new(Fluent::GroongaOutput,
-                                                        tag)
+    driver = Fluent::Test::OutputTestDriver.new(Fluent::GroongaOutput, tag)
     driver.configure(configuration)
     driver
   end
@@ -99,8 +98,9 @@ EOC
       def test_basic_command
         driver = create_driver("groonga.command.table_create")
         time = Time.parse("2012-10-26T08:45:42Z").to_i
-        driver.emit({"name" => "Users"}, time)
-        driver.run
+        driver.run do
+          driver.emit({"name" => "Users"}, time)
+        end
         # p @request_headers
         # p @request_body
       end
@@ -193,8 +193,9 @@ EOC
       def test_basic_command
         driver = create_driver("groonga.command.table_create")
         time = Time.parse("2012-10-26T08:45:42Z")
-        driver.emit({"name" => "Users"}, time)
-        driver.run
+        driver.run do
+          driver.emit({"name" => "Users"}, time)
+        end
         assert_equal([
                        [
                          "--input-fd", actual_input_fd,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-groonga
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-28 00:00:00.000000000 Z
+date: 2013-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: groonga-command
   requirement: !ruby/object:Gem::Requirement
@@ -169,15 +169,15 @@ files:
 - lib/fluent/plugin/out_groonga.rb
 - lib/fluent/plugin/in_groonga.rb
 - sample/gqtp.conf
-- sample/command.conf
 - sample/http.conf
+- sample/command.conf
+- doc/text/configuration.md
 - doc/text/news.md
 - doc/text/constitution.md
 - doc/text/lgpl-2.1.txt
-- doc/text/configuration.md
+- test/test_output.rb
 - test/run-test.rb
 - test/test_input.rb
-- test/test_output.rb
 homepage: https://github.com/groonga/fluent-plugin-groonga
 licenses: []
 post_install_message: 
@@ -190,18 +190,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1000993838467277328
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1000993838467277328
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -209,7 +203,7 @@ signing_key:
 specification_version: 3
 summary: Fluentd plugin collection for groonga users
 test_files:
+- test/test_output.rb
 - test/run-test.rb
 - test/test_input.rb
-- test/test_output.rb
 has_rdoc: