fluent-plugin-groonga 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a9668e4de0a3638480de1d40b46d667d7cab260
-  data.tar.gz: e0698d86a5ce4eeeb038ce8987c68901b377c9b7
+  metadata.gz: 5947627fa2f465811a846d3cd3ced7a8ac367674
+  data.tar.gz: cc846ea63eca88bdadbb9d5dceab7b806e36762d
 SHA512:
-  metadata.gz: 7209bf79d6f9fe172c99ca550cdea907fb745c8bc9c0708e428410799e4f0697ed370678439016f2c636f06fb3a0b34551edc3000ec681122c29d5fcb6226f5c
-  data.tar.gz: eac848d799ece9eba22009a84dc08c87cf8147a64b385bf48b27814d1df4bbe9d262619dbcff2a4953624cd52296b1c068bd571ad83144c4aaebf30a3167da9b
+  metadata.gz: 386b7122fc93b2bba7f8fdfea1648ce66b7ad4e20929ad1ea7ea7fae0fd731d6168e6aac8c18567af14cfb621bbe1b95fe8d9903f11e7e2a78aaf53db18bf6a3
+  data.tar.gz: f56bf5c5848837eca8cc8de484c8493aad1467a3655594963670118ca3dfa14aa6fa326927f3e557fdb58175982beac9a39fda56f263ca852fa7ddb8bc4e2e8a

data/doc/text/news.md

@@ -2,6 +2,21 @@
 
 # News
 
+## 1.1.3: 2016-09-02
+
+### Improvements
+
+* Supported Fluentd 0.14.
+
+* Stopped to emit requests when real Groonga server returns error
+  responses. `load` request and `object_remove` request are
+  exceptions. They are always emitted if real Groonga server returns
+  error responses. Because they may be effected when Groonga server
+  returns error responses.
+
+* Updated the default emit target command list to reflect the recent
+  Groonga command list.
+
 ## 1.1.2: 2016-06-05
 
 ### Improvements

data/fluent-plugin-groonga.gemspec

@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-groonga"
-  spec.version = "1.1.2"
+  spec.version = "1.1.3"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
   spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."

data/lib/fluent/plugin/in_groonga.rb

@@ -24,6 +24,7 @@ require "http_parser"
 require "gqtp"
 require "groonga/command/parser"
 
+require "fluent/input"
 require "fluent/process"
 
 module Fluent
@@ -55,10 +56,12 @@ module Fluent
     end
 
     def start
+      super
       @input.start
     end
 
     def shutdown
+      super
       @input.shutdown
     end
 
@@ -86,13 +89,29 @@ module Fluent
       config_param :real_host, :string
       config_param :real_port, :integer, :default => nil
       DEFAULT_EMIT_COMMANDS = [
-        /\Atable_/,
-        /\Acolumn_/,
+        "clearlock",
+        "column_copy",
+        "column_create",
+        "column_remove",
+        "column_rename",
+        "config_delete",
+        "config_set",
         "delete",
-        /\Aplugin_/,
+        "load",
+        "lock_acquire",
+        "lock_clear",
+        "lock_release",
+        "logical_table_remove",
+        "object_remove",
+        "plugin_register",
+        "plugin_unregister",
         "register",
+        "reindex",
+        "table_copy",
+        "table_create",
+        "table_remove",
+        "table_rename",
         "truncate",
-        "load",
       ]
       config_param :emit_commands, :default => DEFAULT_EMIT_COMMANDS do |value|
         commands = value.split(/\s*,\s*/)
@@ -152,8 +171,9 @@ module Fluent
       end
 
       def emit(command, params)
-        return unless emit_command?(command)
-        @input_plugin.router.emit("groonga.command.#{command}",
+        normalized_command = command.split(".")[0]
+        return unless emit_command?(normalized_command)
+        @input_plugin.router.emit("groonga.command.#{normalized_command}",
                                   Engine.now,
                                   params)
       end
@@ -191,13 +211,17 @@ module Fluent
         end
 
         def on_connect
-          @parser = HTTP::Parser.new(self)
           @repeater = @input.create_repeater(self)
+          @repeater.on_connect_failed do
+            close
+          end
+          @request_handler = RequestHandler.new(@input, @repeater)
+          @response_handler = ResponseHandler.new(self)
         end
 
         def on_read(data)
           begin
-            @parser << data
+            @request_handler << data
           rescue HTTP::Parser::Error
             $log.error("[input][groonga][error] " +
                        "failed to parse HTTP request:",
@@ -207,8 +231,55 @@ module Fluent
           end
         end
 
+        def write(data)
+          @response_handler << data
+          super
+        end
+
+        def on_response_complete(response)
+          if need_emit?(response)
+            @input.emit(@request_handler.command,
+                        @request_handler.params)
+          end
+          on_write_complete do
+            @repeater.close
+          end
+        end
+
+        private
+        def need_emit?(response)
+          case @request_handler.command
+          when "load", "object_remove"
+            return true
+          end
+
+          case response
+          when Array
+            return_code = response[0][0]
+            return_code.zero?
+          else
+            false
+          end
+        end
+      end
+
+      class RequestHandler
+        attr_reader :command
+        attr_reader :params
+        def initialize(input, repeater)
+          @input = input
+          @repeater = repeater
+          @parser = Http::Parser.new(self)
+        end
+
+        def <<(chunk)
+          @parser << chunk
+        end
+
         def on_message_begin
           @body = ""
+          @command = nil
+          @params = nil
         end
 
         def on_headers_complete(headers)
@@ -244,8 +315,52 @@ module Fluent
             if command == "load"
               params["values"] = @body unless @body.empty?
             end
-            @input.emit(command, params)
+            @command = command
+            @params = params
+          end
+        end
+      end
+
+      class ResponseHandler
+        def initialize(handler)
+          @handler = handler
+          @parser = Http::Parser.new(self)
+        end
+
+        def <<(chunk)
+          @parser << chunk
+        end
+
+        def on_message_begin
+          @body = ""
+          @content_type = nil
+        end
+
+        def on_headers_complete(headers)
+          headers.each do |name, value|
+            case name
+            when /\AContent-Type\z/i
+              @content_type = value
+            end
+          end
+        end
+
+        def on_body(chunk)
+          @body << chunk
+        end
+
+        def on_message_complete
+          case @content_type
+          when /\Aapplication\/json\z/
+            response = JSON.parse(@body)
+          when /\Aapplication\/x-msgpack\z/
+            response = MessagePack.unpack(@body)
+          when /\Atext\/x-groonga-command-list/
+            response = @body
+          else
+            response = nil
           end
+          @handler.on_response_complete(response)
         end
       end
     end

data/test/test_input.rb

@@ -101,11 +101,16 @@ EOC
     end
 
     def test_target_command
+      @real_response["Content-Type"] = "application/json"
+      @real_response.body = JSON.generate([[0, 0.0, 0.0], true])
       @driver.expect_emit("groonga.command.table_create",
                           @now,
-                          {"name" => "Users"})
+                          {
+                            "name" => "Users",
+                            "flags" => "TABLE_NO_KEY",
+                          })
       @driver.run do
-        get("/d/table_create", "name" => "Users")
+        get("/d/table_create", "name" => "Users", "flags" => "TABLE_NO_KEY")
         assert_equal("200", @last_response.code)
       end
     end
@@ -119,6 +124,8 @@ EOC
     end
 
     def test_load
+      @real_response["Content-Type"] = "application/json"
+      @real_response.body = JSON.generate([[0, 0.0, 0.0], 2])
       json = <<-EOJ
 [
 {"name": "Alice"},

data/test/test_output.rb

@@ -34,7 +34,8 @@ class GroongaOutputTest < Test::Unit::TestCase
 
   private
   def create_driver(tag)
-    driver = Fluent::Test::OutputTestDriver.new(Fluent::GroongaOutput, tag)
+    driver = Fluent::Test::BufferedOutputTestDriver.new(Fluent::GroongaOutput,
+                                                        tag)
     driver.configure(configuration)
     driver
   end
@@ -99,9 +100,8 @@ EOC
         @response_body = JSON.generate([[0, 0.0, 0.0], true])
         driver = create_driver("groonga.command.table_create")
         time = Time.parse("2012-10-26T08:45:42Z").to_i
-        driver.run do
-          driver.emit({"name" => "Users"}, time)
-        end
+        driver.emit({"name" => "Users"}, time)
+        driver.run
         assert_equal("/d/table_create?name=Users",
                      @request_parser.request_url)
       end
@@ -119,9 +119,8 @@ EOC
         @response_body = JSON.generate([[0, 0.0, 0.0], [1]])
         driver = create_driver("log")
         time = Time.parse("2012-10-26T08:45:42Z").to_i
-        driver.run do
-          driver.emit({"message" => "1st message"}, time)
-        end
+        driver.emit({"message" => "1st message"}, time)
+        driver.run
         assert_equal("/d/load?table=Logs",
                      @request_parser.request_url)
         assert_equal([{"message" => "1st message"}],
@@ -132,10 +131,9 @@ EOC
         @response_body = JSON.generate([[0, 0.0, 0.0], [2]])
         driver = create_driver("log")
         time = Time.parse("2012-10-26T08:45:42Z").to_i
-        driver.run do
-          driver.emit({"message" => "1st message"}, time)
-          driver.emit({"message" => "2nd message"}, time + 1)
-        end
+        driver.emit({"message" => "1st message"}, time)
+        driver.emit({"message" => "2nd message"}, time + 1)
+        driver.run
         assert_equal("/d/load?table=Logs",
                      @request_parser.request_url)
         assert_equal([
@@ -232,10 +230,9 @@ EOC
     class CommandTest < self
       def test_basic_command
         driver = create_driver("groonga.command.table_create")
-        time = Time.parse("2012-10-26T08:45:42Z")
-        driver.run do
-          driver.emit({"name" => "Users"}, time)
-        end
+        time = Time.parse("2012-10-26T08:45:42Z").to_i
+        driver.emit({"name" => "Users"}, time)
+        driver.run
         assert_equal([
                        [
                          "--input-fd", actual_input_fd,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-groonga
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-05 00:00:00.000000000 Z
+date: 2016-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd