fluent-plugin-groonga 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bebd98779b93ff88f2c2411c411c0caf9b98cf8
-  data.tar.gz: 36072342f0b721a770888da4b1aa303741ad6a20
+  metadata.gz: f225a28b0bd052b37c5180b7707ac898f550ca57
+  data.tar.gz: 0c93017fa42a9a2edac9b50d97d0e3d3e5cf0a94
 SHA512:
-  metadata.gz: 944e0061c0a7d200d1e64babcb2363e36eeb319ab36f908a19df15ef3d6a485d15af7208fcb1b609d6bcf7778b399e29fbc84838675afa75a816d607ebb44d78
-  data.tar.gz: 263111e96f80aa571cf8e765a54a35d7095eee541d32f248fe0a7aed7c50e7f2737523a3c77fe8af71faf7f51494ea265f81abde511c5de9158d8510bfc71ded
+  metadata.gz: deb8587173008cf6ddc636aebe1fd7c2eb41ed01b806e91c020af81c1dc38eeccdbac196391ecc2289354458d3172707db85f9d9865a7690bc18f8ff00e917df
+  data.tar.gz: 737f5f9db324b14199f4e2b1734d7bc76e1008b80906eba78febe99f0aae91a428a1db69d809715661473bbc4e4252557be4f75512796f42df6f178195eb89ef

data/doc/text/configuration.md

@@ -32,6 +32,24 @@ Here are available parameters:
 
     * default: `10041`
 
+  * `command_name_position`: It specifies where Groonga command's
+    name.
+
+    If `tag` is specified, the plugin puts Groonga command's name to
+    tag as `groonga.command.#{COMMAND_NAME}` format and record
+    is the arguments of the command.
+
+    If `record` is specified, the plugin puts both Groonga command's
+    name and arguments to record as `{"name": "#{COMMAND_NAME}",
+    "arguments": {...}}` format. Tag is always `groonga.command`.
+
+    `record` is suitable when you want to implement replication. If
+    you `tag` for replication, Groonga command's order may be changed.
+
+    * Available values: `tag`, `record`
+
+    * default: `tag`
+
   * `emit_commands`: TODO
 
 Here is an example:
@@ -43,6 +61,7 @@ Here is an example:
       port 10041
       real_host 192.168.0.1
       real_port 10041
+      command_name_position record
     </source>
 
 ## The `groonga` output plugin

data/doc/text/constitution.md

@@ -66,19 +66,21 @@ Here is an example configuration file:
     # For master Groonga server
     <source>
       @type groonga
-      protocol gqtp          # Or use the below line
+      protocol gqtp                # Or use the below line
       # protocol http
-      bind 127.0.0.1         # For client side Fluentd
-      # bind 192.168.0.1     # For master Groonga server side Fluentd
+      bind 127.0.0.1               # For client side Fluentd
+      # bind 192.168.0.1           # For master Groonga server side Fluentd
       port 10041
-      real_host 192.168.29.1 # IP address of master Groonga server
-      real_port 10041        # Port number of master Groonga server
-      # real_port 20041      # Use different port number
-                             # for master Groonga server side Fluentd
+      real_host 192.168.29.1       # IP address of master Groonga server
+      real_port 10041              # Port number of master Groonga server
+      # real_port 20041            # Use different port number
+                                   # for master Groonga server side Fluentd
+
+      command_name_position record # To keep command order
     </source>
 
     # For slave Groonga server
-    <match groonga.command.*>
+    <match groonga.command.**>
       @type groonga
       protocol gqtp            # Or use the below line
       # protocol http          # You can use different protocol for
@@ -126,7 +128,7 @@ You cannot update data until fluentd is up.
 Here are recover steps when master Groonga server is down:
 
   1. Stop fluentd.
-  2. Run `grndump /PATH/TO/SLAVE/GROONGA/SERVER/DB >
+  2. Run `groonga /PATH/TO/SLAVE/GROONGA/SERVER/DB dump >
      SLAVE_GROONGA_DUMP.grn` on slave Groonga server host.
   3. Run `groonga -n /PATH/TO/MASTER/GROONGA/SERVER/DB <
      SLAVE_GROONGA_DUMP.grn` on master Groonga server.
@@ -139,7 +141,7 @@ You cannot update data until you finish to recover.
 
 Here are recover steps when slave Groonga server is down:
 
-  1. Run `grndump /PATH/TO/MASTER/GROONGA/SERVER/DB >
+  1. Run `groonga /PATH/TO/MASTER/GROONGA/SERVER/DB dump >
      MASTER_GROONGA_DUMP.grn` on master Groonga server host.
   2. Run `groonga -n /PATH/TO/SLAVE/GROONGA/SERVER/DB <
      MASTER_GROONGA_DUMP.grn` on slave Groonga server.
@@ -155,7 +157,7 @@ is full (see `buffer_queue_limit`) or fluentd gives up retrying (see
 Groonga server before those situations:
 
   1. Stop fluentd.
-  2. Run `grndump /PATH/TO/MASTER/GROONGA/SERVER/DB >
+  2. Run `groonga /PATH/TO/MASTER/GROONGA/SERVER/DB dump >
      MASTER_GROONGA_DUMP.grn` on master Groonga server host.
   3. Run `groonga -n /PATH/TO/SLAVE/GROONGA/SERVER/DB <
      MASTER_GROONGA_DUMP.grn` on slave Groonga server host.
@@ -194,19 +196,21 @@ Here is an example configuration file:
     # For master Groonga server
     <source>
       @type groonga
-      protocol gqtp          # Or use the below line
+      protocol gqtp                # Or use the below line
       # protocol http
-      bind 127.0.0.1         # For client side Fluentd
-      # bind 192.168.0.1     # For master Groonga server side Fluentd
+      bind 127.0.0.1               # For client side Fluentd
+      # bind 192.168.0.1           # For master Groonga server side Fluentd
       port 10041
-      real_host 192.168.29.1 # IP address of master Groonga server
-      real_port 10041        # Port number of master Groonga server
-      # real_port 20041      # Use different port number
-                             # for master Groonga server side fluentd
+      real_host 192.168.29.1       # IP address of master Groonga server
+      real_port 10041              # Port number of master Groonga server
+      # real_port 20041            # Use different port number
+                                   # for master Groonga server side fluentd
+
+      command_name_position record # To keep command order
     </source>
 
     # For slave Groonga servers
-    <match groonga.command.*>
+    <match groonga.command.**>
       @type copy
 
       # The first slave Groonga server

data/doc/text/news.md

@@ -2,6 +2,13 @@
 
 # News
 
+## 1.1.8: 2018-01-16
+
+### Improvements
+
+* in: `command_name_position`: Added a new parameter to control
+  command format. The default behavior isn't changed.
+
 ## 1.1.7: 2017-04-04
 
 ### Fixes

data/fluent-plugin-groonga.gemspec

@@ -1,6 +1,7 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2012-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2018  Yasuhiro Horimoto <horimoto@clear-code.com>
+# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,7 +18,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-groonga"
-  spec.version = "1.1.7"
+  spec.version = "1.1.8"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
   spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
@@ -34,7 +35,7 @@ Gem::Specification.new do |spec|
   spec.test_files += Dir.glob("test/**/*")
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency("fluentd")
+  spec.add_runtime_dependency("fluentd", ">= 0.12.20", "< 0.14.0")
   spec.add_runtime_dependency("groonga-client", ">= 0.1.0")
   spec.add_runtime_dependency("groonga-command-parser")
 

data/lib/fluent/plugin/in_groonga.rb

@@ -1,3 +1,4 @@
+# Copyright (C) 2018  Yasuhiro Horimoto <horimoto@clear-code.com>
 # Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -33,14 +34,8 @@ module Fluent
       super
     end
 
-    config_param :protocol, :default => :http do |value|
-      case value
-      when "http", "gqtp"
-        value.to_sym
-      else
-        raise ConfigError, "must be http or gqtp: <#{value}>"
-      end
-    end
+    config_param :protocol, :enum, :list => [:http, :gqtp], :default => :http
+    config_param :command_name_position, :enum, :list => [:tag, :record], :default => :tag
 
     def configure(conf)
       super
@@ -171,9 +166,20 @@ module Fluent
       def emit(command, params)
         normalized_command = command.split(".")[0]
         return unless emit_command?(normalized_command)
-        @input_plugin.router.emit("groonga.command.#{normalized_command}",
+        case @input_plugin.command_name_position
+        when :tag
+          tag = "groonga.command.#{normalized_command}"
+          record = params
+        else
+          tag = "groonga.command"
+          record = {
+            "name" => normalized_command,
+            "arguments" => params
+          }
+        end
+        @input_plugin.router.emit(tag,
                                   Engine.now,
-                                  params)
+                                  record)
       end
 
       private

data/lib/fluent/plugin/out_groonga.rb

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2012-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2018  Yasuhiro Horimoto <horimoto@clear-code.com>
+# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -29,15 +30,7 @@ module Fluent
       super
     end
 
-    config_param :protocol, :default => :http do |value|
-      case value
-      when "http", "gqtp", "command"
-        value.to_sym
-      else
-        raise ConfigError, "must be http, gqtp or command: <#{value}>"
-      end
-    end
-
+    config_param :protocol, :enum, :list => [:http, :gqtp, :command], :default => :http
     # alias is just for backward compatibility
     config_param :store_table, :string, :default => nil, :alias => :table
 
@@ -340,8 +333,7 @@ module Fluent
         @columns = {}
         column_list.each do |column|
           name = column.name
-          vector_p = column.flags.split("|").include?("COLUMN_VECTOR")
-          @columns[name] = Column.new(name, column.range, vector_p)
+          @columns[name] = Column.new(name, column.range, column.vector?)
           ensure_column_indexes(name)
         end
       end
@@ -575,13 +567,18 @@ module Fluent
         records = []
         chunk.msgpack_each do |message|
           tag, _, record = message
-          if /\Agroonga\.command\./ =~ tag
+          case tag
+          when /\Agroonga\.command\./
             name = $POSTMATCH
             unless records.empty?
               store_records(records)
               records.clear
             end
             @client.execute(name, record)
+          when "groonga.command"
+            name = record["name"]
+            arguments = record["arguments"]
+            @client.execute(name, arguments)
           else
             records << record
           end

data/sample/command.conf

@@ -4,9 +4,11 @@
 
   real_host 127.0.0.1
   real_port 20041
+
+#  command_name_position record
 </source>
 
-<match groonga.command.*>
+<match groonga.command.**>
   @type groonga
   protocol command
   database /tmp/groonga/db

data/sample/gqtp.conf

@@ -4,9 +4,11 @@
 
   real_host 127.0.0.1
   real_port 20043
+
+#  command_name_position record
 </source>
 
-<match groonga.command.*>
+<match groonga.command.**>
   @type groonga
   protocol gqtp
   host 127.0.0.1

data/sample/http.conf

@@ -4,9 +4,11 @@
 
   real_host 127.0.0.1
   real_port 20041
+
+#  command_name_position record
 </source>
 
-<match groonga.command.*>
+<match groonga.command.**>
   @type groonga
   protocol http
   host 127.0.0.1

data/test/test_input.rb

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2018  Yasuhiro Horimoto <horimoto@clear-code.com>
+# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -153,6 +154,59 @@ EOJ
       end
     end
 
+    sub_test_case("command_name_position") do
+      sub_test_case("record") do
+        def configuration
+          <<-CONFIGURATION
+            #{super}
+            command_name_position record
+          CONFIGURATION
+        end
+
+        def test_not_load
+          @real_response["Content-Type"] = "application/json"
+          @real_response.body = JSON.generate([[0, 0.0, 0.0], true])
+          @driver.expect_emit("groonga.command",
+                              @now,
+                              {
+                                "name" => "table_create",
+                                "arguments" => {
+                                  "name" => "Users",
+                                  "flags" => "TABLE_NO_KEY",
+                                }
+                              })
+          @driver.run do
+            get("/d/table_create", "name" => "Users", "flags" => "TABLE_NO_KEY")
+            assert_equal("200", @last_response.code)
+          end
+       end
+
+        def test_load
+          @real_response["Content-Type"] = "application/json"
+          @real_response.body = JSON.generate([[0, 0.0, 0.0], true])
+          json = <<-JSON
+[
+{"name": "Alice"},
+{"name": "Bob"}
+]
+          JSON
+          @driver.expect_emit("groonga.command",
+                      @now,
+                      {
+                        "name" => "load",
+                        "arguments" => {
+                          "table" => "Users",
+                          "values" => json,
+                        }
+                      })
+          @driver.run do
+            post("/d/load", json, "table" => "Users")
+            assert_equal("200", @last_response.code)
+          end
+        end
+      end
+    end
+
     private
     def get(path, parameters={})
       http = Net::HTTP.new(@host, @port)

data/test/test_output.rb

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2018  Yasuhiro Horimoto <horimoto@clear-code.com>
+# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -96,7 +97,7 @@ EOC
     end
 
     class CommandTest < self
-      def test_basic_command
+      def test_command_name_position_tag
         @response_body = JSON.generate([[0, 0.0, 0.0], true])
         driver = create_driver("groonga.command.table_create")
         time = Time.parse("2012-10-26T08:45:42Z").to_i
@@ -105,6 +106,21 @@ EOC
         assert_equal("/d/table_create?name=Users",
                      @request_parser.request_url)
       end
+
+      def test_command_name_position_record
+        @response_body = JSON.generate([[0, 0.0, 0.0], true])
+        driver = create_driver("groonga.command")
+        time = Time.parse("2012-10-26T08:45:42Z").to_i
+        driver.emit({
+                      "name" => "table_create",
+                      "arguments" => {
+                        "name" => "Users"
+                      }
+                    }, time)
+        driver.run
+        assert_equal("/d/table_create?name=Users",
+                     @request_parser.request_url)
+      end
     end
 
     class StoreTest < self
@@ -228,7 +244,7 @@ EOC
     end
 
     class CommandTest < self
-      def test_basic_command
+      def test_command_name_position_tag
         driver = create_driver("groonga.command.table_create")
         time = Time.parse("2012-10-26T08:45:42Z").to_i
         driver.emit({"name" => "Users"}, time)
@@ -246,6 +262,28 @@ EOC
                        actual_input,
                      ])
       end
+
+      def test_command_name_position_record
+        driver = create_driver("groonga.command")
+        time = Time.parse("2012-10-26T08:45:42Z").to_i
+        driver.emit({
+                      "name" => "table_create",
+                      "arguments" => {"name" => "Users"}
+                    }, time)
+        driver.run
+        assert_equal([
+                       [
+                         "--input-fd", actual_input_fd,
+                         "--output-fd", actual_output_fd,
+                         "-n", @database_path,
+                       ],
+                       "/d/table_create?name=Users\n",
+                     ],
+                     [
+                       actual_command_line,
+                       actual_input,
+                     ])
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-groonga
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-04 00:00:00.000000000 Z
+date: 2018-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.20
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.20
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
   name: groonga-client
   requirement: !ruby/object:Gem::Requirement
@@ -186,15 +192,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.5.2.1
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to store data into Groonga and implement Groonga replication
   system.
 test_files:
 - test/test_output.rb
-- test/output/test_table_index_definition.rb
-- test/output/test_type_guesser.rb
-- test/output/test_table_definition.rb
 - test/run-test.rb
+- test/output/test_table_definition.rb
+- test/output/test_type_guesser.rb
+- test/output/test_table_index_definition.rb
 - test/test_input.rb