fluent-plugin-groonga 1.0.8 → 1.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5c57a846ebfc9321fd1ab1b79ce794e6aefbfff
-  data.tar.gz: fd6dbc5627188c8cc20d221b27ca802d755dd2ea
+  metadata.gz: bf6b4e7487748c2f6c1afd542bcc01b28cae47d5
+  data.tar.gz: 7a84873889cb7b1fa6a48d6e7335f314c4f8756f
 SHA512:
-  metadata.gz: d9699527fea08ac73e293215282f327f51cafa880eae410ff0ffada7568144032480481e55405ae4217654e7cd503301fc29ade309e9cc76619e5dd322e74d8d
-  data.tar.gz: a9acb39dd64613ff97eb4336e21b7b34e28e9d8876285bb69b6e8fc9e3effbb9d5470aeab16478b5ae09bfeb97807de598d0f6a7c6e9fab081800c208f19cb71
+  metadata.gz: 6c6d1479f6d93ac5eab837923878e3ae973c77700d461b787bb44e0f86048bf93b7900c7651302f830e6fb497afcfa7c28c0af95813bd4f0eb1cf553514e8e15
+  data.tar.gz: 0615408e6acc183ae5d7de3cd2e2ea51e445b2988c83766985230951dcff6a37350733bfd8d12bcc61e2227b41c45ec1b6eb234225cc957bbe45f1f3cb9169d4

data/doc/text/news.md

@@ -2,6 +2,21 @@
 
 # News
 
+## 1.0.9: 2014-11-20
+
+### Improves
+
+* out: Added log message with host, port and command name on Groonga
+  command execution error.
+* out: Added `WITH_POSITION` index flag automatically when it is
+  needed.
+* out: Supported creating index for existing column.
+
+### Fixes
+
+* out: Fixed a bug that needless `WITH_SECTION` flags is used.
+* out: Fixed a bug that wrong index name is used.
+
 ## 1.0.8: 2014-11-05
 
 ### Fixes

data/fluent-plugin-groonga.gemspec

@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-groonga"
-  spec.version = "1.0.8"
+  spec.version = "1.0.9"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
   spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."

data/lib/fluent/plugin/out_groonga.rb

@@ -122,7 +122,28 @@ module Fluent
       end
     end
 
+    module DefinitionParseMethods
+      private
+      def parse_flags(flags)
+        if flags.is_a?(Array)
+          flags
+        else
+          flags.strip.split(/\s*\|\s*/)
+        end
+      end
+
+      def parse_items(items)
+        if items.is_a?(Array)
+          items
+        else
+          items.strip.split(/\s*,\s*/)
+        end
+      end
+    end
+
     class TableDefinition
+      include DefinitionParseMethods
+
       def initialize(raw)
         @raw = raw
       end
@@ -192,24 +213,9 @@ module Fluent
         arguments
       end
 
-      private
-      def parse_flags(flags)
-        if flags.is_a?(Array)
-          flags
-        else
-          flags.strip.split(/\s*\|\s*/)
-        end
-      end
-
-      def parse_items(items)
-        if items.is_a?(Array)
-          items
-        else
-          items.strip.split(/\s*,\s*/)
-        end
-      end
-
       class IndexDefinition
+        include DefinitionParseMethods
+
         def initialize(table, raw)
           @table = table
           @raw = raw
@@ -224,7 +230,7 @@ module Fluent
         end
 
         def source_columns
-          @raw[:source_columns]
+          parse_items(@raw[:source_columns])
         end
 
         def flags
@@ -240,7 +246,7 @@ module Fluent
             "name"   => name,
             "flags"  => flags.join("|"),
             "type"   => source_table,
-            "source" => source_columns,
+            "source" => source_columns.join(","),
           }
         end
       end
@@ -279,7 +285,7 @@ module Fluent
         @client = client
         @table_name = table_name
         @mappings = mappings
-        @table = nil
+        @taget_table = nil
         @columns = nil
       end
 
@@ -305,19 +311,24 @@ module Fluent
 
       private
       def ensure_table
-        return if @table
+        return if @target_table
 
-        table_list = @client.execute("table_list")
-        target_table = table_list.find do |table|
-          table.name == @table_name
+        @tables = {}
+        @client.execute("table_list").collect do |table|
+          name = table.name
+          options = {
+            :default_tokenizer => table.default_tokenizer,
+          }
+          @tables[name] = Table.new(table.name, options)
         end
-        if target_table
-          @table = Table.new(@table_name, target_table.domain)
-        else
+
+        @target_table = @tables[@table_name]
+        unless @target_table
           @client.execute("table_create",
                           "name"  => @table_name,
                           "flags" => "TABLE_NO_KEY")
-          @table = Table.new(@table_name, nil)
+          @target_table = Table.new(@table_name)
+          @tables[@table_name] = @target_table
         end
       end
 
@@ -327,10 +338,10 @@ module Fluent
         column_list = @client.execute("column_list", "table" => @table_name)
         @columns = {}
         column_list.each do |column|
+          name = column.name
           vector_p = column.flags.split("|").include?("COLUMN_VECTOR")
-          @columns[column.name] = Column.new(column.name,
-                                             column.range,
-                                             vector_p)
+          @columns[name] = Column.new(name, column.range, vector_p)
+          ensure_column_indexes(name)
         end
       end
 
@@ -354,21 +365,38 @@ module Fluent
                         "name" => name,
                         "flags" => flags,
                         "type" => value_type)
-        if mapping
-          mapping.indexes.each do |index|
-            index_flags = ["COLUMN_INDEX", index[:flags]].compact
-            @client.execute("column_create",
-                            "table" => index[:table],
-                            "name" => index[:table],
-                            "flags" => index_flags.join("|"),
-                            "type" => @table_name,
-                            "source" => name)
-          end
-        end
+        ensure_column_indexes(name)
 
         Column.new(name, value_type, vector_p)
       end
 
+      def ensure_column_indexes(name)
+        mapping = @mappings.find do |_mapping|
+          _mapping.name == name
+        end
+        return if mapping.nil?
+
+        mapping.indexes.each do |index|
+          table = @tables[index[:table]]
+          if table
+            column_list = @client.execute("column_list", "table" => table.name)
+            exist = column_list.any? {|column| column.name == index[:name]}
+            next if exist
+          end
+
+          index_flags = ["COLUMN_INDEX"]
+          index_flags << "WITH_POSITION" if table and table.default_tokenizer
+          index_flags << index[:flags] if index[:flags]
+
+          @client.execute("column_create",
+                          "table" => index[:table],
+                          "name" => index[:name],
+                          "flags" => index_flags.join("|"),
+                          "type" => @table_name,
+                          "source" => name)
+        end
+      end
+
       class TypeGuesser
         def initialize(sample_values)
           @sample_values = sample_values
@@ -485,9 +513,21 @@ module Fluent
       end
 
       class Table
-        def initialize(name, key_type)
+        attr_reader :name
+        attr_reader :flags
+        attr_reader :domain
+        attr_reader :range
+        attr_reader :default_tokenizer
+        attr_reader :normalizer
+        attr_reader :token_filters
+        def initialize(name, options={})
           @name = name
-          @key_type = key_type
+          @flags             = options[:flags]
+          @domain            = options[:domain]
+          @range             = options[:range]
+          @default_tokenizer = options[:default_tokenizer]
+          @normalizer        = options[:normalizer]
+          @token_filters     = options[:token_filters]
         end
       end
 
@@ -579,7 +619,17 @@ module Fluent
                                         :host     => @host,
                                         :port     => @port,
                                         :backend  => :synchronous)
-        response = @client.execute(command)
+        response = nil
+        begin
+          response = @client.execute(command)
+        rescue Groonga::Client::Error
+          $log.error("[output][groonga][error]",
+                     :protocol => @protocol,
+                     :host => @host,
+                     :port => @port,
+                     :command_name => name)
+          raise
+        end
         unless response.success?
           $log.error("[output][groonga][error]",
                      :status_code => response.status_code,

data/sample/store-apache.conf

@@ -169,7 +169,6 @@
     <index>
       table Terms
       name logs_message_index
-      flags WITH_POSITION
     </index>
   </mapping>
 

data/sample/store-syslog.conf

@@ -0,0 +1,92 @@
+# Required plugins:
+#  * fluent-plugin-config-expander
+#  * fluent-plugin-forest
+#  * fluent-plugin-parser
+#  * fluent-plugin-record-reformer
+#  * fluent-plugin-groonga
+
+<source>
+  type forward
+</source>
+
+<source>
+  type config_expander
+  <config>
+    type tail
+    path /var/log/messages
+    pos_file /var/log/td-agent/messages.pos
+    tag raw.messages.log.${hostname}
+    format none
+  </config>
+</source>
+
+<match raw.*.log.**>
+  type forest
+  subtype parser
+
+  <template>
+    key_name message
+  </template>
+
+  <case raw.messages.log.**>
+    remove_prefix raw
+    format syslog
+  </case>
+</match>
+
+<match *.log.*.**>
+  type record_reformer
+  enable_ruby false
+
+  tag ${tag_parts[1]}
+
+  <record>
+    host ${tag_suffix[2]}
+    type ${tag_parts[0]}
+    timestamp ${time}
+  </record>
+</match>
+
+<match log>
+  type groonga
+  store_table Logs
+
+  protocol http
+  host 127.0.0.1
+
+  buffer_type file
+  buffer_path /var/spool/td-agent/buffer/groonga
+  flush_interval 1
+
+  <table>
+    name Terms
+    flags TABLE_PAT_KEY
+    key_type ShortText
+    default_tokenizer TokenBigram
+    normalizer NormalizerAuto
+  </table>
+
+  <table>
+    name Timestamps
+    flags TABLE_PAT_KEY
+    key_type Time
+  </table>
+
+  <mapping>
+    name timestamp
+    type Time
+    <index>
+      table Timestamps
+      name logs_index
+    </index>
+  </mapping>
+
+  <mapping>
+    name message
+    type Text
+    <index>
+      table Terms
+      name logs_message_index
+    </index>
+  </mapping>
+</match>

data/test/output/test_table_index_definition.rb

@@ -0,0 +1,95 @@
+# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "fluent/plugin/out_groonga"
+
+class OutputTypeTableIndexDefinitionTest < Test::Unit::TestCase
+  def table_definition
+    raw = {
+      :name              => "Terms",
+      :default_tokenizer => "TokenBigram",
+    }
+    Fluent::GroongaOutput::TableDefinition.new(raw)
+  end
+
+  def definition(raw={})
+    default_raw = {
+      :source_columns => "title",
+    }
+    raw = default_raw.merge(raw)
+    Fluent::GroongaOutput::TableDefinition::IndexDefinition.new(table_definition,
+                                                                raw)
+  end
+
+  sub_test_case "readers" do
+    sub_test_case "\#name" do
+      test "specified" do
+        assert_equal("logs_index", definition(:name => "logs_index").name)
+      end
+    end
+
+    sub_test_case "\#flags" do
+      test "default" do
+        assert_equal(["COLUMN_INDEX", "WITH_POSITION"],
+                     definition.flags)
+      end
+
+      test "multiple source columns" do
+        assert_equal(["COLUMN_INDEX", "WITH_POSITION", "WITH_SECTION"],
+                     definition(:source_columns => "title,content").flags)
+      end
+    end
+
+    sub_test_case "\#source_table" do
+      test "specified" do
+        assert_equal("Logs",
+                     definition(:source_table => "Logs").source_table)
+      end
+    end
+
+    sub_test_case "\#source_columns" do
+      test "one" do
+        assert_equal(["title"],
+                     definition(:source_columns => "title").source_columns)
+      end
+
+      test "multiple" do
+        raw = {
+          :source_columns => "title,content",
+        }
+        assert_equal(["title", "content"],
+                     definition(raw).source_columns)
+      end
+    end
+  end
+
+  sub_test_case "\#to_create_arguments" do
+    test "full" do
+      raw = {
+        :name           => "logs_index",
+        :source_table   => "Logs",
+        :source_columns => "title, content",
+      }
+      assert_equal({
+                     "table"  => "Terms",
+                     "name"   => "logs_index",
+                     "flags"  => "COLUMN_INDEX|WITH_POSITION|WITH_SECTION",
+                     "type"   => "Logs",
+                     "source" => "title,content",
+                   },
+                   definition(raw).to_create_arguments)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-groonga
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.0.9
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-05 00:00:00.000000000 Z
+date: 2014-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -158,8 +158,10 @@ files:
 - sample/gqtp.conf
 - sample/http.conf
 - sample/store-apache.conf
+- sample/store-syslog.conf
 - sample/store.conf
 - test/output/test_table_definition.rb
+- test/output/test_table_index_definition.rb
 - test/output/test_type_guesser.rb
 - test/run-test.rb
 - test/test_input.rb
@@ -193,6 +195,7 @@ test_files:
 - test/test_output.rb
 - test/test_input.rb
 - test/run-test.rb
+- test/output/test_table_index_definition.rb
 - test/output/test_type_guesser.rb
 - test/output/test_table_definition.rb
 has_rdoc: