fluent-plugin-hbase 0.1.0 → 0.1.1

data/ChangeLog

@@ -4,3 +4,7 @@ Release 0.1.0 - 2012/11/19
 
   * First release
 
+Release 0.1.1. - 2012/12/11
+
+  * Stay with fluentd's well-known configuration keys for tag and time logging
+  * Support for logging each event's time in formatted text

data/README.rdoc

@@ -15,19 +15,57 @@ Simply use RubyGems:
     <match pattern>
       type hbase
 
-      tag_column_name HBASE_COLUMN
-      time_column_name HBASE_COLUMN
+      include_time_key TRUE_OR_FALSE
+      time_key KEY_IN_RECORD
+      include_tag_key TRUE_OR_FALSE
+      tag_key KEY_IN_RECORD
       fields_to_columns_mapping MAPPING_FROM_JSON_FIELDS_TO_HBASE_COLUMNS
       hbase_host YOUR_HBASE_HOST
       hbase_port YOUR_HBASE_PORT
       hbase_table YOUR_HBASE_TABLE_NAME
+
+      # DEPRECATED
+      # The below configuration keys are deprecated to
+      # stay along with fluentd's well-known configuration keys.
+      # Instead, use:
+      # - include_time_key
+      # - time_key
+      # - include_tag_key
+      # - tag_key
+      # - fields_to_columns_mapping
+      tag_column_name HBASE_COLUMN
+      time_column_name HBASE_COLUMN
     </match>
 
-[tag_column_name (required)] The HBase column to save the tag attached to each Fleuntd event log,
+[include_time_key (optional)] If you need to write each event's time to HBase, set this true
+and add a mapping from the 'time_key' to HBase column to the 'fields_to_columns_mapping' configuration.
+
+The default is false.
+
+[time_key (optional)] The key where each event's time is kept for mapping to a HBase column.
+
+You MUST add a mapping from the key configured here to a HBase column,
+to actually write times to HBase.
+
+The default is "time".
+
+[include_tag_key (optional)] If you need to write each event's tag to HBase, set this true
+and add a mapping from the 'tag_key' to HBase column to the 'fields_to_columns_mapping' configuration.
+
+The default is false.
+
+[tag_key (optional)] The key where each event's tag is kept for mapping to a HBase column.
+
+You MUST add a mapping from the key configured here to a HBase column,
+to actually write tags to HBase.
+
+The default is "tag".
+
+[tag_column_name (deprecated)] The HBase column to save the tag attached to each Fleuntd event log,
 in the format "[Column family name]:[Column name]".
 For example, to save tags to the column "c" in the column family "cf", use "cf:c".
 
-[time_column_name (required)] The HBase column to save the time each Fluentd event log was sent,
+[time_column_name (deprecated)] The HBase column to save the time each Fluentd event log was sent,
 in the format "[Column family name]:[Colum name]".
 For example, to save the time to the column "t" in the column family "cf", use "cf:t".
 

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/example/fluent.conf

@@ -82,15 +82,21 @@
 #  compress gz
 #</match>
 
+<match fluent.*>
+  type stdout
+</match>
+
 <match **>
   type hbase
 
-  # Just to write to HBase rapidly
+  # Just to write to HBase rapidly for testing purpose
+  # Make this bigger in production
   flush_interval 1
 
-  tag_column_name event:tag
-  time_column_name event:time
-  fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested
+  include_tag_key true
+  include_time_key true
+  time_format %Y-%m-%dT%H:%M:%S.%L%:z
+  fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag=>event:tag,time=>event:time
   hbase_host localhost
   hbase_port 9090
   hbase_table events

data/lib/fluent/plugin/out_hbase.rb

@@ -8,6 +8,18 @@ module Fluent
       require 'massive_record'
     end
 
+    # Format dates with ISO 8606 by default
+    # http://www.w3.org/TR/NOTE-datetime
+    config_param :time_format, :string, :default => '%Y-%m-%dT%H:%M:%S.%L%:z'
+
+    include SetTagKeyMixin
+    config_set_default :include_tag_key, false
+    config_set_default :tag_key, nil
+
+    include SetTimeKeyMixin
+    config_set_default :include_time_key, false
+    config_set_default :time_key, nil
+
     config_param :tag_column_name, :string, :default => nil
     config_param :time_column_name, :string, :default => nil
     config_param :fields_to_columns_mapping, :string

data/test/out_hbase.rb

@@ -9,7 +9,11 @@ class HBaseOutputTest < Test::Unit::TestCase
   CONFIG = %[
     tag_column_name event:tag
     time_column_name event:time
-    fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested
+    include_time_key true
+    time_key time2
+    include_tag_key true
+    tag_key tag2
+    fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2
     hbase_host localhost
     hbase_port 9090
     hbase_table events
@@ -34,10 +38,33 @@ class HBaseOutputTest < Test::Unit::TestCase
     d = create_driver
     assert_equal 'event:tag', d.instance.tag_column_name
     assert_equal 'event:time', d.instance.time_column_name
-    assert_equal 'foo=>event:foo,iam.nested=>event:nested', d.instance.fields_to_columns_mapping
+    assert_equal 'foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2', d.instance.fields_to_columns_mapping
     assert_equal 'localhost', d.instance.hbase_host
     assert_equal 9090, d.instance.hbase_port
     assert_equal 'events', d.instance.hbase_table
+    assert_equal true, d.instance.include_tag_key
+    assert_equal 'tag2', d.instance.tag_key
+    assert_equal true, d.instance.include_time_key
+    assert_equal '%Y-%m-%dT%H:%M:%S.%L%:z', d.instance.time_format
+  end
+
+  def test_alternative_time_format_config
+    alt_conf = %[
+      tag_column_name event:tag
+      time_column_name event:time
+      include_time_key true
+      time_key time2
+      include_tag_key true
+      tag_key tag2
+      fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2
+      hbase_host localhost
+      hbase_port 9090
+      hbase_table events
+      buffer_type memory
+      time_format %Y-%m-%d
+    ]
+    d = create_driver(conf = alt_conf)
+    assert_equal '%Y-%m-%d', d.instance.time_format
   end
 
   def test_format
@@ -69,14 +96,18 @@ class HBaseOutputTest < Test::Unit::TestCase
         "event:tag" => "test",
         "event:time" => time_in_int,
         "event:foo" => "foo1",
-        "event:nested" => "nested1"
+        "event:nested" => "nested1",
+        "event:tag2" => "test",
+        "event:time2" => "2011-01-02T13:14:15.000+00:00"
     }.to_msgpack
 
     expected2 = {
         "event:tag" => "test",
         "event:time" => time_in_int,
         "event:foo" => "foo2",
-        "event:nested" => "nested2"
+        "event:nested" => "nested2",
+        "event:tag2" => "test",
+        "event:time2" => "2011-01-02T13:14:15.000+00:00"
     }.to_msgpack
 
     d.expect_format expected1
@@ -114,14 +145,18 @@ class HBaseOutputTest < Test::Unit::TestCase
         "event:tag" => "test",
         "event:time" => time_in_int,
         "event:foo" => "foo1",
-        "event:nested" => "nested1"
+        "event:nested" => "nested1",
+        "event:tag2" => "test",
+        "event:time2" => "2011-01-02T13:14:15.000+00:00"
     }.to_msgpack
 
     expected2 = {
         "event:tag" => "test",
         "event:time" => time_in_int,
         "event:foo" => "foo2",
-        "event:nested" => "nested2"
+        "event:nested" => "nested2",
+        "event:tag2" => "test",
+        "event:time2" => "2011-01-02T13:14:15.000+00:00"
     }.to_msgpack
 
     # HBaseOutputTest#write returns chunk.read

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-hbase
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-21 00:00:00.000000000 Z
+date: 2012-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -91,7 +91,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1566921651398845132
+      hash: -479372999590250441
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1566921651398845132
+      hash: -479372999590250441
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24