fluent-plugin-hbase 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/ChangeLog CHANGED
@@ -4,3 +4,7 @@ Release 0.1.0 - 2012/11/19
4
4
 
5
5
  * First release
6
6
 
7
+ Release 0.1.1. - 2012/12/11
8
+
9
+ * Stay with fluentd's well-known configuration keys for tag and time logging
10
+ * Support for logging each event's time in formatted text
data/README.rdoc CHANGED
@@ -15,19 +15,57 @@ Simply use RubyGems:
15
15
  <match pattern>
16
16
  type hbase
17
17
 
18
- tag_column_name HBASE_COLUMN
19
- time_column_name HBASE_COLUMN
18
+ include_time_key TRUE_OR_FALSE
19
+ time_key KEY_IN_RECORD
20
+ include_tag_key TRUE_OR_FALSE
21
+ tag_key KEY_IN_RECORD
20
22
  fields_to_columns_mapping MAPPING_FROM_JSON_FIELDS_TO_HBASE_COLUMNS
21
23
  hbase_host YOUR_HBASE_HOST
22
24
  hbase_port YOUR_HBASE_PORT
23
25
  hbase_table YOUR_HBASE_TABLE_NAME
26
+
27
+ # DEPRECATED
28
+ # The below configuration keys are deprecated to
29
+ # stay along with fluentd's well-known configuration keys.
30
+ # Instead, use:
31
+ # - include_time_key
32
+ # - time_key
33
+ # - include_tag_key
34
+ # - tag_key
35
+ # - fields_to_columns_mapping
36
+ tag_column_name HBASE_COLUMN
37
+ time_column_name HBASE_COLUMN
24
38
  </match>
25
39
 
26
- [tag_column_name (required)] The HBase column to save the tag attached to each Fleuntd event log,
40
+ [include_time_key (optional)] If you need to write each event's time to HBase, set this true
41
+ and add a mapping from the 'time_key' to HBase column to the 'fields_to_columns_mapping' configuration.
42
+
43
+ The default is false.
44
+
45
+ [time_key (optional)] The key where each event's time is kept for mapping to a HBase column.
46
+
47
+ You MUST add a mapping from the key configured here to a HBase column,
48
+ to actually write times to HBase.
49
+
50
+ The default is "time".
51
+
52
+ [include_tag_key (optional)] If you need to write each event's tag to HBase, set this true
53
+ and add a mapping from the 'tag_key' to HBase column to the 'fields_to_columns_mapping' configuration.
54
+
55
+ The default is false.
56
+
57
+ [tag_key (optional)] The key where each event's tag is kept for mapping to a HBase column.
58
+
59
+ You MUST add a mapping from the key configured here to a HBase column,
60
+ to actually write tags to HBase.
61
+
62
+ The default is "tag".
63
+
64
+ [tag_column_name (deprecated)] The HBase column to save the tag attached to each Fleuntd event log,
27
65
  in the format "[Column family name]:[Column name]".
28
66
  For example, to save tags to the column "c" in the column family "cf", use "cf:c".
29
67
 
30
- [time_column_name (required)] The HBase column to save the time each Fluentd event log was sent,
68
+ [time_column_name (deprecated)] The HBase column to save the time each Fluentd event log was sent,
31
69
  in the format "[Column family name]:[Colum name]".
32
70
  For example, to save the time to the column "t" in the column family "cf", use "cf:t".
33
71
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.0
1
+ 0.1.1
data/example/fluent.conf CHANGED
@@ -82,15 +82,21 @@
82
82
  # compress gz
83
83
  #</match>
84
84
 
85
+ <match fluent.*>
86
+ type stdout
87
+ </match>
88
+
85
89
  <match **>
86
90
  type hbase
87
91
 
88
- # Just to write to HBase rapidly
92
+ # Just to write to HBase rapidly for testing purpose
93
+ # Make this bigger in production
89
94
  flush_interval 1
90
95
 
91
- tag_column_name event:tag
92
- time_column_name event:time
93
- fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested
96
+ include_tag_key true
97
+ include_time_key true
98
+ time_format %Y-%m-%dT%H:%M:%S.%L%:z
99
+ fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag=>event:tag,time=>event:time
94
100
  hbase_host localhost
95
101
  hbase_port 9090
96
102
  hbase_table events
@@ -8,6 +8,18 @@ module Fluent
8
8
  require 'massive_record'
9
9
  end
10
10
 
11
+ # Format dates with ISO 8606 by default
12
+ # http://www.w3.org/TR/NOTE-datetime
13
+ config_param :time_format, :string, :default => '%Y-%m-%dT%H:%M:%S.%L%:z'
14
+
15
+ include SetTagKeyMixin
16
+ config_set_default :include_tag_key, false
17
+ config_set_default :tag_key, nil
18
+
19
+ include SetTimeKeyMixin
20
+ config_set_default :include_time_key, false
21
+ config_set_default :time_key, nil
22
+
11
23
  config_param :tag_column_name, :string, :default => nil
12
24
  config_param :time_column_name, :string, :default => nil
13
25
  config_param :fields_to_columns_mapping, :string
data/test/out_hbase.rb CHANGED
@@ -9,7 +9,11 @@ class HBaseOutputTest < Test::Unit::TestCase
9
9
  CONFIG = %[
10
10
  tag_column_name event:tag
11
11
  time_column_name event:time
12
- fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested
12
+ include_time_key true
13
+ time_key time2
14
+ include_tag_key true
15
+ tag_key tag2
16
+ fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2
13
17
  hbase_host localhost
14
18
  hbase_port 9090
15
19
  hbase_table events
@@ -34,10 +38,33 @@ class HBaseOutputTest < Test::Unit::TestCase
34
38
  d = create_driver
35
39
  assert_equal 'event:tag', d.instance.tag_column_name
36
40
  assert_equal 'event:time', d.instance.time_column_name
37
- assert_equal 'foo=>event:foo,iam.nested=>event:nested', d.instance.fields_to_columns_mapping
41
+ assert_equal 'foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2', d.instance.fields_to_columns_mapping
38
42
  assert_equal 'localhost', d.instance.hbase_host
39
43
  assert_equal 9090, d.instance.hbase_port
40
44
  assert_equal 'events', d.instance.hbase_table
45
+ assert_equal true, d.instance.include_tag_key
46
+ assert_equal 'tag2', d.instance.tag_key
47
+ assert_equal true, d.instance.include_time_key
48
+ assert_equal '%Y-%m-%dT%H:%M:%S.%L%:z', d.instance.time_format
49
+ end
50
+
51
+ def test_alternative_time_format_config
52
+ alt_conf = %[
53
+ tag_column_name event:tag
54
+ time_column_name event:time
55
+ include_time_key true
56
+ time_key time2
57
+ include_tag_key true
58
+ tag_key tag2
59
+ fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2
60
+ hbase_host localhost
61
+ hbase_port 9090
62
+ hbase_table events
63
+ buffer_type memory
64
+ time_format %Y-%m-%d
65
+ ]
66
+ d = create_driver(conf = alt_conf)
67
+ assert_equal '%Y-%m-%d', d.instance.time_format
41
68
  end
42
69
 
43
70
  def test_format
@@ -69,14 +96,18 @@ class HBaseOutputTest < Test::Unit::TestCase
69
96
  "event:tag" => "test",
70
97
  "event:time" => time_in_int,
71
98
  "event:foo" => "foo1",
72
- "event:nested" => "nested1"
99
+ "event:nested" => "nested1",
100
+ "event:tag2" => "test",
101
+ "event:time2" => "2011-01-02T13:14:15.000+00:00"
73
102
  }.to_msgpack
74
103
 
75
104
  expected2 = {
76
105
  "event:tag" => "test",
77
106
  "event:time" => time_in_int,
78
107
  "event:foo" => "foo2",
79
- "event:nested" => "nested2"
108
+ "event:nested" => "nested2",
109
+ "event:tag2" => "test",
110
+ "event:time2" => "2011-01-02T13:14:15.000+00:00"
80
111
  }.to_msgpack
81
112
 
82
113
  d.expect_format expected1
@@ -114,14 +145,18 @@ class HBaseOutputTest < Test::Unit::TestCase
114
145
  "event:tag" => "test",
115
146
  "event:time" => time_in_int,
116
147
  "event:foo" => "foo1",
117
- "event:nested" => "nested1"
148
+ "event:nested" => "nested1",
149
+ "event:tag2" => "test",
150
+ "event:time2" => "2011-01-02T13:14:15.000+00:00"
118
151
  }.to_msgpack
119
152
 
120
153
  expected2 = {
121
154
  "event:tag" => "test",
122
155
  "event:time" => time_in_int,
123
156
  "event:foo" => "foo2",
124
- "event:nested" => "nested2"
157
+ "event:nested" => "nested2",
158
+ "event:tag2" => "test",
159
+ "event:time2" => "2011-01-02T13:14:15.000+00:00"
125
160
  }.to_msgpack
126
161
 
127
162
  # HBaseOutputTest#write returns chunk.read
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-hbase
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-11-21 00:00:00.000000000 Z
12
+ date: 2012-12-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -91,7 +91,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
91
91
  version: '0'
92
92
  segments:
93
93
  - 0
94
- hash: 1566921651398845132
94
+ hash: -479372999590250441
95
95
  required_rubygems_version: !ruby/object:Gem::Requirement
96
96
  none: false
97
97
  requirements:
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
100
100
  version: '0'
101
101
  segments:
102
102
  - 0
103
- hash: 1566921651398845132
103
+ hash: -479372999590250441
104
104
  requirements: []
105
105
  rubyforge_project:
106
106
  rubygems_version: 1.8.24