fluent-plugin-hbase 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +4 -0
- data/README.rdoc +42 -4
- data/VERSION +1 -1
- data/example/fluent.conf +10 -4
- data/lib/fluent/plugin/out_hbase.rb +12 -0
- data/test/out_hbase.rb +41 -6
- metadata +4 -4
data/ChangeLog
CHANGED
data/README.rdoc
CHANGED
@@ -15,19 +15,57 @@ Simply use RubyGems:
|
|
15
15
|
<match pattern>
|
16
16
|
type hbase
|
17
17
|
|
18
|
-
|
19
|
-
|
18
|
+
include_time_key TRUE_OR_FALSE
|
19
|
+
time_key KEY_IN_RECORD
|
20
|
+
include_tag_key TRUE_OR_FALSE
|
21
|
+
tag_key KEY_IN_RECORD
|
20
22
|
fields_to_columns_mapping MAPPING_FROM_JSON_FIELDS_TO_HBASE_COLUMNS
|
21
23
|
hbase_host YOUR_HBASE_HOST
|
22
24
|
hbase_port YOUR_HBASE_PORT
|
23
25
|
hbase_table YOUR_HBASE_TABLE_NAME
|
26
|
+
|
27
|
+
# DEPRECATED
|
28
|
+
# The below configuration keys are deprecated to
|
29
|
+
# stay along with fluentd's well-known configuration keys.
|
30
|
+
# Instead, use:
|
31
|
+
# - include_time_key
|
32
|
+
# - time_key
|
33
|
+
# - include_tag_key
|
34
|
+
# - tag_key
|
35
|
+
# - fields_to_columns_mapping
|
36
|
+
tag_column_name HBASE_COLUMN
|
37
|
+
time_column_name HBASE_COLUMN
|
24
38
|
</match>
|
25
39
|
|
26
|
-
[
|
40
|
+
[include_time_key (optional)] If you need to write each event's time to HBase, set this true
|
41
|
+
and add a mapping from the 'time_key' to HBase column to the 'fields_to_columns_mapping' configuration.
|
42
|
+
|
43
|
+
The default is false.
|
44
|
+
|
45
|
+
[time_key (optional)] The key where each event's time is kept for mapping to a HBase column.
|
46
|
+
|
47
|
+
You MUST add a mapping from the key configured here to a HBase column,
|
48
|
+
to actually write times to HBase.
|
49
|
+
|
50
|
+
The default is "time".
|
51
|
+
|
52
|
+
[include_tag_key (optional)] If you need to write each event's tag to HBase, set this true
|
53
|
+
and add a mapping from the 'tag_key' to HBase column to the 'fields_to_columns_mapping' configuration.
|
54
|
+
|
55
|
+
The default is false.
|
56
|
+
|
57
|
+
[tag_key (optional)] The key where each event's tag is kept for mapping to a HBase column.
|
58
|
+
|
59
|
+
You MUST add a mapping from the key configured here to a HBase column,
|
60
|
+
to actually write tags to HBase.
|
61
|
+
|
62
|
+
The default is "tag".
|
63
|
+
|
64
|
+
[tag_column_name (deprecated)] The HBase column to save the tag attached to each Fleuntd event log,
|
27
65
|
in the format "[Column family name]:[Column name]".
|
28
66
|
For example, to save tags to the column "c" in the column family "cf", use "cf:c".
|
29
67
|
|
30
|
-
[time_column_name (
|
68
|
+
[time_column_name (deprecated)] The HBase column to save the time each Fluentd event log was sent,
|
31
69
|
in the format "[Column family name]:[Colum name]".
|
32
70
|
For example, to save the time to the column "t" in the column family "cf", use "cf:t".
|
33
71
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.1
|
data/example/fluent.conf
CHANGED
@@ -82,15 +82,21 @@
|
|
82
82
|
# compress gz
|
83
83
|
#</match>
|
84
84
|
|
85
|
+
<match fluent.*>
|
86
|
+
type stdout
|
87
|
+
</match>
|
88
|
+
|
85
89
|
<match **>
|
86
90
|
type hbase
|
87
91
|
|
88
|
-
# Just to write to HBase rapidly
|
92
|
+
# Just to write to HBase rapidly for testing purpose
|
93
|
+
# Make this bigger in production
|
89
94
|
flush_interval 1
|
90
95
|
|
91
|
-
|
92
|
-
|
93
|
-
|
96
|
+
include_tag_key true
|
97
|
+
include_time_key true
|
98
|
+
time_format %Y-%m-%dT%H:%M:%S.%L%:z
|
99
|
+
fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag=>event:tag,time=>event:time
|
94
100
|
hbase_host localhost
|
95
101
|
hbase_port 9090
|
96
102
|
hbase_table events
|
@@ -8,6 +8,18 @@ module Fluent
|
|
8
8
|
require 'massive_record'
|
9
9
|
end
|
10
10
|
|
11
|
+
# Format dates with ISO 8606 by default
|
12
|
+
# http://www.w3.org/TR/NOTE-datetime
|
13
|
+
config_param :time_format, :string, :default => '%Y-%m-%dT%H:%M:%S.%L%:z'
|
14
|
+
|
15
|
+
include SetTagKeyMixin
|
16
|
+
config_set_default :include_tag_key, false
|
17
|
+
config_set_default :tag_key, nil
|
18
|
+
|
19
|
+
include SetTimeKeyMixin
|
20
|
+
config_set_default :include_time_key, false
|
21
|
+
config_set_default :time_key, nil
|
22
|
+
|
11
23
|
config_param :tag_column_name, :string, :default => nil
|
12
24
|
config_param :time_column_name, :string, :default => nil
|
13
25
|
config_param :fields_to_columns_mapping, :string
|
data/test/out_hbase.rb
CHANGED
@@ -9,7 +9,11 @@ class HBaseOutputTest < Test::Unit::TestCase
|
|
9
9
|
CONFIG = %[
|
10
10
|
tag_column_name event:tag
|
11
11
|
time_column_name event:time
|
12
|
-
|
12
|
+
include_time_key true
|
13
|
+
time_key time2
|
14
|
+
include_tag_key true
|
15
|
+
tag_key tag2
|
16
|
+
fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2
|
13
17
|
hbase_host localhost
|
14
18
|
hbase_port 9090
|
15
19
|
hbase_table events
|
@@ -34,10 +38,33 @@ class HBaseOutputTest < Test::Unit::TestCase
|
|
34
38
|
d = create_driver
|
35
39
|
assert_equal 'event:tag', d.instance.tag_column_name
|
36
40
|
assert_equal 'event:time', d.instance.time_column_name
|
37
|
-
assert_equal 'foo=>event:foo,iam.nested=>event:nested', d.instance.fields_to_columns_mapping
|
41
|
+
assert_equal 'foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2', d.instance.fields_to_columns_mapping
|
38
42
|
assert_equal 'localhost', d.instance.hbase_host
|
39
43
|
assert_equal 9090, d.instance.hbase_port
|
40
44
|
assert_equal 'events', d.instance.hbase_table
|
45
|
+
assert_equal true, d.instance.include_tag_key
|
46
|
+
assert_equal 'tag2', d.instance.tag_key
|
47
|
+
assert_equal true, d.instance.include_time_key
|
48
|
+
assert_equal '%Y-%m-%dT%H:%M:%S.%L%:z', d.instance.time_format
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_alternative_time_format_config
|
52
|
+
alt_conf = %[
|
53
|
+
tag_column_name event:tag
|
54
|
+
time_column_name event:time
|
55
|
+
include_time_key true
|
56
|
+
time_key time2
|
57
|
+
include_tag_key true
|
58
|
+
tag_key tag2
|
59
|
+
fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested,tag2=>event:tag2,time2=>event:time2
|
60
|
+
hbase_host localhost
|
61
|
+
hbase_port 9090
|
62
|
+
hbase_table events
|
63
|
+
buffer_type memory
|
64
|
+
time_format %Y-%m-%d
|
65
|
+
]
|
66
|
+
d = create_driver(conf = alt_conf)
|
67
|
+
assert_equal '%Y-%m-%d', d.instance.time_format
|
41
68
|
end
|
42
69
|
|
43
70
|
def test_format
|
@@ -69,14 +96,18 @@ class HBaseOutputTest < Test::Unit::TestCase
|
|
69
96
|
"event:tag" => "test",
|
70
97
|
"event:time" => time_in_int,
|
71
98
|
"event:foo" => "foo1",
|
72
|
-
"event:nested" => "nested1"
|
99
|
+
"event:nested" => "nested1",
|
100
|
+
"event:tag2" => "test",
|
101
|
+
"event:time2" => "2011-01-02T13:14:15.000+00:00"
|
73
102
|
}.to_msgpack
|
74
103
|
|
75
104
|
expected2 = {
|
76
105
|
"event:tag" => "test",
|
77
106
|
"event:time" => time_in_int,
|
78
107
|
"event:foo" => "foo2",
|
79
|
-
"event:nested" => "nested2"
|
108
|
+
"event:nested" => "nested2",
|
109
|
+
"event:tag2" => "test",
|
110
|
+
"event:time2" => "2011-01-02T13:14:15.000+00:00"
|
80
111
|
}.to_msgpack
|
81
112
|
|
82
113
|
d.expect_format expected1
|
@@ -114,14 +145,18 @@ class HBaseOutputTest < Test::Unit::TestCase
|
|
114
145
|
"event:tag" => "test",
|
115
146
|
"event:time" => time_in_int,
|
116
147
|
"event:foo" => "foo1",
|
117
|
-
"event:nested" => "nested1"
|
148
|
+
"event:nested" => "nested1",
|
149
|
+
"event:tag2" => "test",
|
150
|
+
"event:time2" => "2011-01-02T13:14:15.000+00:00"
|
118
151
|
}.to_msgpack
|
119
152
|
|
120
153
|
expected2 = {
|
121
154
|
"event:tag" => "test",
|
122
155
|
"event:time" => time_in_int,
|
123
156
|
"event:foo" => "foo2",
|
124
|
-
"event:nested" => "nested2"
|
157
|
+
"event:nested" => "nested2",
|
158
|
+
"event:tag2" => "test",
|
159
|
+
"event:time2" => "2011-01-02T13:14:15.000+00:00"
|
125
160
|
}.to_msgpack
|
126
161
|
|
127
162
|
# HBaseOutputTest#write returns chunk.read
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-hbase
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-11
|
12
|
+
date: 2012-12-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -91,7 +91,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
91
91
|
version: '0'
|
92
92
|
segments:
|
93
93
|
- 0
|
94
|
-
hash:
|
94
|
+
hash: -479372999590250441
|
95
95
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
96
96
|
none: false
|
97
97
|
requirements:
|
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
100
|
version: '0'
|
101
101
|
segments:
|
102
102
|
- 0
|
103
|
-
hash:
|
103
|
+
hash: -479372999590250441
|
104
104
|
requirements: []
|
105
105
|
rubyforge_project:
|
106
106
|
rubygems_version: 1.8.24
|