fluent-plugin-kafka 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +3 -3
- data/fluent-plugin-kafka.gemspec +2 -1
- data/lib/fluent/plugin/in_kafka.rb +4 -0
- data/lib/fluent/plugin/out_kafka.rb +6 -0
- data/lib/fluent/plugin/out_kafka_buffered.rb +6 -0
- metadata +29 -25
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 65f16e3cb305ed9abe2c9969c929b1f63a73ad9c
|
4
|
+
data.tar.gz: fa75eb12ac1fc0d3596972c9d61c6a2d879e2af3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 74255febdd2152a9cc8665c1d4415bd51b07f3af4089fc82c2de11e2001465f3149e0c8b7fed6bfc5db9a9d729b7f251968a2ac5858bb6c4ce0a435603f8f867
|
7
|
+
data.tar.gz: 6581acbfc5fb1b00df4510f7786e616b8142cc6e168c8bff548752748db629ca3e99df3d3e7b39a429567fb8f41ae9df5bbbf676c12b611561ce1fb2c5f9d061
|
data/README.md
CHANGED
@@ -26,7 +26,7 @@ Or install it yourself as:
|
|
26
26
|
host <broker host>
|
27
27
|
port <broker port: default=9092>
|
28
28
|
topics <listening topics(separate with comma',')>
|
29
|
-
format <input text type (text|json|ltsv)>
|
29
|
+
format <input text type (text|json|ltsv|msgpack)>
|
30
30
|
add_prefix <tag prefix (Optional)>
|
31
31
|
add_suffix <tag suffix (Optional)>
|
32
32
|
</source>
|
@@ -37,7 +37,7 @@ Or install it yourself as:
|
|
37
37
|
type kafka
|
38
38
|
brokers <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
|
39
39
|
default_topic <output topic>
|
40
|
-
output_data_type (json|ltsv|attr:<record name>)
|
40
|
+
output_data_type (json|ltsv|msgpack|attr:<record name>)
|
41
41
|
output_include_tag (true|false) :default => false
|
42
42
|
output_include_time (true|false) :default => false
|
43
43
|
</match>
|
@@ -50,7 +50,7 @@ Or install it yourself as:
|
|
50
50
|
default_topic <output topic>
|
51
51
|
flush_interval <flush interval (sec) :default => 60>
|
52
52
|
buffer_type (file|memory)
|
53
|
-
output_data_type (json|ltsv|attr:<record name>)
|
53
|
+
output_data_type (json|ltsv|msgpack|attr:<record name>)
|
54
54
|
output_include_tag (true|false) :default => false
|
55
55
|
output_include_time (true|false) :default => false
|
56
56
|
</match>
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,9 +12,10 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.0.
|
15
|
+
gem.version = '0.0.7'
|
16
16
|
gem.add_dependency 'fluentd'
|
17
17
|
gem.add_dependency 'poseidon'
|
18
18
|
gem.add_dependency 'ltsv'
|
19
19
|
gem.add_dependency 'yajl-ruby'
|
20
|
+
gem.add_dependency 'msgpack'
|
20
21
|
end
|
@@ -31,6 +31,8 @@ class KafkaInput < Input
|
|
31
31
|
require 'yajl'
|
32
32
|
when 'ltsv'
|
33
33
|
require 'ltsv'
|
34
|
+
when 'msgpack'
|
35
|
+
require 'msgpack'
|
34
36
|
end
|
35
37
|
end
|
36
38
|
|
@@ -110,6 +112,8 @@ class KafkaInput < Input
|
|
110
112
|
parsed_record = Yajl::Parser.parse(record)
|
111
113
|
when 'ltsv'
|
112
114
|
parsed_record = LTSV.parse(record)
|
115
|
+
when 'msgpack'
|
116
|
+
parsed_record = MessagePack.unpack(record)
|
113
117
|
when 'text'
|
114
118
|
parsed_record = record
|
115
119
|
end
|
@@ -25,6 +25,8 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
25
25
|
require 'yajl'
|
26
26
|
when 'ltsv'
|
27
27
|
require 'ltsv'
|
28
|
+
when 'msgpack'
|
29
|
+
require 'msgpack'
|
28
30
|
end
|
29
31
|
|
30
32
|
@f_separator = case @field_separator
|
@@ -38,6 +40,8 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
38
40
|
nil
|
39
41
|
elsif @output_data_type == 'ltsv'
|
40
42
|
nil
|
43
|
+
elsif @output_data_type == 'msgpack'
|
44
|
+
nil
|
41
45
|
elsif @output_data_type =~ /^attr:(.*)$/
|
42
46
|
$1.split(',').map(&:strip).reject(&:empty?)
|
43
47
|
else
|
@@ -61,6 +65,8 @@ class Fluent::KafkaOutput < Fluent::Output
|
|
61
65
|
Yajl::Encoder.encode(record)
|
62
66
|
when 'ltsv'
|
63
67
|
LTSV.dump(record)
|
68
|
+
when 'msgpack'
|
69
|
+
record.to_msgpack
|
64
70
|
else
|
65
71
|
record.to_s
|
66
72
|
end
|
@@ -26,6 +26,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
26
26
|
require 'yajl'
|
27
27
|
when 'ltsv'
|
28
28
|
require 'ltsv'
|
29
|
+
when 'msgpack'
|
30
|
+
require 'msgpack'
|
29
31
|
end
|
30
32
|
|
31
33
|
@f_separator = case @field_separator
|
@@ -39,6 +41,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
39
41
|
nil
|
40
42
|
elsif @output_data_type == 'ltsv'
|
41
43
|
nil
|
44
|
+
elsif @output_data_type == 'msgpack'
|
45
|
+
nil
|
42
46
|
elsif @output_data_type =~ /^attr:(.*)$/
|
43
47
|
$1.split(',').map(&:strip).reject(&:empty?)
|
44
48
|
else
|
@@ -65,6 +69,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
|
65
69
|
Yajl::Encoder.encode(record)
|
66
70
|
when 'ltsv'
|
67
71
|
LTSV.dump(record)
|
72
|
+
when 'msgpack'
|
73
|
+
record.to_msgpack
|
68
74
|
else
|
69
75
|
record.to_s
|
70
76
|
end
|
metadata
CHANGED
@@ -1,78 +1,83 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.7
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Hidemasa Togashi
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2014-
|
11
|
+
date: 2014-07-13 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: fluentd
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - '>='
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - '>='
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: poseidon
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - '>='
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - '>='
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: ltsv
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - '>='
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '0'
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - '>='
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: '0'
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: yajl-ruby
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - '>='
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: '0'
|
70
62
|
type: :runtime
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: msgpack
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
76
81
|
- !ruby/object:Gem::Version
|
77
82
|
version: '0'
|
78
83
|
description: Fluentd plugin for Apache Kafka > 0.8
|
@@ -94,27 +99,26 @@ files:
|
|
94
99
|
- test/plugin/test_out_kafka.rb
|
95
100
|
homepage: https://github.com/htgc/fluent-plugin-kafka
|
96
101
|
licenses: []
|
102
|
+
metadata: {}
|
97
103
|
post_install_message:
|
98
104
|
rdoc_options: []
|
99
105
|
require_paths:
|
100
106
|
- lib
|
101
107
|
required_ruby_version: !ruby/object:Gem::Requirement
|
102
|
-
none: false
|
103
108
|
requirements:
|
104
|
-
- -
|
109
|
+
- - '>='
|
105
110
|
- !ruby/object:Gem::Version
|
106
111
|
version: '0'
|
107
112
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
-
none: false
|
109
113
|
requirements:
|
110
|
-
- -
|
114
|
+
- - '>='
|
111
115
|
- !ruby/object:Gem::Version
|
112
116
|
version: '0'
|
113
117
|
requirements: []
|
114
118
|
rubyforge_project:
|
115
|
-
rubygems_version:
|
119
|
+
rubygems_version: 2.0.14
|
116
120
|
signing_key:
|
117
|
-
specification_version:
|
121
|
+
specification_version: 4
|
118
122
|
summary: Fluentd plugin for Apache Kafka > 0.8
|
119
123
|
test_files:
|
120
124
|
- test/helper.rb
|