fluent-plugin-mqtt 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-mqtt.gemspec +1 -1
- data/lib/fluent/plugin/in_mqtt.rb +32 -30
- data/lib/fluent/plugin/out_mqtt.rb +9 -4
- data/test/helper.rb +1 -0
- data/test/plugin/test_in_mqtt.rb +26 -25
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 19e8964bfff6ad942926e4e1607b91a099f8a1f9
|
4
|
+
data.tar.gz: dcf3560aca07e73b59eb1ff319620977dc63ac6b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a86199765c2bdcd8309c2e79c0ea4a073a443acef5a55574ee8fc951d8974c45116f0fdaf5ae94d6c72c6477bb565746be676de52f6ca8ec44f9836cd382b378
|
7
|
+
data.tar.gz: b3d1b82362c57f3860338946b44f5af6419a133173caee3fc8b0fbf6f2978aa8f5b1a8189bc8f912797911d9bc5b1bdf9588a400a9dd50169fae9496b0ac3f4f
|
data/fluent-plugin-mqtt.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-mqtt"
|
7
|
-
spec.version = "0.0.
|
7
|
+
spec.version = "0.0.8"
|
8
8
|
spec.authors = ["Yuuna Kurita"]
|
9
9
|
spec.email = ["yuuna.m@gmail.com"]
|
10
10
|
spec.summary = %q{fluentd input plugin for mqtt server}
|
@@ -1,23 +1,23 @@
|
|
1
|
-
|
1
|
+
require 'mqtt'
|
2
|
+
require 'fluent/plugin/input'
|
3
|
+
require 'fluent/plugin/parser'
|
4
|
+
|
5
|
+
module Fluent::Plugin
|
2
6
|
class MqttInput < Input
|
3
|
-
Plugin.register_input('mqtt', self)
|
7
|
+
Fluent::Plugin.register_input('mqtt', self)
|
8
|
+
|
9
|
+
helpers :thread, :inject, :compat_parameters, :parser
|
10
|
+
|
11
|
+
DEFAULT_PARSER_TYPE = 'none'
|
4
12
|
|
5
|
-
include Fluent::SetTagKeyMixin
|
6
13
|
config_set_default :include_tag_key, false
|
7
|
-
|
8
|
-
include Fluent::SetTimeKeyMixin
|
9
14
|
config_set_default :include_time_key, true
|
10
15
|
|
11
|
-
# Define `router` method of v0.12 to support v0.10 or earlier
|
12
|
-
unless method_defined?(:router)
|
13
|
-
define_method("router") { Fluent::Engine }
|
14
|
-
end
|
15
|
-
|
16
|
-
|
17
16
|
config_param :port, :integer, :default => 1883
|
18
17
|
config_param :bind, :string, :default => '127.0.0.1'
|
19
18
|
config_param :topic, :string, :default => '#'
|
20
|
-
config_param :format, :string, :default =>
|
19
|
+
config_param :format, :string, :default => DEFAULT_PARSER_TYPE
|
20
|
+
config_param :client_id, :string, :default => nil
|
21
21
|
config_param :username, :string, :default => nil
|
22
22
|
config_param :password, :string, :default => nil
|
23
23
|
config_param :ssl, :bool, :default => nil
|
@@ -25,31 +25,33 @@ module Fluent
|
|
25
25
|
config_param :key, :string, :default => nil
|
26
26
|
config_param :cert, :string, :default => nil
|
27
27
|
|
28
|
-
|
28
|
+
config_section :parse do
|
29
|
+
config_set_default :@type, DEFAULT_PARSER_TYPE
|
30
|
+
end
|
29
31
|
|
30
32
|
def configure(conf)
|
33
|
+
compat_parameters_convert(conf, :inject, :parser)
|
31
34
|
super
|
32
|
-
@bind ||= conf['bind']
|
33
|
-
@topic ||= conf['topic']
|
34
|
-
@port ||= conf['port']
|
35
|
-
|
36
35
|
configure_parser(conf)
|
37
36
|
end
|
38
37
|
|
39
38
|
def configure_parser(conf)
|
40
|
-
@parser =
|
41
|
-
@parser.configure(conf)
|
39
|
+
@parser = parser_create(usage: 'in_mqtt_parser', type: @format, conf: conf)
|
42
40
|
end
|
43
41
|
|
44
42
|
# Return [time (if not available return now), message]
|
45
43
|
def parse(message)
|
46
|
-
|
44
|
+
@parser.parse(message) {|time, record|
|
45
|
+
return (time || Fluent::Engine.now), record
|
46
|
+
}
|
47
47
|
end
|
48
48
|
|
49
49
|
def start
|
50
|
-
|
50
|
+
super
|
51
|
+
log.debug "start mqtt #{@bind}"
|
51
52
|
opts = {host: @bind,
|
52
53
|
port: @port}
|
54
|
+
opts[:client_id] = @client_id if @client_id
|
53
55
|
opts[:username] = @username if @username
|
54
56
|
opts[:password] = @password if @password
|
55
57
|
opts[:ssl] = @ssl if @ssl
|
@@ -59,25 +61,26 @@ module Fluent
|
|
59
61
|
@connect = MQTT::Client.connect(opts)
|
60
62
|
@connect.subscribe(@topic)
|
61
63
|
|
62
|
-
|
64
|
+
thread_create(:in_mqtt_worker) do
|
63
65
|
@connect.get do |topic,message|
|
64
66
|
topic.gsub!("/","\.")
|
65
|
-
|
67
|
+
log.debug "#{topic}: #{message}"
|
66
68
|
begin
|
67
|
-
|
69
|
+
time, record = self.parse(message)
|
70
|
+
record = inject_values_to_record(topic, time, record)
|
68
71
|
rescue Exception => e
|
69
|
-
|
72
|
+
log.error e
|
70
73
|
end
|
71
|
-
emit topic,
|
74
|
+
emit topic, record, time
|
72
75
|
end
|
73
76
|
end
|
74
77
|
end
|
75
78
|
|
76
|
-
|
79
|
+
|
77
80
|
def emit topic, message, time = Fluent::Engine.now
|
78
81
|
if message.class == Array
|
79
82
|
message.each do |data|
|
80
|
-
|
83
|
+
log.debug "#{topic}: #{data}"
|
81
84
|
router.emit(topic , time , data)
|
82
85
|
end
|
83
86
|
else
|
@@ -86,9 +89,8 @@ module Fluent
|
|
86
89
|
end
|
87
90
|
|
88
91
|
def shutdown
|
89
|
-
@thread.kill
|
90
92
|
@connect.disconnect
|
93
|
+
super
|
91
94
|
end
|
92
95
|
end
|
93
96
|
end
|
94
|
-
|
@@ -13,12 +13,14 @@ module Fluent
|
|
13
13
|
config_param :bind, :string, :default => '127.0.0.1'
|
14
14
|
config_param :topic, :string, :default => 'td-agent'
|
15
15
|
config_param :format, :string, :default => 'none'
|
16
|
+
config_param :client_id, :string, :default => nil
|
16
17
|
config_param :username, :string, :default => nil
|
17
18
|
config_param :password, :string, :default => nil
|
18
19
|
config_param :ssl, :bool, :default => nil
|
19
20
|
config_param :ca, :string, :default => nil
|
20
21
|
config_param :key, :string, :default => nil
|
21
22
|
config_param :cert, :string, :default => nil
|
23
|
+
config_param :retain, :string, :default => true
|
22
24
|
|
23
25
|
require 'mqtt'
|
24
26
|
|
@@ -38,8 +40,10 @@ module Fluent
|
|
38
40
|
def configure(conf)
|
39
41
|
super
|
40
42
|
@bind ||= conf['bind']
|
41
|
-
@topic ||= conf['topic']
|
42
|
-
@port ||= conf['port']
|
43
|
+
@topic ||= conf['topic']
|
44
|
+
@port ||= conf['port']
|
45
|
+
@formatter = Plugin.new_formatter(@format)
|
46
|
+
@formatter.configure(conf)
|
43
47
|
end
|
44
48
|
|
45
49
|
def start
|
@@ -49,6 +53,7 @@ module Fluent
|
|
49
53
|
$log.debug "start mqtt #{@bind}"
|
50
54
|
opts = {host: @bind,
|
51
55
|
port: @port}
|
56
|
+
opts[:client_id] = @client_id if @client_id
|
52
57
|
opts[:username] = @username if @username
|
53
58
|
opts[:password] = @password if @password
|
54
59
|
opts[:ssl] = @ssl if @ssl
|
@@ -69,9 +74,9 @@ module Fluent
|
|
69
74
|
end
|
70
75
|
|
71
76
|
def write(chunk)
|
72
|
-
$log.debug "write"
|
73
77
|
chunk.msgpack_each { |tag, time, record|
|
74
|
-
@
|
78
|
+
log.debug "write #{@topic} #{@formatter.format(tag,time,record)}"
|
79
|
+
@connect.publish(@topic, @formatter.format(tag,time,record), retain=@retain)
|
75
80
|
}
|
76
81
|
end
|
77
82
|
|
data/test/helper.rb
CHANGED
@@ -12,6 +12,7 @@ require 'test/unit'
|
|
12
12
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
13
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
14
|
require 'fluent/test'
|
15
|
+
require 'fluent/test/helpers'
|
15
16
|
unless ENV.has_key?('VERBOSE')
|
16
17
|
nulllogger = Object.new
|
17
18
|
nulllogger.instance_eval {|obj|
|
data/test/plugin/test_in_mqtt.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'helper'
|
2
|
+
require 'fluent/test/driver/input'
|
2
3
|
|
3
|
-
class Fluent::MqttInput
|
4
|
+
class Fluent::Plugin::MqttInput
|
4
5
|
#def emit topic, message , time = Fluent::Engine.now
|
5
6
|
#if message.class == Array
|
6
7
|
#message.each do |data|
|
@@ -14,6 +15,8 @@ class Fluent::MqttInput
|
|
14
15
|
|
15
16
|
end
|
16
17
|
|
18
|
+
include Fluent::Test::Helpers
|
19
|
+
|
17
20
|
class MqttInputTest < Test::Unit::TestCase
|
18
21
|
def setup
|
19
22
|
Fluent::Test.setup
|
@@ -24,7 +27,7 @@ class MqttInputTest < Test::Unit::TestCase
|
|
24
27
|
format json ]
|
25
28
|
|
26
29
|
def create_driver(conf = CONFIG)
|
27
|
-
Fluent::Test::
|
30
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::MqttInput).configure(conf)
|
28
31
|
end
|
29
32
|
|
30
33
|
def test_configure_1
|
@@ -45,7 +48,7 @@ class MqttInputTest < Test::Unit::TestCase
|
|
45
48
|
time_key time]
|
46
49
|
)
|
47
50
|
assert_equal 'csv', d2.instance.format
|
48
|
-
assert_equal 'time', d2.instance.time_key
|
51
|
+
assert_equal 'time', d2.instance.inject_config.time_key
|
49
52
|
end
|
50
53
|
|
51
54
|
def test_configure_2
|
@@ -72,21 +75,20 @@ class MqttInputTest < Test::Unit::TestCase
|
|
72
75
|
port 1883
|
73
76
|
format json ]
|
74
77
|
)
|
75
|
-
time =
|
78
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
76
79
|
data = [
|
77
80
|
{tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
|
78
81
|
{tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
|
79
82
|
{tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
|
80
83
|
]
|
81
84
|
|
82
|
-
d.run do
|
85
|
+
d.run(expect_emits: 3, timeout: 5) do
|
83
86
|
data.each do |record|
|
84
87
|
send_data record[:tag], record[:message], d.instance.format
|
85
|
-
sleep 0.1
|
86
88
|
end
|
87
89
|
end
|
88
90
|
|
89
|
-
emits = d.
|
91
|
+
emits = d.events
|
90
92
|
assert_equal('tag1', emits[0][0])
|
91
93
|
assert_equal({"t" => time, "v" => {"a"=>1}}, emits[0][2])
|
92
94
|
|
@@ -104,7 +106,7 @@ class MqttInputTest < Test::Unit::TestCase
|
|
104
106
|
format json
|
105
107
|
time_key t ]
|
106
108
|
)
|
107
|
-
time =
|
109
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
108
110
|
data = [
|
109
111
|
{tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
|
110
112
|
{tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
|
@@ -112,16 +114,16 @@ class MqttInputTest < Test::Unit::TestCase
|
|
112
114
|
{tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
|
113
115
|
]
|
114
116
|
|
115
|
-
d.run do
|
117
|
+
d.run(expect_emits: 4, timeout: 5) do
|
116
118
|
data.each do |record|
|
117
119
|
send_data record[:tag], record[:message], d.instance.format
|
118
120
|
end
|
119
121
|
end
|
120
122
|
|
121
|
-
emits = d.
|
123
|
+
emits = d.events
|
122
124
|
assert_equal('tag1', emits[0][0])
|
123
125
|
assert_equal(time, emits[0][1])
|
124
|
-
assert_equal({"v" => {"a"=>1}}, emits[0][2])
|
126
|
+
assert_equal({"v" => {"a"=>1}, "t" => time}, emits[0][2])
|
125
127
|
end
|
126
128
|
|
127
129
|
def test_format_none
|
@@ -137,13 +139,13 @@ class MqttInputTest < Test::Unit::TestCase
|
|
137
139
|
{tag: "tag3", message: ''},
|
138
140
|
]
|
139
141
|
|
140
|
-
d.run do
|
142
|
+
d.run(expect_emits: 3, timeout: 5) do
|
141
143
|
data.each do |record|
|
142
144
|
send_data record[:tag], record[:message], d.instance.format
|
143
145
|
end
|
144
146
|
end
|
145
147
|
|
146
|
-
emits = d.
|
148
|
+
emits = d.events
|
147
149
|
time = Fluent::Engine.now
|
148
150
|
assert_equal('tag1', emits[0][0])
|
149
151
|
assert_equal({'message' => 'hello world'}, emits[0][2])
|
@@ -161,20 +163,20 @@ class MqttInputTest < Test::Unit::TestCase
|
|
161
163
|
keys time,message]
|
162
164
|
)
|
163
165
|
|
164
|
-
time =
|
166
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
165
167
|
data = [
|
166
168
|
{tag: "tag1", message: "#{time},hello world" },
|
167
169
|
{tag: "tag2", message: "#{time},hello to you to" },
|
168
170
|
{tag: "tag3", message: "#{time}," },
|
169
171
|
]
|
170
172
|
|
171
|
-
d.run do
|
173
|
+
d.run(expect_emits: 3, timeout: 5) do
|
172
174
|
data.each do |record|
|
173
175
|
send_data record[:tag], record[:message], d.instance.format
|
174
176
|
end
|
175
177
|
end
|
176
178
|
|
177
|
-
emits = d.
|
179
|
+
emits = d.events
|
178
180
|
#puts 'emits length', emits.length.to_s
|
179
181
|
assert_equal('tag1', emits[0][0])
|
180
182
|
assert_equal({'time' => time.to_s, 'message' => 'hello world'}, emits[0][2])
|
@@ -192,10 +194,10 @@ class MqttInputTest < Test::Unit::TestCase
|
|
192
194
|
format csv
|
193
195
|
keys time2,message
|
194
196
|
time_key time2
|
195
|
-
time_format %
|
197
|
+
time_format %s]
|
196
198
|
)
|
197
199
|
|
198
|
-
time =
|
200
|
+
time = event_time("2011-01-02 13:14:15 UTC").to_i # to obtain unixtime stamp
|
199
201
|
data = [
|
200
202
|
{tag: "tag1", message: "#{time},abc" },
|
201
203
|
{tag: "tag2", message: "#{time},def" },
|
@@ -203,24 +205,24 @@ class MqttInputTest < Test::Unit::TestCase
|
|
203
205
|
{tag: "tag3", message: "#{time}," },
|
204
206
|
]
|
205
207
|
|
206
|
-
d.run do
|
208
|
+
d.run(expect_emits: 4, timeout: 5) do
|
207
209
|
data.each do |record|
|
208
210
|
send_data record[:tag], record[:message], d.instance.format
|
209
211
|
end
|
210
212
|
end
|
211
213
|
|
212
|
-
emits = d.
|
214
|
+
emits = d.events
|
213
215
|
assert_equal('tag1', emits[0][0])
|
214
|
-
assert_equal({'message' => 'abc'}, emits[0][2])
|
216
|
+
assert_equal({'message' => 'abc', "time2" => time}, emits[0][2])
|
215
217
|
|
216
218
|
assert_equal('tag2', emits[1][0])
|
217
|
-
assert_equal({'message' => 'def'}, emits[1][2])
|
219
|
+
assert_equal({'message' => 'def', "time2" => time}, emits[1][2])
|
218
220
|
|
219
221
|
assert_equal('tag3', emits[2][0])
|
220
|
-
assert_equal({'message' => 'ghi'}, emits[2][2])
|
222
|
+
assert_equal({'message' => 'ghi', "time2" => time}, emits[2][2])
|
221
223
|
|
222
224
|
assert_equal('tag3', emits[3][0])
|
223
|
-
assert_equal({'message' => nil}, emits[3][2])
|
225
|
+
assert_equal({'message' => nil, "time2"=> time}, emits[3][2])
|
224
226
|
end
|
225
227
|
|
226
228
|
def send_data tag, record, format
|
@@ -234,6 +236,5 @@ class MqttInputTest < Test::Unit::TestCase
|
|
234
236
|
else
|
235
237
|
sub_client.publish(tag, record)
|
236
238
|
end
|
237
|
-
sleep 0.2
|
238
239
|
end
|
239
240
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mqtt
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuuna Kurita
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-04-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mqtt
|
@@ -131,11 +131,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
131
131
|
version: '0'
|
132
132
|
requirements: []
|
133
133
|
rubyforge_project:
|
134
|
-
rubygems_version: 2.
|
134
|
+
rubygems_version: 2.5.1
|
135
135
|
signing_key:
|
136
136
|
specification_version: 4
|
137
137
|
summary: fluentd input plugin for mqtt server
|
138
138
|
test_files:
|
139
139
|
- test/helper.rb
|
140
140
|
- test/plugin/test_in_mqtt.rb
|
141
|
-
has_rdoc:
|