fluent-plugin-diss-nats 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.travis.yml +16 -0
- data/COPYING +13 -0
- data/Gemfile +3 -0
- data/README.md +91 -0
- data/Rakefile +13 -0
- data/fluent-plugin-nats.gemspec +25 -0
- data/lib/fluent/plugin/in_nats.rb +92 -0
- data/lib/fluent/plugin/out_nats.rb +90 -0
- data/test/nats-publish-message.rb +40 -0
- data/test/plugin/test_in_nats.rb +237 -0
- data/test/plugin/test_out_nats.rb +80 -0
- data/test/test_helper.rb +33 -0
- metadata +150 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ad4fd19564d3975dc9b1de76c10b4c93a1791befc2702662940ed31b9ee23324
|
4
|
+
data.tar.gz: e78275501d8031af335de50c899e3db30edf9f98dd9124e8d266563a6695f371
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c4b25840f57bc41ccebad1ec047514af9c7a61e2ed7a896a7d797db34fab34a1520ef04d34564134a1ae85dfc0a7c4965672be7663f3f315146d94a563f713e1
|
7
|
+
data.tar.gz: '06653186eaad88dfedfe937e1e255158d6a2f65d9af2d723f56fc09d60a0597474396a46cf93cdb095f165c8bdb50b324bcaeba9c82f9857022601cb9d956171'
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
language: ruby
|
2
|
+
rvm:
|
3
|
+
- 2.4.2
|
4
|
+
- 2.3.5
|
5
|
+
- 2.2
|
6
|
+
- 2.1
|
7
|
+
|
8
|
+
env:
|
9
|
+
global:
|
10
|
+
GNATSD_VERSION: 1.0.4
|
11
|
+
|
12
|
+
before_install:
|
13
|
+
- sudo rm -f /usr/local/bin/gnatsd
|
14
|
+
- curl -L https://github.com/nats-io/gnatsd/releases/download/v${GNATSD_VERSION}/gnatsd-v${GNATSD_VERSION}-linux-amd64.zip > /tmp/gnatsd-v${GNATSD_VERSION}-linux-amd64.zip
|
15
|
+
- unzip /tmp/gnatsd-v${GNATSD_VERSION}-linux-amd64.zip -d /tmp
|
16
|
+
- sudo mv /tmp/gnatsd-v${GNATSD_VERSION}-linux-amd64/gnatsd /usr/local/bin/gnatsd
|
data/COPYING
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (C) 2012 Eduardo Aceituno <achied@gmail.com> All rights reserved.
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
fluent-plugin-nats
|
2
|
+
==================
|
3
|
+
|
4
|
+
[NATS](https://github.com/derekcollison/nats) plugin for
|
5
|
+
[fluentd](https://github.com/fluent/fluentd) Event Collector
|
6
|
+
|
7
|
+
[![Build Status](https://travis-ci.org/cloudfoundry-community/fluent-plugin-nats.svg?branch=master)](https://travis-ci.org/cloudfoundry-community/fluent-plugin-nats)
|
8
|
+
|
9
|
+
## Requirements
|
10
|
+
|
11
|
+
| fluent-plugin-nats | Fluentd | Ruby |
|
12
|
+
|--------------------|-------------|--------|
|
13
|
+
| >= 1.0.0 | >= v0.14.20 | >= 2.1 |
|
14
|
+
| < 1.0.0 | >= v0.12.0 | >= 1.9 |
|
15
|
+
|
16
|
+
# Getting Started
|
17
|
+
Setup the NATS input:
|
18
|
+
|
19
|
+
~~~~~
|
20
|
+
<source>
|
21
|
+
@type nats
|
22
|
+
tag nats
|
23
|
+
host localhost
|
24
|
+
port 4222
|
25
|
+
user nats
|
26
|
+
password nats
|
27
|
+
queues fluent.>,fluent2.>
|
28
|
+
ssl false
|
29
|
+
</source>
|
30
|
+
~~~~~
|
31
|
+
|
32
|
+
Setup the match output:
|
33
|
+
|
34
|
+
~~~~
|
35
|
+
<match nats.fluent.**>
|
36
|
+
@type stdout
|
37
|
+
</match>
|
38
|
+
~~~~
|
39
|
+
|
40
|
+
# Configuration
|
41
|
+
|
42
|
+
## Plugin helpers
|
43
|
+
|
44
|
+
* thread
|
45
|
+
|
46
|
+
* See also: Fluent::Plugin::Input
|
47
|
+
|
48
|
+
## Fluent::Plugin::NATSInput
|
49
|
+
|
50
|
+
* **host** (string) (optional): NATS server hostname
|
51
|
+
* Default value: `localhost`
|
52
|
+
* **user** (string) (optional): Username for authorized connection
|
53
|
+
* Default value: `nats`
|
54
|
+
* **password** (string) (optional): Password for authorized connection
|
55
|
+
* Default value: `nats`
|
56
|
+
* **port** (integer) (optional): NATS server port
|
57
|
+
* Default value: `4222`
|
58
|
+
* **queues** (array) (optional): Subscribing queue names
|
59
|
+
* Default value: `["fluent.>"]`
|
60
|
+
* **queue** (string) (optional):
|
61
|
+
* Default value: `fluent.>`
|
62
|
+
* Obsoleted: Use `queues` instead
|
63
|
+
* **tag** (string) (optional): The tag prepend before queue name
|
64
|
+
* Default value: `nats`
|
65
|
+
* **ssl** (bool) (optional): Enable secure SSL/TLS connection
|
66
|
+
* Default value: `false`
|
67
|
+
* **max_reconnect_attempts** (integer) (optional): The max number of reconnect tries
|
68
|
+
* Default value: `150`
|
69
|
+
* **reconnect_time_wait** (integer) (optional): The number of seconds to wait between reconnect tries
|
70
|
+
* Default value: `2`
|
71
|
+
|
72
|
+
# Suitable Queues
|
73
|
+
|
74
|
+
## Components
|
75
|
+
* dea.>
|
76
|
+
* cloudcontrollers.>
|
77
|
+
* router.>
|
78
|
+
* cloudcontroller.>
|
79
|
+
* vcap.>
|
80
|
+
* droplet.>
|
81
|
+
* healthmanager.>
|
82
|
+
|
83
|
+
## Services
|
84
|
+
* FilesystemaaS.>
|
85
|
+
* AtmosaaS.>
|
86
|
+
* MongoaaS.>
|
87
|
+
* MyaaS.>
|
88
|
+
* Neo4jaaS.>
|
89
|
+
* AuaaS.>
|
90
|
+
* RMQaaS.>
|
91
|
+
* RaaS.>
|
data/Rakefile
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
|
4
|
+
require 'rake/testtask'
|
5
|
+
Rake::TestTask.new(:test) do |test|
|
6
|
+
test.libs << 'lib' << 'test'
|
7
|
+
test.pattern = 'test/**/test_*.rb'
|
8
|
+
test.test_files = Dir['test/plugin/*.rb']
|
9
|
+
test.verbose = false
|
10
|
+
test.warning = false
|
11
|
+
end
|
12
|
+
|
13
|
+
task :default => :test
|
@@ -0,0 +1,25 @@
|
|
1
|
+
$:.push File.expand_path("../lib", __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-diss-nats"
|
5
|
+
gem.version = "0.1.0"
|
6
|
+
gem.authors = ["Eduardo Aceituno"]
|
7
|
+
gem.email = ["achied@gmail.com"]
|
8
|
+
gem.homepage = "https://github.com/achied/fluent-plugin-nats"
|
9
|
+
gem.summary = %q{nats plugin for fluentd, an event collector}
|
10
|
+
gem.description = %q{nats plugin for fluentd, an event collector}
|
11
|
+
gem.license = "Apache-2.0"
|
12
|
+
|
13
|
+
gem.files = `git ls-files`.split("\n")
|
14
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
15
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
16
|
+
gem.require_paths = ["lib"]
|
17
|
+
|
18
|
+
gem.add_dependency "eventmachine", ">= 0.12.10"
|
19
|
+
gem.add_dependency "fluentd", ">= 0.14.20", "< 2"
|
20
|
+
gem.add_dependency "nats", ">= 0.4.22"
|
21
|
+
|
22
|
+
gem.add_development_dependency "rake", ">= 0.9.2"
|
23
|
+
gem.add_development_dependency "test-unit", "> 3.1"
|
24
|
+
gem.add_development_dependency "timecop"
|
25
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require "fluent/plugin/input"
|
2
|
+
require "nats/client"
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
module Plugin
|
6
|
+
class NATSInput < Fluent::Plugin::Input
|
7
|
+
Fluent::Plugin.register_input("diss-nats", self)
|
8
|
+
|
9
|
+
helpers :thread
|
10
|
+
|
11
|
+
desc "NATS server hostname"
|
12
|
+
config_param :host, :string, default: "localhost"
|
13
|
+
desc "NATS server port"
|
14
|
+
config_param :port, :integer, default: 4222
|
15
|
+
desc "Username for authorized connection"
|
16
|
+
config_param :user, :string, default: "nats"
|
17
|
+
desc "Password for authorized connection"
|
18
|
+
config_param :password, :string, default: "nats", secret: true
|
19
|
+
desc "Subscribing queue names"
|
20
|
+
config_param :queues, :array, default: ["fluent.>"]
|
21
|
+
config_param :queue, :string, default: "fluent.>", obsoleted: "Use `queues` instead"
|
22
|
+
desc "The tag prepend before queue name"
|
23
|
+
config_param :tag, :string, default: "nats"
|
24
|
+
desc "Enable secure SSL/TLS connection"
|
25
|
+
config_param :ssl, :bool, default: false
|
26
|
+
desc "The max number of reconnect tries"
|
27
|
+
config_param :max_reconnect_attempts, :integer, default: 150
|
28
|
+
desc "The number of seconds to wait between reconnect tries"
|
29
|
+
config_param :reconnect_time_wait, :integer, default: 2
|
30
|
+
|
31
|
+
def configure(conf)
|
32
|
+
super
|
33
|
+
|
34
|
+
@nats_config = {
|
35
|
+
uri: "nats://#{@host}:#{@port}",
|
36
|
+
ssl: @ssl,
|
37
|
+
user: @user,
|
38
|
+
pass: @password,
|
39
|
+
reconnect_time_wait: @reconnect_time_wait,
|
40
|
+
max_reconnect_attempts: @max_reconnect_attempts,
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def start
|
45
|
+
super
|
46
|
+
NATS.on_error do |error|
|
47
|
+
log.error "Server Error:", error: error
|
48
|
+
# supervisor will restart worker
|
49
|
+
exit!
|
50
|
+
end
|
51
|
+
run_reactor_thread
|
52
|
+
thread_create(:nats_input_main, &method(:run))
|
53
|
+
log.info "listening nats on #{@uri}/#{@queue}"
|
54
|
+
end
|
55
|
+
|
56
|
+
def shutdown
|
57
|
+
@nats_conn.close
|
58
|
+
EM.stop if EM.reactor_running?
|
59
|
+
super
|
60
|
+
end
|
61
|
+
|
62
|
+
def run
|
63
|
+
EM.next_tick do
|
64
|
+
@nats_conn = NATS.connect(@nats_config) do
|
65
|
+
@queues.each do |queue|
|
66
|
+
@nats_conn.subscribe(queue) do |msg, _reply, sub|
|
67
|
+
tag = "#{@tag}.#{sub}"
|
68
|
+
begin
|
69
|
+
message = JSON.parse(msg)
|
70
|
+
rescue JSON::ParserError => e
|
71
|
+
log.error "Failed parsing JSON #{e.inspect}. Passing as a normal string"
|
72
|
+
message = msg
|
73
|
+
end
|
74
|
+
time = Engine.now
|
75
|
+
router.emit(tag, time, message || {})
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def run_reactor_thread
|
85
|
+
return if EM.reactor_running?
|
86
|
+
thread_create(:nats_input_reactor_thread) do
|
87
|
+
EM.run
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require "fluent/plugin/output"
|
2
|
+
require "nats/client"
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
module Plugin
|
6
|
+
class NATSOutput < Fluent::Plugin::Output
|
7
|
+
Fluent::Plugin.register_output('diss-nats', self)
|
8
|
+
|
9
|
+
helpers :formatter, :thread, :inject
|
10
|
+
|
11
|
+
DEFAULT_FORMAT_TYPE = 'json'
|
12
|
+
|
13
|
+
desc "NATS server hostname"
|
14
|
+
config_param :host, :string, default: "localhost"
|
15
|
+
desc "NATS server port"
|
16
|
+
config_param :port, :integer, default: 4222
|
17
|
+
desc "Username for authorized connection"
|
18
|
+
config_param :user, :string, default: "nats"
|
19
|
+
desc "Password for authorized connection"
|
20
|
+
config_param :password, :string, default: "nats", secret: true
|
21
|
+
desc "Enable secure SSL/TLS connection"
|
22
|
+
config_param :ssl, :bool, default: false
|
23
|
+
desc "The max number of reconnect tries"
|
24
|
+
config_param :max_reconnect_attempts, :integer, default: 150
|
25
|
+
desc "The number of seconds to wait between reconnect tries"
|
26
|
+
config_param :reconnect_time_wait, :integer, default: 2
|
27
|
+
|
28
|
+
config_section :format do
|
29
|
+
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
30
|
+
config_set_default :add_newline, false
|
31
|
+
end
|
32
|
+
|
33
|
+
def multi_workers_ready?
|
34
|
+
true
|
35
|
+
end
|
36
|
+
|
37
|
+
attr_accessor :formatter
|
38
|
+
|
39
|
+
def configure(conf)
|
40
|
+
super
|
41
|
+
|
42
|
+
@nats_config = {
|
43
|
+
uri: "nats://#{@host}:#{@port}",
|
44
|
+
ssl: @ssl,
|
45
|
+
user: @user,
|
46
|
+
pass: @password,
|
47
|
+
reconnect_time_wait: @reconnect_time_wait,
|
48
|
+
max_reconnect_attempts: @max_reconnect_attempts,
|
49
|
+
}
|
50
|
+
@formatter = formatter_create
|
51
|
+
end
|
52
|
+
|
53
|
+
def start
|
54
|
+
super
|
55
|
+
thread_create(:nats_output_main, &method(:run))
|
56
|
+
end
|
57
|
+
|
58
|
+
def shutdown
|
59
|
+
EM.next_tick do
|
60
|
+
NATS.stop
|
61
|
+
end
|
62
|
+
super
|
63
|
+
end
|
64
|
+
|
65
|
+
def run
|
66
|
+
NATS.on_error do |error|
|
67
|
+
log.error "Server Error:", error: error
|
68
|
+
# supervisor will restart worker
|
69
|
+
exit!
|
70
|
+
end
|
71
|
+
NATS.start(@nats_config) do
|
72
|
+
log.info "nats client is running for #{@nats_config[:uri]}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def process(tag, es)
|
77
|
+
es = inject_values_to_event_stream(tag, es)
|
78
|
+
es.each do |time,record|
|
79
|
+
EM.next_tick do
|
80
|
+
NATS.publish(tag, format(tag, time, record))
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def format(tag, time, record)
|
86
|
+
@formatter.format(tag, time, record)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "nats/client"
|
5
|
+
require "optparse"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
def main(argv)
|
9
|
+
uri = nil
|
10
|
+
queue = nil
|
11
|
+
message = nil
|
12
|
+
parser = OptionParser.new
|
13
|
+
parser.on("--uri=URI", "NATS server URI") do |value|
|
14
|
+
uri = URI.parse(value)
|
15
|
+
end
|
16
|
+
parser.on("--queue=QUEUE", "Queue names") do |value|
|
17
|
+
queue = value
|
18
|
+
end
|
19
|
+
parser.on("--message=MESSAGE", "Message JSON") do |value|
|
20
|
+
message = value
|
21
|
+
end
|
22
|
+
begin
|
23
|
+
parser.parse!
|
24
|
+
rescue OptionParser::ParseError => ex
|
25
|
+
puts ex.message
|
26
|
+
end
|
27
|
+
options = {
|
28
|
+
uri: uri.to_s
|
29
|
+
}
|
30
|
+
options[:user] = uri.user if uri.user
|
31
|
+
options[:pass] = uri.password if uri.password
|
32
|
+
NATS.start(options) do
|
33
|
+
NATS.publish(queue, message) do
|
34
|
+
sleep 0.5
|
35
|
+
NATS.stop
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
main(ARGV)
|
@@ -0,0 +1,237 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
require "fluent/test/driver/input"
|
3
|
+
require "fluent/plugin/in_nats"
|
4
|
+
|
5
|
+
class NATSInputTest < Test::Unit::TestCase
|
6
|
+
include NATSTestHelper
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
port 4222
|
10
|
+
host localhost
|
11
|
+
user nats
|
12
|
+
password nats
|
13
|
+
]
|
14
|
+
|
15
|
+
def basic_queue_conf
|
16
|
+
CONFIG + %[
|
17
|
+
queues fluent.>
|
18
|
+
]
|
19
|
+
end
|
20
|
+
|
21
|
+
def multiple_queue_conf
|
22
|
+
CONFIG + %[
|
23
|
+
queues fluent.>, fluent2.>
|
24
|
+
]
|
25
|
+
end
|
26
|
+
|
27
|
+
def ssl_conf
|
28
|
+
basic_queue_conf + %[
|
29
|
+
ssl true
|
30
|
+
]
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_driver(conf)
|
34
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::NATSInput).configure(conf)
|
35
|
+
end
|
36
|
+
|
37
|
+
def setup
|
38
|
+
Fluent::Test.setup
|
39
|
+
@time = Time.parse("2011-01-02 13:14:15 UTC")
|
40
|
+
Timecop.freeze(@time)
|
41
|
+
end
|
42
|
+
|
43
|
+
def teardown
|
44
|
+
Timecop.return
|
45
|
+
end
|
46
|
+
|
47
|
+
sub_test_case "configure" do
|
48
|
+
test "basic" do
|
49
|
+
d = create_driver basic_queue_conf
|
50
|
+
assert_equal 4222, d.instance.port
|
51
|
+
assert_equal "localhost", d.instance.host
|
52
|
+
assert_equal "nats", d.instance.user
|
53
|
+
assert_equal "nats", d.instance.password
|
54
|
+
assert_equal ["fluent.>"], d.instance.queues
|
55
|
+
end
|
56
|
+
|
57
|
+
test "multiple queue" do
|
58
|
+
d = create_driver multiple_queue_conf
|
59
|
+
assert_equal 4222, d.instance.port
|
60
|
+
assert_equal "localhost", d.instance.host
|
61
|
+
assert_equal "nats", d.instance.user
|
62
|
+
assert_equal "nats", d.instance.password
|
63
|
+
assert_equal ["fluent.>", "fluent2.>"], d.instance.queues
|
64
|
+
end
|
65
|
+
|
66
|
+
test "basic with ssl" do
|
67
|
+
d = create_driver ssl_conf
|
68
|
+
assert_equal 4222, d.instance.port
|
69
|
+
assert_equal "localhost", d.instance.host
|
70
|
+
assert_equal "nats", d.instance.user
|
71
|
+
assert_equal "nats", d.instance.password
|
72
|
+
assert_equal ["fluent.>"], d.instance.queues
|
73
|
+
assert_equal true, d.instance.ssl
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
sub_test_case "events" do
|
78
|
+
test "with credentials" do
|
79
|
+
d = create_driver basic_queue_conf
|
80
|
+
|
81
|
+
time = @time.to_i
|
82
|
+
|
83
|
+
records = [
|
84
|
+
["nats.fluent.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
|
85
|
+
["nats.fluent.test2", time, { "message" => "nats", "fluent_timestamp" => time }]
|
86
|
+
]
|
87
|
+
|
88
|
+
uri = generate_uri(d)
|
89
|
+
|
90
|
+
run_server(uri) do
|
91
|
+
d.run(expect_records: records.size, timeout: 5) do
|
92
|
+
records.each do |tag, _time, record|
|
93
|
+
send(uri, tag[5..-1], record)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
assert_equal(records, d.events)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
test "without credentials" do
|
101
|
+
d = create_driver basic_queue_conf
|
102
|
+
time = @time.to_i
|
103
|
+
|
104
|
+
records = [
|
105
|
+
["nats.fluent.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
|
106
|
+
["nats.fluent.test2", time, { "message" => "nats", "fluent_timestamp" => time }]
|
107
|
+
]
|
108
|
+
|
109
|
+
uri = generate_uri(d)
|
110
|
+
|
111
|
+
run_server(uri) do
|
112
|
+
d.run(expect_records: records.size, timeout: 5) do
|
113
|
+
records.each do |tag, _time, record|
|
114
|
+
send(uri, tag[5..-1], record)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
assert_equal(records, d.events)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
test "multiple queues" do
|
122
|
+
d = create_driver multiple_queue_conf
|
123
|
+
time = @time.to_i
|
124
|
+
|
125
|
+
records = [
|
126
|
+
["nats.fluent.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
|
127
|
+
["nats.fluent.test2", time, { "message" => "nats", "fluent_timestamp" => time }],
|
128
|
+
["nats.fluent2.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
|
129
|
+
["nats.fluent2.test2", time, { "message" => "nats", "fluent_timestamp" => time }]
|
130
|
+
]
|
131
|
+
|
132
|
+
uri = generate_uri(d)
|
133
|
+
|
134
|
+
run_server(uri) do
|
135
|
+
d.run(expect_records: records.size, timeout: 5) do
|
136
|
+
records.each do |tag, _time, record|
|
137
|
+
send(uri, tag[5..-1], record)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
assert_equal(records, d.events)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
test "without fluent timestamp" do
|
145
|
+
d = create_driver basic_queue_conf
|
146
|
+
time = Time.now.to_i
|
147
|
+
|
148
|
+
records = [
|
149
|
+
["nats.fluent.test1", time, { "message" => "nats" }]
|
150
|
+
]
|
151
|
+
|
152
|
+
uri = generate_uri(d)
|
153
|
+
run_server(uri) do
|
154
|
+
d.run(expect_records: records.size, timeout: 5) do
|
155
|
+
records.each do |tag, _time, record|
|
156
|
+
send(uri, tag[5..-1], record)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
assert_equal(records, d.events)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
test "arrays" do
|
164
|
+
d = create_driver basic_queue_conf
|
165
|
+
time = Time.now.to_i
|
166
|
+
|
167
|
+
records = [
|
168
|
+
["nats.fluent.empty_array", time, []],
|
169
|
+
["nats.fluent.string_array", time, %w(one two three)]
|
170
|
+
]
|
171
|
+
|
172
|
+
uri = generate_uri(d)
|
173
|
+
run_server(uri) do
|
174
|
+
d.run(expect_records: records.size, timeout: 5) do
|
175
|
+
records.each do |tag, _time, record|
|
176
|
+
send(uri, tag[5..-1], record)
|
177
|
+
end
|
178
|
+
end
|
179
|
+
assert_equal(records, d.events)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
test "empty publish string" do
|
184
|
+
d = create_driver basic_queue_conf
|
185
|
+
time = Time.now.to_i
|
186
|
+
|
187
|
+
records = [
|
188
|
+
["nats.fluent.nil", time, {}]
|
189
|
+
]
|
190
|
+
|
191
|
+
uri = generate_uri(d)
|
192
|
+
run_server(uri) do
|
193
|
+
d.run(expect_records: records.size, timeout: 5) do
|
194
|
+
records.each do |tag, _time, _record|
|
195
|
+
send(uri, tag[5..-1], nil)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
assert_equal(records, d.events)
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
test "regular publish string" do
|
203
|
+
d = create_driver basic_queue_conf
|
204
|
+
time = Time.now.to_i
|
205
|
+
|
206
|
+
records = [
|
207
|
+
["nats.fluent.string", time, "Lorem ipsum dolor sit amet"]
|
208
|
+
]
|
209
|
+
|
210
|
+
uri = generate_uri(d)
|
211
|
+
run_server(uri) do
|
212
|
+
d.run(expect_records: records.size, timeout: 5) do
|
213
|
+
records.each do |tag, _time, _record|
|
214
|
+
send(uri, tag[5..-1], "Lorem ipsum dolor sit amet")
|
215
|
+
end
|
216
|
+
end
|
217
|
+
assert_equal(records, d.events)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def send(uri, tag, msg)
|
223
|
+
system("test/nats-publish-message.rb", "--uri", uri, "--queue", tag, "--message", msg.to_json)
|
224
|
+
end
|
225
|
+
|
226
|
+
def generate_uri(driver)
|
227
|
+
user = driver.instance.user
|
228
|
+
pass = driver.instance.password
|
229
|
+
host = driver.instance.host
|
230
|
+
port = driver.instance.port
|
231
|
+
if user && pass
|
232
|
+
"nats://#{user}:#{pass}@#{host}:#{port}"
|
233
|
+
else
|
234
|
+
"nats://#{host}:#{port}"
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
require "fluent/test/driver/output"
|
3
|
+
require "fluent/test/driver/input"
|
4
|
+
require "fluent/plugin/out_nats"
|
5
|
+
require "fluent/plugin/in_nats"
|
6
|
+
require "fluent/time"
|
7
|
+
|
8
|
+
class NATSOutputTest < Test::Unit::TestCase
|
9
|
+
include NATSTestHelper
|
10
|
+
|
11
|
+
CONFIG = %[
|
12
|
+
port 4222
|
13
|
+
host localhost
|
14
|
+
user nats
|
15
|
+
password nats
|
16
|
+
]
|
17
|
+
|
18
|
+
CONFIG_INPUT = CONFIG + %[
|
19
|
+
queues test.>
|
20
|
+
tag nats
|
21
|
+
]
|
22
|
+
|
23
|
+
def create_driver(conf)
|
24
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::NATSOutput).configure(conf)
|
25
|
+
end
|
26
|
+
|
27
|
+
def create_input_driver(conf)
|
28
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::NATSInput).configure(conf)
|
29
|
+
end
|
30
|
+
|
31
|
+
def setup
|
32
|
+
Fluent::Test.setup
|
33
|
+
@time = Time.parse("2011-01-02 13:14:15 UTC")
|
34
|
+
Timecop.freeze(@time)
|
35
|
+
end
|
36
|
+
|
37
|
+
def teardown
|
38
|
+
Timecop.return
|
39
|
+
end
|
40
|
+
|
41
|
+
test "configuration test" do
|
42
|
+
d = create_driver(CONFIG)
|
43
|
+
assert_equal 4222, d.instance.port
|
44
|
+
assert_equal "localhost", d.instance.host
|
45
|
+
assert_equal "nats", d.instance.user
|
46
|
+
assert_equal "nats", d.instance.password
|
47
|
+
end
|
48
|
+
|
49
|
+
test "publish an event to NATS" do
|
50
|
+
d = create_driver(CONFIG)
|
51
|
+
input_driver = create_input_driver(CONFIG_INPUT)
|
52
|
+
|
53
|
+
time = Fluent::EventTime.now
|
54
|
+
|
55
|
+
uri = generate_uri(d)
|
56
|
+
|
57
|
+
run_server(uri) do
|
58
|
+
input_driver.run(expect_records: 1) do
|
59
|
+
d.run(default_tag: 'test.log') do
|
60
|
+
d.feed(time, {"test" => "test1"})
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
event = input_driver.events[0]
|
65
|
+
assert_equal(event[0], 'nats.test.log')
|
66
|
+
assert_equal(event[2], {"test" => "test1"})
|
67
|
+
end
|
68
|
+
|
69
|
+
def generate_uri(driver)
|
70
|
+
user = driver.instance.user
|
71
|
+
pass = driver.instance.password
|
72
|
+
host = driver.instance.host
|
73
|
+
port = driver.instance.port
|
74
|
+
if user && pass
|
75
|
+
"nats://#{user}:#{pass}@#{host}:#{port}"
|
76
|
+
else
|
77
|
+
"nats://#{host}:#{port}"
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
require "bundler/setup"
|
2
|
+
require "test/unit"
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift(File.join(__dir__, "..", "lib"))
|
5
|
+
$LOAD_PATH.unshift(__dir__)
|
6
|
+
require "fluent/test"
|
7
|
+
require "nats/client"
|
8
|
+
require "timecop"
|
9
|
+
|
10
|
+
module NATSTestHelper
|
11
|
+
def run_server(uri)
|
12
|
+
uri = URI.parse(uri)
|
13
|
+
unless NATS.server_running?(uri)
|
14
|
+
args = prepare_args(uri)
|
15
|
+
# We can invoke gnatsd before run test
|
16
|
+
pid = spawn("gnatsd", *args, out: "/dev/null", err: "/dev/null")
|
17
|
+
NATS.wait_for_server(uri, 10)
|
18
|
+
end
|
19
|
+
yield
|
20
|
+
rescue
|
21
|
+
raise
|
22
|
+
ensure
|
23
|
+
Process.kill(:INT, pid) if pid
|
24
|
+
end
|
25
|
+
|
26
|
+
def prepare_args(uri)
|
27
|
+
args = ["-p", uri.port.to_s]
|
28
|
+
args.push("--user", uri.user) if uri.user
|
29
|
+
args.push("--pass", uri.password) if uri.password
|
30
|
+
args.push(*@flags) if @flags
|
31
|
+
args
|
32
|
+
end
|
33
|
+
end
|
metadata
ADDED
@@ -0,0 +1,150 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-diss-nats
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Eduardo Aceituno
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-05-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: eventmachine
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.12.10
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.12.10
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.14.20
|
34
|
+
- - "<"
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '2'
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 0.14.20
|
44
|
+
- - "<"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '2'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: nats
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 0.4.22
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 0.4.22
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: rake
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: 0.9.2
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 0.9.2
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: test-unit
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '3.1'
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '3.1'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: timecop
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
description: nats plugin for fluentd, an event collector
|
104
|
+
email:
|
105
|
+
- achied@gmail.com
|
106
|
+
executables: []
|
107
|
+
extensions: []
|
108
|
+
extra_rdoc_files: []
|
109
|
+
files:
|
110
|
+
- ".gitignore"
|
111
|
+
- ".travis.yml"
|
112
|
+
- COPYING
|
113
|
+
- Gemfile
|
114
|
+
- README.md
|
115
|
+
- Rakefile
|
116
|
+
- fluent-plugin-nats.gemspec
|
117
|
+
- lib/fluent/plugin/in_nats.rb
|
118
|
+
- lib/fluent/plugin/out_nats.rb
|
119
|
+
- test/nats-publish-message.rb
|
120
|
+
- test/plugin/test_in_nats.rb
|
121
|
+
- test/plugin/test_out_nats.rb
|
122
|
+
- test/test_helper.rb
|
123
|
+
homepage: https://github.com/achied/fluent-plugin-nats
|
124
|
+
licenses:
|
125
|
+
- Apache-2.0
|
126
|
+
metadata: {}
|
127
|
+
post_install_message:
|
128
|
+
rdoc_options: []
|
129
|
+
require_paths:
|
130
|
+
- lib
|
131
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
137
|
+
requirements:
|
138
|
+
- - ">="
|
139
|
+
- !ruby/object:Gem::Version
|
140
|
+
version: '0'
|
141
|
+
requirements: []
|
142
|
+
rubygems_version: 3.0.3.1
|
143
|
+
signing_key:
|
144
|
+
specification_version: 4
|
145
|
+
summary: nats plugin for fluentd, an event collector
|
146
|
+
test_files:
|
147
|
+
- test/nats-publish-message.rb
|
148
|
+
- test/plugin/test_in_nats.rb
|
149
|
+
- test/plugin/test_out_nats.rb
|
150
|
+
- test/test_helper.rb
|