fluent-plugin-nsq-local 0.0.7.dev.4

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: b113f08ba211549199e711cec9471db5fbefc6b9
4
+ data.tar.gz: 26a802fb77649360ffd9824128967e212983f6ab
5
+ SHA512:
6
+ metadata.gz: 157d4adbc16da363cf502183e17ca7f6e0624c59788e23e2d23fc892905e5ee5f79f8c090558465919b595164c1aef946df05999e3d6bcc01ecd9d632ef7b4a6
7
+ data.tar.gz: 7a6591bfb251766fb67fbbc5d36646e867df704ee9c46111d1bb58494d2c70da91b4d3e8f5e567aad7e6bc523600ebcdd134781a2b40193ef5adacf09d18e1bd
@@ -0,0 +1,25 @@
1
+ $:.push File.expand_path("../lib", __FILE__)
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = "fluent-plugin-nsq-local"
5
+ s.version = `cat VERSION`
6
+ s.authors = ["lxfontes", "dterror"]
7
+ s.email = ["lucas@uken.com", "diogo@uken.com"]
8
+ s.description = %q{NSQ output plugin for Fluentd}
9
+ s.summary = %q{output plugin for fluentd}
10
+ s.homepage = "https://github.com/uken/fluent-plugin-nsq"
11
+ s.license = 'MIT'
12
+
13
+ git_files = `git ls-files`.split($/)
14
+
15
+ s.files = git_files.grep(%r{^(lib|fluent|bin)})
16
+ s.executables = git_files.grep(%r{^bin/}).map{ |f| File.basename(f) }
17
+ s.test_files = git_files.grep(%r{^(test|spec|features)/})
18
+ s.require_paths = ["lib"]
19
+
20
+ s.add_runtime_dependency 'fluentd', ['> 0.14', '< 2']
21
+ s.add_runtime_dependency 'nsq-ruby', '~> 2.3'
22
+ s.add_development_dependency 'rake', '~> 10'
23
+ s.add_development_dependency 'json', '~> 2'
24
+ s.add_development_dependency("test-unit", ["~> 3.2"])
25
+ end
@@ -0,0 +1,95 @@
1
+ # coding: utf-8
2
+
3
+ require 'fluent/plugin/input'
4
+ require 'nsq'
5
+ require 'yajl'
6
+
7
+ module Fluent::Plugin
8
+ class NSQInput < Input
9
+ Fluent::Plugin.register_input('nsq', self)
10
+
11
+ config_param :topic, :string, default: nil
12
+ config_param :channel, :string, default: 'fluent_nsq_input'
13
+ config_param :in_flight, :integer, default: 100
14
+ config_param :nsqlookupd, :array, default: nil
15
+ config_param :tag, :string, default: '_key'
16
+ config_param :time_key, :string, default: nil
17
+ config_param :tag_source, default: :key do |val|
18
+ case val.downcase
19
+ when 'key'
20
+ :key
21
+ when 'topic'
22
+ :topic
23
+ when 'static'
24
+ :static
25
+ else
26
+ fail Fluent::ConfigError, 'tag_source should be either "key", "static" or "topic"'
27
+ end
28
+ end
29
+
30
+ def configure(conf)
31
+ super
32
+
33
+ fail Fluent::ConfigError, 'Missing nsqlookupd' unless @nsqlookupd
34
+ fail Fluent::ConfigError, 'Missing topic' unless @topic
35
+ fail Fluent::ConfigError, 'Missing channel' unless @channel
36
+ fail Fluent::ConfigError, 'in_flight needs to be bigger than 0' unless @in_flight > 0
37
+ end
38
+
39
+ def start
40
+ super
41
+ @consumer = Nsq::Consumer.new(
42
+ nsqlookupd: @nsqlookupd,
43
+ topic: @topic,
44
+ channel: @channel,
45
+ max_in_flight: @in_flight
46
+ )
47
+ @running = true
48
+ @thread = Thread.new(&method(:consume))
49
+ end
50
+
51
+ def shutdown
52
+ super
53
+ @running = false
54
+ @consumer.terminate
55
+ end
56
+
57
+ private
58
+ def consume
59
+ while @running
60
+ consume_one
61
+ end
62
+ end
63
+
64
+ def consume_one
65
+ msg = @consumer.pop
66
+ record = Yajl.load(msg.body.force_encoding('UTF-8'))
67
+ record_tag = tag_for_record(record)
68
+ record_time = time_for_record(record, msg)
69
+ router.emit(record_tag, record_time, record)
70
+ msg.finish
71
+ rescue => e
72
+ log.warn("nsq: #{e}")
73
+ msg.requeue if msg
74
+ end
75
+
76
+ def tag_for_record(record)
77
+ case @tag_source
78
+ when :static
79
+ @tag
80
+ when :key
81
+ record[@tag]
82
+ when :topic
83
+ @topic
84
+ end
85
+ end
86
+
87
+ def time_for_record(record, msg)
88
+ if @time_key
89
+ record[@time_key]
90
+ else
91
+ Fluent::EventTime.new(msg.timestamp.to_i, msg.timestamp.nsec)
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,79 @@
1
+ # coding: utf-8
2
+
3
+ require 'fluent/plugin/output'
4
+ require 'nsq'
5
+ require 'yajl'
6
+
7
+ module Fluent::Plugin
8
+ class NSQOutput < Output
9
+ Fluent::Plugin.register_output('nsq', self)
10
+
11
+ config_param :topic, :string, default: nil
12
+ config_param :nsqlookupd, :array, default: nil
13
+ config_param :nsqd, :array, default: nil
14
+ config_param :enable_tls, :bool, default: false
15
+ config_param :key, :string, default: nil
16
+ config_param :certificate, :string, default: nil
17
+ config_param :ca_certificate, :string, default: nil
18
+
19
+ config_section :buffer do
20
+ config_set_default :chunk_keys, ['tag']
21
+ end
22
+
23
+ def configure(conf)
24
+ super
25
+
26
+ fail Fluent::ConfigError, 'Missing nsqlookupd or nsqd' unless @nsqlookupd || @nsqd
27
+ fail Fluent::ConfigError, 'Missing topic' unless @topic
28
+ if @enable_tls
29
+ fail Fluent::ConfigError, 'Missing tls config params' unless @key && @certificate && @ca_certificate
30
+ end
31
+ end
32
+
33
+ def start
34
+ super
35
+ producer_config = {topic: @topic}
36
+
37
+ if @nsqlookupd
38
+ producer_config[:nsqlookupd] = @nsqlookupd
39
+ elsif @nsqd
40
+ producer_config[:nsqd] = @nsqd
41
+ end
42
+
43
+ if @enable_tls
44
+ producer_config[:tls_v1] = true
45
+ producer_config[:tls_options] = {
46
+ key: @key,
47
+ certificate: @certificate,
48
+ ca_certificate: @ca_certificate,
49
+ verify_mode: OpenSSL::SSL::VERIFY_PEER
50
+ }
51
+ end
52
+
53
+ @producer = Nsq::Producer.new(producer_config)
54
+ end
55
+
56
+ def shutdown
57
+ @producer.terminate
58
+ super
59
+ end
60
+
61
+ def write(chunk)
62
+ return if chunk.empty?
63
+
64
+ tag = chunk.metadata.tag
65
+ chunk.each do |time, record|
66
+ tagged_record = record.merge(
67
+ :_key => tag,
68
+ :_ts => time.to_f,
69
+ :'@timestamp' => Time.at(time).iso8601(3) # kibana/elasticsearch friendly
70
+ )
71
+ begin
72
+ @producer.write(Yajl.dump(tagged_record))
73
+ rescue => e
74
+ log.warn("nsq: #{e}")
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
File without changes
@@ -0,0 +1,66 @@
1
+ require 'test/unit'
2
+
3
+ require 'fluent/test'
4
+ require 'fluent/test/driver/input'
5
+ require 'fluent/plugin/in_nsq'
6
+
7
+ require 'date'
8
+
9
+ require 'helper'
10
+
11
+ $:.push File.expand_path("../lib", __FILE__)
12
+ $:.push File.dirname(__FILE__)
13
+
14
+ class TestNSQInput < Test::Unit::TestCase
15
+ TCONFIG = %[
16
+ nsqlookupd localhost:4161
17
+ topic logs_in
18
+ time_key _ts
19
+ ]
20
+ def setup
21
+ #Nsq.logger = Logger.new(STDOUT)
22
+ Fluent::Test.setup
23
+ end
24
+
25
+ def test_configure
26
+ d = create_driver
27
+ assert_not_nil d.instance.topic
28
+ end
29
+
30
+ def create_driver(conf=TCONFIG)
31
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::NSQInput).configure(conf)
32
+ end
33
+
34
+ def create_producer
35
+ Nsq::Producer.new(
36
+ nsqlookupd: ['127.0.0.1:4161'],
37
+ topic: 'logs_in'
38
+ )
39
+ end
40
+
41
+ def sample_record
42
+ {_ts: Time.now, _key: 'somekey', age:26, request_id: '42', parent_id: 'parent', sub: {field: {pos: 15}}}
43
+ end
44
+
45
+ def test_wrong_config
46
+ assert_raise Fluent::ConfigError do
47
+ create_driver('')
48
+ end
49
+ end
50
+
51
+ def test_sample_record_loop
52
+ d = create_driver
53
+ d.run do
54
+ prod = create_producer
55
+ sleep(1)
56
+ prod.write(sample_record.to_json)
57
+ prod.write(sample_record.to_json)
58
+ prod.write(sample_record.to_json)
59
+ prod.write(sample_record.to_json)
60
+ sleep(1)
61
+ prod.terminate
62
+ end
63
+ puts("emitz")
64
+ puts(d.events)
65
+ end
66
+ end
@@ -0,0 +1,51 @@
1
+ require 'test/unit'
2
+
3
+ require 'fluent/test'
4
+ require 'fluent/test/driver/output'
5
+ require 'fluent/plugin/out_nsq'
6
+
7
+ require 'date'
8
+
9
+ require 'helper'
10
+
11
+ $:.push File.expand_path("../lib", __FILE__)
12
+ $:.push File.dirname(__FILE__)
13
+
14
+ class TestNSQOutput < Test::Unit::TestCase
15
+ TCONFIG = %[
16
+ nsqlookupd localhost:4161
17
+ topic logs_out
18
+ ]
19
+ def setup
20
+ #Nsq.logger = Logger.new(STDOUT)
21
+ Fluent::Test.setup
22
+ end
23
+
24
+ def test_configure
25
+ d = create_driver
26
+ assert_not_nil d.instance.topic
27
+ end
28
+
29
+ def create_driver(conf=TCONFIG)
30
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::NSQOutput).configure(conf)
31
+ end
32
+
33
+ def sample_record
34
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'sub' => {'field'=>{'pos'=>15}}}
35
+ end
36
+
37
+ def test_wrong_config
38
+ assert_raise Fluent::ConfigError do
39
+ create_driver('')
40
+ end
41
+ end
42
+
43
+ def test_sample_record_loop
44
+ d = create_driver
45
+ d.run(default_tag: 'test') do
46
+ 100.times.each do |t|
47
+ d.feed(sample_record)
48
+ end
49
+ end
50
+ end
51
+ end
metadata ADDED
@@ -0,0 +1,131 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-nsq-local
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.7.dev.4
5
+ platform: ruby
6
+ authors:
7
+ - lxfontes
8
+ - dterror
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2020-06-01 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - ">"
19
+ - !ruby/object:Gem::Version
20
+ version: '0.14'
21
+ - - "<"
22
+ - !ruby/object:Gem::Version
23
+ version: '2'
24
+ type: :runtime
25
+ prerelease: false
26
+ version_requirements: !ruby/object:Gem::Requirement
27
+ requirements:
28
+ - - ">"
29
+ - !ruby/object:Gem::Version
30
+ version: '0.14'
31
+ - - "<"
32
+ - !ruby/object:Gem::Version
33
+ version: '2'
34
+ - !ruby/object:Gem::Dependency
35
+ name: nsq-ruby
36
+ requirement: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '2.3'
41
+ type: :runtime
42
+ prerelease: false
43
+ version_requirements: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '2.3'
48
+ - !ruby/object:Gem::Dependency
49
+ name: rake
50
+ requirement: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10'
55
+ type: :development
56
+ prerelease: false
57
+ version_requirements: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '10'
62
+ - !ruby/object:Gem::Dependency
63
+ name: json
64
+ requirement: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '2'
69
+ type: :development
70
+ prerelease: false
71
+ version_requirements: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '2'
76
+ - !ruby/object:Gem::Dependency
77
+ name: test-unit
78
+ requirement: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '3.2'
83
+ type: :development
84
+ prerelease: false
85
+ version_requirements: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '3.2'
90
+ description: NSQ output plugin for Fluentd
91
+ email:
92
+ - lucas@uken.com
93
+ - diogo@uken.com
94
+ executables: []
95
+ extensions: []
96
+ extra_rdoc_files: []
97
+ files:
98
+ - fluent-plugin-nsq.gemspec
99
+ - lib/fluent/plugin/in_nsq.rb
100
+ - lib/fluent/plugin/out_nsq.rb
101
+ - test/helper.rb
102
+ - test/plugin/test_in_nsq.rb
103
+ - test/plugin/test_out_nsq.rb
104
+ homepage: https://github.com/uken/fluent-plugin-nsq
105
+ licenses:
106
+ - MIT
107
+ metadata: {}
108
+ post_install_message:
109
+ rdoc_options: []
110
+ require_paths:
111
+ - lib
112
+ required_ruby_version: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - ">="
115
+ - !ruby/object:Gem::Version
116
+ version: '0'
117
+ required_rubygems_version: !ruby/object:Gem::Requirement
118
+ requirements:
119
+ - - ">"
120
+ - !ruby/object:Gem::Version
121
+ version: 1.3.1
122
+ requirements: []
123
+ rubyforge_project:
124
+ rubygems_version: 2.6.14.4
125
+ signing_key:
126
+ specification_version: 4
127
+ summary: output plugin for fluentd
128
+ test_files:
129
+ - test/helper.rb
130
+ - test/plugin/test_in_nsq.rb
131
+ - test/plugin/test_out_nsq.rb