fluent-plugin-keep-forward 0.0.4 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dbbee68f0007b1260a98d7daa5f5c5cc948dae3c
4
- data.tar.gz: 29e55d39b2ac8c062e896a4f52a55c7c98879165
3
+ metadata.gz: 43773aac13d82ed29b4a225a8c2f943202fe2755
4
+ data.tar.gz: 49aebeb40d1351ebdb035ef4beefce70d085defd
5
5
  SHA512:
6
- metadata.gz: e5feef720bd183dff7b0abcb778246d4176700f94aee5520f519a19211c9783161977822943fc76f340f3fc18e9d0ebb1a8d8cefd106f0ab123c7d46b8c99862
7
- data.tar.gz: 5d955251c44825f448be0d9fb969b58ed7c2b1611367cc5b01591e8af176b7c22066fa16e6fc072d6345a6413248b11128300441a85dcc405cf5f847d1d022b1
6
+ metadata.gz: d5dcb984363125e6633332300337d1ec79a74c4725d0491e8952fe53c3333b8bd1c1c2bcbd1923c0edd1d20c8049f5ee777c85914176bf17118cb6a7af686bd5
7
+ data.tar.gz: 607e402b31507639667ba68035cdd1d30c05e4caa9cc779b29196ba94b68c1a912e2435c1b42458eceee66d2ee787f7fdf6e3d4203ef4eb8d34cd88640ffe43c
data/.gitignore CHANGED
@@ -9,3 +9,5 @@ vendor
9
9
  doc/*
10
10
  tmp/*
11
11
  .yardoc
12
+ .ruby-version
13
+ pkg/*
data/CHANGELOG.md ADDED
@@ -0,0 +1,21 @@
1
+ ## 0.1.1
2
+
3
+ Changes:
4
+
5
+ * Change default of `keepalive` option to `false`
6
+
7
+ ## 0.1.0
8
+
9
+ Enhancements:
10
+
11
+ * Add `keepalive` and `keepalive_time` option
12
+
13
+ ## 0.0.4
14
+
15
+ Enhancements:
16
+
17
+ * Add `prefer_recover` option
18
+
19
+ ## 0.0.1
20
+
21
+ First version
data/README.md CHANGED
@@ -4,17 +4,25 @@ testing ruby: 1.9.2, 1.9.3, 2.0.0; fluentd: 0.10.x
4
4
 
5
5
  ## About
6
6
 
7
- This is an extension of fluentd out\_forward plugin to keep fowarding log data to the same node (as much as possible).
7
+ This is an extension of fluentd out\_forward plugin to keep fowarding log data to the same node (as long as possible).
8
8
 
9
- ## Configuration
9
+ ## Parameters
10
10
 
11
11
  Basically same with out\_forward plugin. See http://docs.fluentd.org/articles/out_forward
12
12
 
13
- ## Additional Parameters
13
+ Following parameters are additionally available:
14
14
 
15
- * prefer_recover
15
+ - keepalive (bool)
16
16
 
17
- Switch to a recovered node from standby nodes or less weighted nodes. Default is `true`.
17
+ Keepalive connection. Default is `false`.
18
+
19
+ - keepalive_time (time)
20
+
21
+ Keepalive expired time. Default is nil (which means as long as possible).
22
+
23
+ - prefer_recover (bool)
24
+
25
+ Switch connection to a recovered node from standby nodes or less weighted nodes. Default is `true`.
18
26
 
19
27
  ## Contributing
20
28
 
@@ -24,6 +32,10 @@ Basically same with out\_forward plugin. See http://docs.fluentd.org/articles/ou
24
32
  4. Push to the branch (`git push origin my-new-feature`)
25
33
  5. Create new [Pull Request](../../pull/new/master)
26
34
 
35
+ ## ChangeLog
36
+
37
+ See [CHANGELOG.md](CHANGELOG.md) for details.
38
+
27
39
  ## Copyright
28
40
 
29
- Copyright (c) 2013 Naotoshi SEO. See [LICENSE](LICENSE) for details.
41
+ Copyright (c) 2013 Naotoshi Seo. See [LICENSE](LICENSE) for details.
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-keep-forward"
6
- s.version = "0.0.4"
7
- s.authors = ["Naotoshi SEO"]
6
+ s.version = "0.1.1"
7
+ s.authors = ["Naotoshi Seo"]
8
8
  s.email = ["sonots@gmail.com"]
9
9
  s.homepage = "https://github.com/sonots/fluent-plugin-keep-forward"
10
- s.summary = "out_foward extension to keep forwarding to a node"
10
+ s.summary = "Fluentd plugin to keep forwarding to a node"
11
11
  s.description = s.summary
12
12
 
13
13
  s.rubyforge_project = "fluent-plugin-keep-forward"
@@ -21,4 +21,6 @@ Gem::Specification.new do |s|
21
21
  s.add_development_dependency "rake"
22
22
  s.add_development_dependency "rspec"
23
23
  s.add_development_dependency "pry"
24
+ s.add_development_dependency "pry-nav"
25
+ s.add_development_dependency "delorean"
24
26
  end
@@ -4,22 +4,64 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
4
4
  Fluent::Plugin.register_output('keep_forward', self)
5
5
 
6
6
  config_param :prefer_recover, :bool, :default => true
7
+ config_param :keepalive, :bool, :default => false
8
+ config_param :keepalive_time, :time, :default => nil # infinite
7
9
 
8
- def write_objects(tag, es)
9
- @node ||= {}
10
- if @node[tag] and @node[tag].available? and (!@prefer_recover or @weight_array.include?(@node[tag]))
10
+ # for test
11
+ attr_accessor :watcher_interval
12
+
13
+ def configure(conf)
14
+ super
15
+
16
+ @node = {}
17
+ @sock = {}
18
+ @sock_expired_at = {}
19
+ @mutex = {}
20
+ @watcher_interval = 1
21
+ end
22
+
23
+ def start
24
+ super
25
+ start_watcher
26
+ end
27
+
28
+ def shutdown
29
+ super
30
+ stop_watcher
31
+ end
32
+
33
+ def start_watcher
34
+ if @keepalive and @keepalive_time
35
+ @watcher = Thread.new(&method(:watch_keepalive_time))
36
+ end
37
+ end
38
+
39
+ def stop_watcher
40
+ if @watcher
41
+ @watcher.terminate
42
+ @watcher.join
43
+ end
44
+ end
45
+
46
+ # Override
47
+ def write_objects(tag, chunk)
48
+ return if chunk.empty?
49
+ error = nil
50
+ node = @node[tag]
51
+
52
+ if node and node.available? and (!@prefer_recover or @weight_array.include?(node))
11
53
  begin
12
- send_data(@node[tag], tag, es)
54
+ send_data(node, tag, chunk)
13
55
  return
14
56
  rescue
15
- weight_send_data(tag, es)
57
+ weight_send_data(tag, chunk)
16
58
  end
17
59
  else
18
- weight_send_data(tag, es)
60
+ weight_send_data(tag, chunk)
19
61
  end
20
62
  end
21
63
 
22
- def weight_send_data(tag, es)
64
+ def weight_send_data(tag, chunk)
23
65
  error = nil
24
66
 
25
67
  wlen = @weight_array.length
@@ -29,9 +71,8 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
29
71
 
30
72
  if node.available?
31
73
  begin
32
- send_data(node, tag, es)
74
+ send_data(node, tag, chunk)
33
75
  @node[tag] = node
34
- # $log.info "keep forwarding tag '#{tag}' to node '#{node.name}'", :host=>node.host, :port=>node.port, :weight=>node.weight
35
76
  return
36
77
  rescue
37
78
  # for load balancing during detecting crashed servers
@@ -41,11 +82,102 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
41
82
  end
42
83
 
43
84
  @node[tag] = nil
44
- # $log.info "keep forwarding tag '#{tag}' is lost"
45
85
  if error
46
86
  raise error
47
87
  else
48
88
  raise "no nodes are available" # TODO message
49
89
  end
50
90
  end
91
+
92
+ # Override for keepalive
93
+ def send_data(node, tag, chunk)
94
+ get_mutex(node).synchronize do
95
+ sock = get_sock[node]
96
+ unless sock
97
+ sock = reconnect(node)
98
+ end
99
+
100
+ begin
101
+ sock_write(sock, tag, chunk)
102
+ node.heartbeat(false)
103
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::ETIMEDOUT => e
104
+ $log.warn "out_keep_forward: #{e.class} #{e.message}"
105
+ sock = reconnect(node)
106
+ retry
107
+ end
108
+ end
109
+ end
110
+
111
+ def reconnect(node)
112
+ sock = connect(node)
113
+ opt = [1, @send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
114
+ sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
115
+
116
+ opt = [@send_timeout.to_i, 0].pack('L!L!') # struct timeval
117
+ sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
118
+
119
+ if @keepalive
120
+ get_sock[node] = sock
121
+ get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
122
+ end
123
+
124
+ sock
125
+ end
126
+
127
+ def sock_write(sock, tag, chunk)
128
+ # beginArray(2)
129
+ sock.write FORWARD_HEADER
130
+
131
+ # writeRaw(tag)
132
+ sock.write tag.to_msgpack # tag
133
+
134
+ # beginRaw(size)
135
+ sz = chunk.size
136
+ #if sz < 32
137
+ # # FixRaw
138
+ # sock.write [0xa0 | sz].pack('C')
139
+ #elsif sz < 65536
140
+ # # raw 16
141
+ # sock.write [0xda, sz].pack('Cn')
142
+ #else
143
+ # raw 32
144
+ sock.write [0xdb, sz].pack('CN')
145
+ #end
146
+
147
+ # writeRawBody(packed_es)
148
+ chunk.write_to(sock)
149
+ end
150
+
151
+ # watcher thread callback
152
+ def watch_keepalive_time
153
+ while true
154
+ sleep @watcher_interval
155
+ thread_ids = @sock.keys
156
+ thread_ids.each do |thread_id|
157
+ @sock[thread_id].each do |node, sock|
158
+ @mutex[thread_id][node].synchronize do
159
+ next unless sock_expired_at = @sock_expired_at[thread_id][node]
160
+ next unless Time.now >= sock_expired_at
161
+ sock.close rescue IOError if sock
162
+ @sock[thread_id][node] = nil
163
+ @sock_expired_at[thread_id][node] = nil
164
+ end
165
+ end
166
+ end
167
+ end
168
+ end
169
+
170
+ def get_mutex(node)
171
+ thread_id = Thread.current.object_id
172
+ @mutex[thread_id] ||= {}
173
+ @mutex[thread_id][node] ||= Mutex.new
174
+ end
175
+
176
+ def get_sock
177
+ @sock[Thread.current.object_id] ||= {}
178
+ end
179
+
180
+ def get_sock_expired_at
181
+ @sock_expired_at[Thread.current.object_id] ||= {}
182
+ end
51
183
  end
@@ -14,72 +14,134 @@ shared_context 'keep_forward_init' do
14
14
  </server>
15
15
  ]
16
16
  let(:tag) { 'syslog.host1' }
17
- let(:es) { Object.new }
17
+ let(:chunk) { [1] }
18
18
  let(:config) { CONFIG }
19
19
  let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::KeepForwardOutput, tag).configure(config).instance }
20
20
  end
21
21
 
22
22
  shared_context 'keep_forward_try_once' do
23
23
  before do
24
+ # stub connection
25
+ stub_sock = StringIO.new
26
+ driver.stub(:connect).and_return { stub_sock }
27
+ stub_sock.stub(:setsockopt)
28
+ driver.stub(:sock_write).and_return { nil }
29
+ Fluent::ForwardOutput::Node.any_instance.stub(:heartbeat).and_return { nil }
24
30
  # simpler version of Fluent::ForwardOutput#start method
31
+ driver.watcher_interval = 0
32
+ driver.start_watcher
25
33
  driver.instance_variable_set(:@rand_seed, Random.new.seed)
26
34
  driver.send(:rebuild_weight_array)
27
35
  driver.instance_variable_set(:@rr, 0)
28
- # try send once to cache keep_node
29
- driver.stub(:send_data) # stub
30
- driver.write_objects(tag, es)
36
+ driver.write_objects(tag, chunk)
37
+ end
38
+ after do
39
+ driver.stop_watcher
31
40
  end
32
41
  let!(:keep_node) { driver.instance_variable_get(:@node)[tag] }
33
- let!(:unkeep_node) { (driver.instance_variable_get(:@nodes) - [keep_node]).first }
42
+ let!(:another_node) { (driver.instance_variable_get(:@nodes) - [keep_node]).first }
43
+ end
44
+
45
+ shared_examples "keep_node_available" do
46
+ it { driver.write_objects(tag, chunk) }
47
+ end
48
+
49
+ shared_examples "keep_node_not_available" do
50
+ before { keep_node.available = false }
51
+ it { driver.write_objects(tag, chunk) }
52
+ end
53
+
54
+ shared_examples "prefer_recover false" do
55
+ let(:config) { CONFIG + %[prefer_recover false] }
56
+ before { driver.instance_variable_set(:@weight_array, [another_node]) }
57
+ it { driver.write_objects(tag, chunk) }
58
+ end
59
+
60
+ shared_context "prefer_recover true" do
61
+ let(:config) { CONFIG + %[prefer_recover true] }
62
+ before { driver.instance_variable_set(:@weight_array, [another_node]) }
63
+ it { driver.write_objects(tag, chunk) }
64
+ end
65
+
66
+ shared_examples "error_occurs" do
67
+ before { driver.stub(:send_data).with(keep_node, tag, chunk).and_raise(StandardError) }
68
+ it { driver.write_objects(tag, chunk) }
34
69
  end
35
70
 
36
71
  describe Fluent::KeepForwardOutput do
37
72
  include_context 'keep_forward_init'
38
73
  include_context 'keep_forward_try_once'
39
- before { driver.should_receive(:send_data).with(target, tag, es) } # mock
40
74
 
41
- describe 'keep forwarding if no problem?' do
42
- let(:target) { keep_node }
43
- it { driver.write_objects(tag, es) }
75
+ describe "keep_node" do
76
+ it_should_behave_like 'keep_node_available' do
77
+ before { driver.should_receive(:send_data).with(keep_node, tag, chunk) }
78
+ end
79
+ it_should_behave_like 'keep_node_not_available' do
80
+ before { driver.should_receive(:send_data).with(another_node, tag, chunk) }
81
+ end
82
+ it_should_behave_like 'prefer_recover true' do
83
+ before { driver.should_receive(:send_data).with(another_node, tag, chunk) }
84
+ end
85
+ it_should_behave_like 'prefer_recover false' do
86
+ before { driver.should_receive(:send_data).with(keep_node, tag, chunk) }
87
+ end
88
+ it_should_behave_like 'error_occurs' do
89
+ before { driver.stub(:weight_send_data).with(tag, chunk) } # re-call weight_send_data
90
+ end
44
91
  end
45
92
 
46
- describe 'switch if not available?' do
47
- before { keep_node.available = false }
48
-
49
- let(:target) { unkeep_node }
50
- it { driver.write_objects(tag, es) }
93
+ describe "keepalive false" do
94
+ let(:config) { CONFIG + %[keepalive false] }
95
+ it_should_behave_like 'keep_node_available' do
96
+ before { driver.should_receive(:reconnect) }
97
+ end
98
+ it_should_behave_like 'keep_node_not_available' do
99
+ before { driver.should_receive(:reconnect) }
100
+ end
101
+ it_should_behave_like 'prefer_recover true' do
102
+ let(:config) { CONFIG + %[prefer_recover true\nkeepalive false] }
103
+ before { driver.should_receive(:reconnect) }
104
+ end
105
+ it_should_behave_like 'prefer_recover false' do
106
+ let(:config) { CONFIG + %[prefer_recover false\nkeepalive false] }
107
+ before { driver.should_receive(:reconnect) }
108
+ end
51
109
  end
52
110
 
53
- describe 'not included in weight_array?' do
54
- context "switch if recover true" do
55
- let(:config) {
56
- CONFIG + %[
57
- prefer_recover true
58
- ]
59
- }
60
- before { driver.instance_variable_set(:@weight_array, [unkeep_node]) }
61
-
62
- let(:target) { unkeep_node }
63
- it { driver.write_objects(tag, es) }
111
+ describe "keepalive true" do
112
+ let(:config) { CONFIG + %[keepalive true] }
113
+ it_should_behave_like 'keep_node_available' do
114
+ before { driver.should_not_receive(:reconnect) }
64
115
  end
65
-
66
- context "not switch if recorver false" do
67
- let(:config) {
68
- CONFIG + %[
69
- prefer_recover false
70
- ]
71
- }
72
- before { driver.instance_variable_set(:@weight_array, [unkeep_node]) }
73
-
74
- let(:target) { keep_node }
75
- it { driver.write_objects(tag, es) }
116
+ it_should_behave_like 'keep_node_not_available' do
117
+ before { driver.should_receive(:reconnect) }
118
+ end
119
+ it_should_behave_like 'prefer_recover true' do
120
+ let(:config) { CONFIG + %[prefer_recover true\nkeepalive true] }
121
+ before { driver.should_receive(:reconnect) }
122
+ end
123
+ it_should_behave_like 'prefer_recover false' do
124
+ let(:config) { CONFIG + %[prefer_recover false\nkeepalive true] }
125
+ before { driver.should_not_receive(:reconnect) }
76
126
  end
77
127
  end
78
128
 
79
- describe 'switch if send_data to keep_node raises?' do
80
- before { driver.stub(:send_data).with(keep_node, tag, es).and_raise(StandardError) }
81
-
82
- let(:target) { unkeep_node }
83
- it { driver.write_objects(tag, es) }
129
+ describe "keepalive_time expired" do
130
+ let(:config) { CONFIG + %[keepalive true\nkeepalive_time 30] }
131
+ before { Delorean.jump 30 }
132
+ it_should_behave_like 'keep_node_available' do
133
+ before { sleep 1; driver.should_receive(:reconnect) }
134
+ end
135
+ it_should_behave_like 'keep_node_not_available' do
136
+ before { sleep 1; driver.should_receive(:reconnect) }
137
+ end
138
+ it_should_behave_like 'prefer_recover true' do
139
+ let(:config) { CONFIG + %[prefer_recover true\nkeepalive true\nkeepalive_time 30] }
140
+ before { sleep 1; driver.should_receive(:reconnect) }
141
+ end
142
+ it_should_behave_like 'prefer_recover false' do
143
+ let(:config) { CONFIG + %[prefer_recover false\nkeepalive true\nkeepalive_time 30] }
144
+ before { sleep 1; driver.should_receive(:reconnect) }
145
+ end
84
146
  end
85
147
  end
data/spec/spec_helper.rb CHANGED
@@ -7,6 +7,7 @@ Bundler.require(:default, :test)
7
7
  require 'fluent/test'
8
8
  require 'rspec'
9
9
  require 'pry'
10
+ require 'delorean'
10
11
 
11
12
  $TESTING=true
12
13
  $:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-keep-forward
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
- - Naotoshi SEO
7
+ - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-05-08 00:00:00.000000000 Z
11
+ date: 2013-12-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -66,7 +66,35 @@ dependencies:
66
66
  - - '>='
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
- description: out_foward extension to keep forwarding to a node
69
+ - !ruby/object:Gem::Dependency
70
+ name: pry-nav
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '>='
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '>='
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: delorean
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '>='
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '>='
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description: Fluentd plugin to keep forwarding to a node
70
98
  email:
71
99
  - sonots@gmail.com
72
100
  executables: []
@@ -76,8 +104,8 @@ files:
76
104
  - .gitignore
77
105
  - .rdebugrc
78
106
  - .rspec
79
- - .ruby-version
80
107
  - .travis.yml
108
+ - CHANGELOG.md
81
109
  - Gemfile
82
110
  - LICENSE
83
111
  - README.md
@@ -105,10 +133,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
105
133
  version: '0'
106
134
  requirements: []
107
135
  rubyforge_project: fluent-plugin-keep-forward
108
- rubygems_version: 2.0.0
136
+ rubygems_version: 2.0.3
109
137
  signing_key:
110
138
  specification_version: 4
111
- summary: out_foward extension to keep forwarding to a node
139
+ summary: Fluentd plugin to keep forwarding to a node
112
140
  test_files:
113
141
  - spec/out_keep_forward_spec.rb
114
142
  - spec/spec_helper.rb
data/.ruby-version DELETED
@@ -1 +0,0 @@
1
- 2.0.0-p0