fluent-plugin-keep-forward 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ae46708b29b44194062b9985c6d5cab371785a27
4
- data.tar.gz: 2038eb737536f4bacd779e0d2f40a8a2081f6f14
3
+ metadata.gz: 9c2af8c036f4fd144a1db429d94cc08988eec860
4
+ data.tar.gz: 376bb6ac2acceab104079b72c214e619e1f8afac
5
5
  SHA512:
6
- metadata.gz: 8f885899158c22a334a6fe027dad4fd5c2ff48e081bb78b93c56593aeb2337411d4b64251ffb54d43f8b5f0a579b30d3cc01a7daf80030536e67006484777366
7
- data.tar.gz: 74cec077f9b1bf83d62eac10a9ab6e8308a58a5b0baa1f0868ae4ace0a234e3fc5a313e9e18330aca0d2556a1194a32639b74f791bb9277d5a490b3b4809eece
6
+ metadata.gz: 51070129d6930e68aabad1e66c195f1355b3548302673c6bf3bf82585a4f9b3f3df31fdff45c5ce22da00b13c4c31154269f7285d066dcee1c8b19b3aaf5457a
7
+ data.tar.gz: 7708a5f03c1dae8c93b35525610f1cee0b3f9025aa2519b8fc58e6937dc421c779a3937262693736aa622d10cbd0db74ff4f31f1826d27059cac1d7d0f582630
data/.travis.yml CHANGED
@@ -1,6 +1,6 @@
1
1
  rvm:
2
- - 1.9.2
3
2
  - 1.9.3
4
3
  - 2.0.0
4
+ - 2.1.*
5
5
  gemfile:
6
6
  - Gemfile
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ ## 0.1.3 (2014/03/19)
2
+
3
+ Enhancement
4
+
5
+ * Enrich warn log message
6
+ * Add `keepforward` option
7
+
8
+ Fixes
9
+
10
+ * Explicitly close socket the case of non-keepalive
11
+
1
12
  ## 0.1.2
2
13
 
3
14
  Enhancement:
data/README.md CHANGED
@@ -20,6 +20,12 @@ Following parameters are additionally available:
20
20
 
21
21
  Keepalive expired time. Default is nil (which means to keep connection as long as possible).
22
22
 
23
+ - keepforward
24
+
25
+ `one` for keep forwarding all data to the one node.
26
+ `tag` for keep forwarding data with the same tag to the same node.
27
+ Default is `one`.
28
+
23
29
  - prefer_recover (bool)
24
30
 
25
31
  Switch connection to a recovered node from standby nodes or less weighted nodes. Default is `true`.
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-keep-forward"
6
- s.version = "0.1.2"
6
+ s.version = "0.1.3"
7
7
  s.authors = ["Naotoshi Seo"]
8
8
  s.email = ["sonots@gmail.com"]
9
9
  s.homepage = "https://github.com/sonots/fluent-plugin-keep-forward"
@@ -11,6 +11,16 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
11
11
  config_param :prefer_recover, :bool, :default => true
12
12
  config_param :keepalive, :bool, :default => false
13
13
  config_param :keepalive_time, :time, :default => nil # infinite
14
+ config_param :keepforward, :default => :one do |val|
15
+ case val.downcase
16
+ when 'one'
17
+ :one
18
+ when 'tag'
19
+ :tag
20
+ else
21
+ raise ConfigError, "out_keep_forward keepforward should be 'one' or 'tag'"
22
+ end
23
+ end
14
24
 
15
25
  # for test
16
26
  attr_accessor :watcher_interval
@@ -25,6 +35,18 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
25
35
  @watcher_interval = 1
26
36
  end
27
37
 
38
+ def get_node(tag)
39
+ @node[keepforward(tag)]
40
+ end
41
+
42
+ def keepforward(tag)
43
+ @keepforward == :one ? :one : tag
44
+ end
45
+
46
+ def cache_node(tag, node)
47
+ @node[keepforward(tag)] = node
48
+ end
49
+
28
50
  def start
29
51
  super
30
52
  start_watcher
@@ -52,23 +74,29 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
52
74
  def write_objects(tag, chunk)
53
75
  return if chunk.empty?
54
76
  error = nil
55
- node = @node[tag]
77
+ node = get_node(tag)
56
78
 
57
79
  if node and node.available? and (!@prefer_recover or @weight_array.include?(node))
58
80
  begin
59
81
  send_data(node, tag, chunk)
60
82
  return
61
83
  rescue
62
- weight_send_data(tag, chunk)
84
+ node = weight_send_data(tag, chunk, error_node = node)
85
+ cache_node(tag, node)
63
86
  end
64
87
  else
65
- weight_send_data(tag, chunk)
88
+ node = weight_send_data(tag, chunk, error_node = node)
89
+ cache_node(tag, node)
66
90
  end
67
91
  end
68
92
 
69
- def weight_send_data(tag, chunk)
93
+ def weight_send_data(tag, chunk, error_node = nil)
70
94
  error = nil
71
95
 
96
+ if error_node
97
+ sock_close(error_node) if @keepalive and @keepforward == :one
98
+ end
99
+
72
100
  wlen = @weight_array.length
73
101
  wlen.times do
74
102
  @rr = (@rr + 1) % wlen
@@ -77,8 +105,7 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
77
105
  if node.available?
78
106
  begin
79
107
  send_data(node, tag, chunk)
80
- @node[tag] = node
81
- return
108
+ return node
82
109
  rescue
83
110
  # for load balancing during detecting crashed servers
84
111
  error = $! # use the latest error
@@ -86,7 +113,7 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
86
113
  end
87
114
  end
88
115
 
89
- @node[tag] = nil
116
+ cache_node(tag, nil)
90
117
  if error
91
118
  raise error
92
119
  else
@@ -100,15 +127,20 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
100
127
  sock = get_sock[node]
101
128
  unless sock
102
129
  sock = reconnect(node)
130
+ cache_sock(node, sock) if @keepalive
103
131
  end
104
132
 
105
133
  begin
106
134
  sock_write(sock, tag, chunk)
107
135
  node.heartbeat(false)
108
136
  rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::ETIMEDOUT => e
109
- log.warn "out_keep_forward: #{e.class} #{e.message}"
137
+ log.warn "out_keep_forward: send_data failed #{e.class} #{e.message}, try to reconnect", :host=>node.host, :port=>node.port
138
+ sock.close rescue IOError
110
139
  sock = reconnect(node)
140
+ cache_sock(node, sock) if @keepalive
111
141
  retry
142
+ ensure
143
+ sock.close if sock and !@keepalive
112
144
  end
113
145
  end
114
146
  end
@@ -121,11 +153,6 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
121
153
  opt = [@send_timeout.to_i, 0].pack('L!L!') # struct timeval
122
154
  sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
123
155
 
124
- if @keepalive
125
- get_sock[node] = sock
126
- get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
127
- end
128
-
129
156
  sock
130
157
  end
131
158
 
@@ -172,12 +199,26 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
172
199
  end
173
200
  end
174
201
 
202
+ def sock_close(node)
203
+ get_mutex(node).synchronize do
204
+ sock = get_sock[node]
205
+ sock.close rescue IOError if sock
206
+ get_sock[node] = nil
207
+ get_sock_expired_at[node] = nil
208
+ end
209
+ end
210
+
175
211
  def get_mutex(node)
176
212
  thread_id = Thread.current.object_id
177
213
  @mutex[thread_id] ||= {}
178
214
  @mutex[thread_id][node] ||= Mutex.new
179
215
  end
180
216
 
217
+ def cache_sock(node, sock)
218
+ get_sock[node] = sock
219
+ get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
220
+ end
221
+
181
222
  def get_sock
182
223
  @sock[Thread.current.object_id] ||= {}
183
224
  end
@@ -38,7 +38,7 @@ shared_context 'keep_forward_try_once' do
38
38
  after do
39
39
  driver.stop_watcher
40
40
  end
41
- let!(:keep_node) { driver.instance_variable_get(:@node)[tag] }
41
+ let!(:keep_node) { driver.get_node(tag) }
42
42
  let!(:another_node) { (driver.instance_variable_get(:@nodes) - [keep_node]).first }
43
43
  end
44
44
 
@@ -86,7 +86,7 @@ describe Fluent::KeepForwardOutput do
86
86
  before { driver.should_receive(:send_data).with(keep_node, tag, chunk) }
87
87
  end
88
88
  it_should_behave_like 'error_occurs' do
89
- before { driver.stub(:weight_send_data).with(tag, chunk) } # re-call weight_send_data
89
+ before { driver.stub(:weight_send_data).with(tag, chunk, keep_node) } # re-call weight_send_data
90
90
  end
91
91
  end
92
92
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-keep-forward
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-02-04 00:00:00.000000000 Z
11
+ date: 2014-03-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd