fluent-plugin-keep-forward 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/CHANGELOG.md +11 -0
- data/README.md +6 -0
- data/fluent-plugin-keep-forward.gemspec +1 -1
- data/lib/fluent/plugin/out_keep_forward.rb +54 -13
- data/spec/out_keep_forward_spec.rb +2 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9c2af8c036f4fd144a1db429d94cc08988eec860
|
4
|
+
data.tar.gz: 376bb6ac2acceab104079b72c214e619e1f8afac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 51070129d6930e68aabad1e66c195f1355b3548302673c6bf3bf82585a4f9b3f3df31fdff45c5ce22da00b13c4c31154269f7285d066dcee1c8b19b3aaf5457a
|
7
|
+
data.tar.gz: 7708a5f03c1dae8c93b35525610f1cee0b3f9025aa2519b8fc58e6937dc421c779a3937262693736aa622d10cbd0db74ff4f31f1826d27059cac1d7d0f582630
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -20,6 +20,12 @@ Following parameters are additionally available:
|
|
20
20
|
|
21
21
|
Keepalive expired time. Default is nil (which means to keep connection as long as possible).
|
22
22
|
|
23
|
+
- keepforward
|
24
|
+
|
25
|
+
`one` for keep forwarding all data to the one node.
|
26
|
+
`tag` for keep forwarding data with the same tag to the same node.
|
27
|
+
Default is `one`.
|
28
|
+
|
23
29
|
- prefer_recover (bool)
|
24
30
|
|
25
31
|
Switch connection to a recovered node from standby nodes or less weighted nodes. Default is `true`.
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-keep-forward"
|
6
|
-
s.version = "0.1.
|
6
|
+
s.version = "0.1.3"
|
7
7
|
s.authors = ["Naotoshi Seo"]
|
8
8
|
s.email = ["sonots@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/sonots/fluent-plugin-keep-forward"
|
@@ -11,6 +11,16 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
11
11
|
config_param :prefer_recover, :bool, :default => true
|
12
12
|
config_param :keepalive, :bool, :default => false
|
13
13
|
config_param :keepalive_time, :time, :default => nil # infinite
|
14
|
+
config_param :keepforward, :default => :one do |val|
|
15
|
+
case val.downcase
|
16
|
+
when 'one'
|
17
|
+
:one
|
18
|
+
when 'tag'
|
19
|
+
:tag
|
20
|
+
else
|
21
|
+
raise ConfigError, "out_keep_forward keepforward should be 'one' or 'tag'"
|
22
|
+
end
|
23
|
+
end
|
14
24
|
|
15
25
|
# for test
|
16
26
|
attr_accessor :watcher_interval
|
@@ -25,6 +35,18 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
25
35
|
@watcher_interval = 1
|
26
36
|
end
|
27
37
|
|
38
|
+
def get_node(tag)
|
39
|
+
@node[keepforward(tag)]
|
40
|
+
end
|
41
|
+
|
42
|
+
def keepforward(tag)
|
43
|
+
@keepforward == :one ? :one : tag
|
44
|
+
end
|
45
|
+
|
46
|
+
def cache_node(tag, node)
|
47
|
+
@node[keepforward(tag)] = node
|
48
|
+
end
|
49
|
+
|
28
50
|
def start
|
29
51
|
super
|
30
52
|
start_watcher
|
@@ -52,23 +74,29 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
52
74
|
def write_objects(tag, chunk)
|
53
75
|
return if chunk.empty?
|
54
76
|
error = nil
|
55
|
-
node =
|
77
|
+
node = get_node(tag)
|
56
78
|
|
57
79
|
if node and node.available? and (!@prefer_recover or @weight_array.include?(node))
|
58
80
|
begin
|
59
81
|
send_data(node, tag, chunk)
|
60
82
|
return
|
61
83
|
rescue
|
62
|
-
weight_send_data(tag, chunk)
|
84
|
+
node = weight_send_data(tag, chunk, error_node = node)
|
85
|
+
cache_node(tag, node)
|
63
86
|
end
|
64
87
|
else
|
65
|
-
weight_send_data(tag, chunk)
|
88
|
+
node = weight_send_data(tag, chunk, error_node = node)
|
89
|
+
cache_node(tag, node)
|
66
90
|
end
|
67
91
|
end
|
68
92
|
|
69
|
-
def weight_send_data(tag, chunk)
|
93
|
+
def weight_send_data(tag, chunk, error_node = nil)
|
70
94
|
error = nil
|
71
95
|
|
96
|
+
if error_node
|
97
|
+
sock_close(error_node) if @keepalive and @keepforward == :one
|
98
|
+
end
|
99
|
+
|
72
100
|
wlen = @weight_array.length
|
73
101
|
wlen.times do
|
74
102
|
@rr = (@rr + 1) % wlen
|
@@ -77,8 +105,7 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
77
105
|
if node.available?
|
78
106
|
begin
|
79
107
|
send_data(node, tag, chunk)
|
80
|
-
|
81
|
-
return
|
108
|
+
return node
|
82
109
|
rescue
|
83
110
|
# for load balancing during detecting crashed servers
|
84
111
|
error = $! # use the latest error
|
@@ -86,7 +113,7 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
86
113
|
end
|
87
114
|
end
|
88
115
|
|
89
|
-
|
116
|
+
cache_node(tag, nil)
|
90
117
|
if error
|
91
118
|
raise error
|
92
119
|
else
|
@@ -100,15 +127,20 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
100
127
|
sock = get_sock[node]
|
101
128
|
unless sock
|
102
129
|
sock = reconnect(node)
|
130
|
+
cache_sock(node, sock) if @keepalive
|
103
131
|
end
|
104
132
|
|
105
133
|
begin
|
106
134
|
sock_write(sock, tag, chunk)
|
107
135
|
node.heartbeat(false)
|
108
136
|
rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::ETIMEDOUT => e
|
109
|
-
log.warn "out_keep_forward: #{e.class} #{e.message}"
|
137
|
+
log.warn "out_keep_forward: send_data failed #{e.class} #{e.message}, try to reconnect", :host=>node.host, :port=>node.port
|
138
|
+
sock.close rescue IOError
|
110
139
|
sock = reconnect(node)
|
140
|
+
cache_sock(node, sock) if @keepalive
|
111
141
|
retry
|
142
|
+
ensure
|
143
|
+
sock.close if sock and !@keepalive
|
112
144
|
end
|
113
145
|
end
|
114
146
|
end
|
@@ -121,11 +153,6 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
121
153
|
opt = [@send_timeout.to_i, 0].pack('L!L!') # struct timeval
|
122
154
|
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
123
155
|
|
124
|
-
if @keepalive
|
125
|
-
get_sock[node] = sock
|
126
|
-
get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
|
127
|
-
end
|
128
|
-
|
129
156
|
sock
|
130
157
|
end
|
131
158
|
|
@@ -172,12 +199,26 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
|
|
172
199
|
end
|
173
200
|
end
|
174
201
|
|
202
|
+
def sock_close(node)
|
203
|
+
get_mutex(node).synchronize do
|
204
|
+
sock = get_sock[node]
|
205
|
+
sock.close rescue IOError if sock
|
206
|
+
get_sock[node] = nil
|
207
|
+
get_sock_expired_at[node] = nil
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
175
211
|
def get_mutex(node)
|
176
212
|
thread_id = Thread.current.object_id
|
177
213
|
@mutex[thread_id] ||= {}
|
178
214
|
@mutex[thread_id][node] ||= Mutex.new
|
179
215
|
end
|
180
216
|
|
217
|
+
def cache_sock(node, sock)
|
218
|
+
get_sock[node] = sock
|
219
|
+
get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
|
220
|
+
end
|
221
|
+
|
181
222
|
def get_sock
|
182
223
|
@sock[Thread.current.object_id] ||= {}
|
183
224
|
end
|
@@ -38,7 +38,7 @@ shared_context 'keep_forward_try_once' do
|
|
38
38
|
after do
|
39
39
|
driver.stop_watcher
|
40
40
|
end
|
41
|
-
let!(:keep_node) { driver.
|
41
|
+
let!(:keep_node) { driver.get_node(tag) }
|
42
42
|
let!(:another_node) { (driver.instance_variable_get(:@nodes) - [keep_node]).first }
|
43
43
|
end
|
44
44
|
|
@@ -86,7 +86,7 @@ describe Fluent::KeepForwardOutput do
|
|
86
86
|
before { driver.should_receive(:send_data).with(keep_node, tag, chunk) }
|
87
87
|
end
|
88
88
|
it_should_behave_like 'error_occurs' do
|
89
|
-
before { driver.stub(:weight_send_data).with(tag, chunk) } # re-call weight_send_data
|
89
|
+
before { driver.stub(:weight_send_data).with(tag, chunk, keep_node) } # re-call weight_send_data
|
90
90
|
end
|
91
91
|
end
|
92
92
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-keep-forward
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naotoshi Seo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-03-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|