fluentd 1.2.0.pre1 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

@@ -0,0 +1,65 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ module Counter
19
+ class BaseError < StandardError
20
+ def to_hash
21
+ { 'code' => code, 'message' => message }
22
+ end
23
+
24
+ def code
25
+ raise NotImplementedError
26
+ end
27
+ end
28
+
29
+ class InvalidParams < BaseError
30
+ def code
31
+ 'invalid_params'
32
+ end
33
+ end
34
+
35
+ class UnknownKey < BaseError
36
+ def code
37
+ 'unknown_key'
38
+ end
39
+ end
40
+
41
+ class ParseError < BaseError
42
+ def code
43
+ 'parse_error'
44
+ end
45
+ end
46
+
47
+ class InvalidRequest < BaseError
48
+ def code
49
+ 'invalid_request'
50
+ end
51
+ end
52
+
53
+ class MethodNotFound < BaseError
54
+ def code
55
+ 'method_not_found'
56
+ end
57
+ end
58
+
59
+ class InternalServerError < BaseError
60
+ def code
61
+ 'internal_server_error'
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,163 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'timeout'
18
+
19
+ module Fluent
20
+ module Counter
21
+ class MutexHash
22
+ def initialize(data_store)
23
+ @mutex = Mutex.new
24
+ @data_store = data_store
25
+ @mutex_hash = {}
26
+ @thread = nil
27
+ @cleanup_thread = CleanupThread.new(@data_store, @mutex_hash, @mutex)
28
+ end
29
+
30
+ def start
31
+ @data_store.start
32
+ @cleanup_thread.start
33
+ end
34
+
35
+ def stop
36
+ @data_store.stop
37
+ @cleanup_thread.stop
38
+ end
39
+
40
+ def synchronize(*keys)
41
+ return if keys.empty?
42
+
43
+ locks = {}
44
+ loop do
45
+ @mutex.synchronize do
46
+ keys.each do |key|
47
+ mutex = @mutex_hash[key]
48
+ unless mutex
49
+ v = Mutex.new
50
+ @mutex_hash[key] = v
51
+ mutex = v
52
+ end
53
+
54
+ if mutex.try_lock
55
+ locks[key] = mutex
56
+ else
57
+ locks.values.each(&:unlock)
58
+ locks = {} # flush locked keys
59
+ break
60
+ end
61
+ end
62
+ end
63
+
64
+ next if locks.empty? # failed to lock all keys
65
+
66
+ locks.each do |(k, v)|
67
+ yield @data_store, k
68
+ v.unlock
69
+ end
70
+ break
71
+ end
72
+ end
73
+
74
+ def synchronize_keys(*keys)
75
+ return if keys.empty?
76
+ keys = keys.dup
77
+
78
+ while key = keys.shift
79
+ @mutex.lock
80
+
81
+ mutex = @mutex_hash[key]
82
+ unless mutex
83
+ v = Mutex.new
84
+ @mutex_hash[key] = v
85
+ mutex = v
86
+ end
87
+
88
+ if mutex.try_lock
89
+ @mutex.unlock
90
+ yield @data_store, key
91
+ mutex.unlock
92
+ else
93
+ # release global lock
94
+ @mutex.unlock
95
+ keys.push(key) # failed lock, retry this key
96
+ end
97
+ end
98
+ end
99
+ end
100
+
101
+ class CleanupThread
102
+ CLEANUP_INTERVAL = 60 * 15 # 15 min
103
+
104
+ def initialize(store, mutex_hash, mutex)
105
+ @store = store
106
+ @mutex_hash = mutex_hash
107
+ @mutex = mutex
108
+ @thread = nil
109
+ @running = false
110
+ end
111
+
112
+ def start
113
+ @running = true
114
+ @thread = Thread.new do
115
+ while @running
116
+ sleep CLEANUP_INTERVAL
117
+ run_once
118
+ end
119
+ end
120
+ end
121
+
122
+ def stop
123
+ return unless @running
124
+ @running = false
125
+ begin
126
+ # Avoid waiting CLEANUP_INTERVAL
127
+ Timeout.timeout(1) do
128
+ @thread.join
129
+ end
130
+ rescue Timeout::Error
131
+ @thread.kill
132
+ end
133
+ end
134
+
135
+ private
136
+
137
+ def run_once
138
+ @mutex.synchronize do
139
+ last_cleanup_at = (Time.now - CLEANUP_INTERVAL).to_i
140
+ @mutex_hash.each do |(key, mutex)|
141
+ v = @store.get(key, raw: true)
142
+ next unless v
143
+ next if last_cleanup_at < v['last_modified_at'][0] # v['last_modified_at'] = [sec, nsec]
144
+ next unless mutex.try_lock
145
+
146
+ @mutex_hash[key] = nil
147
+ mutex.unlock
148
+
149
+ # Check that a waiting thread is in a lock queue.
150
+ # Can't get a lock here means this key is used in other places.
151
+ # So restore a mutex value to a corresponding key.
152
+ if mutex.try_lock
153
+ @mutex_hash.delete(key)
154
+ mutex.unlock
155
+ else
156
+ @mutex_hash[key] = mutex
157
+ end
158
+ end
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -0,0 +1,273 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'cool.io'
18
+ require 'fluent/counter/base_socket'
19
+ require 'fluent/counter/validator'
20
+ require 'fluent/counter/store'
21
+ require 'fluent/counter/mutex_hash'
22
+
23
+ module Fluent
24
+ module Counter
25
+ class Server
26
+ DEFAULT_ADDR = '127.0.0.1'
27
+ DEFAULT_PORT = 24321
28
+
29
+ def initialize(name, opt = {})
30
+ raise 'Counter server name is invalid' unless Validator::VALID_NAME =~ name
31
+ @name = name
32
+ @opt = opt
33
+ @addr = @opt[:addr] || DEFAULT_ADDR
34
+ @port = @opt[:port] || DEFAULT_PORT
35
+ @loop = @opt[:loop] || Coolio::Loop.new
36
+ @log = @opt[:log] || $log
37
+
38
+ @store = Fluent::Counter::Store.new(opt)
39
+ @mutex_hash = MutexHash.new(@store)
40
+
41
+ @server = Coolio::TCPServer.new(@addr, @port, Handler, method(:on_message))
42
+ @thread = nil
43
+ @running = false
44
+ end
45
+
46
+ def start
47
+ @server.attach(@loop)
48
+ @thread = Thread.new do
49
+ @running = true
50
+ @loop.run(0.5)
51
+ @running = false
52
+ end
53
+ @log.debug("starting counter server #{@addr}:#{@port}")
54
+ @mutex_hash.start
55
+ self
56
+ end
57
+
58
+ def stop
59
+ # This `sleep` for a test to wait for a `@loop` to begin to run
60
+ sleep 0.1
61
+ @server.close
62
+ @loop.stop if @running
63
+ @mutex_hash.stop
64
+ @thread.join if @thread
65
+ @log.debug("calling stop in counter server #{@addr}:#{@port}")
66
+ end
67
+
68
+ def on_message(data)
69
+ errors = Validator.request(data)
70
+ unless errors.empty?
71
+ return { 'id' => data['id'], 'data' => [], 'errors' => errors }
72
+ end
73
+
74
+ result = safe_run do
75
+ send(data['method'], data['params'], data['scope'], data['options'])
76
+ end
77
+ result.merge('id' => data['id'])
78
+ rescue => e
79
+ @log.error e.to_s
80
+ end
81
+
82
+ private
83
+
84
+ def establish(params, _scope, _options)
85
+ validator = Fluent::Counter::ArrayValidator.new(:empty, :scope)
86
+ valid_params, errors = validator.call(params)
87
+ res = Response.new(errors)
88
+
89
+ if scope = valid_params.first
90
+ new_scope = "#{@name}\t#{scope}"
91
+ res.push_data new_scope
92
+ @log.debug("Establish new key: #{new_scope}")
93
+ end
94
+
95
+ res.to_hash
96
+ end
97
+
98
+ def init(params, scope, options)
99
+ validator = Fluent::Counter::HashValidator.new(:empty, :name, :reset_interval)
100
+ valid_params, errors = validator.call(params)
101
+ res = Response.new(errors)
102
+ key_hash = valid_params.reduce({}) do |acc, vp|
103
+ acc.merge(Store.gen_key(scope, vp['name']) => vp)
104
+ end
105
+
106
+ do_init = lambda do |store, key|
107
+ begin
108
+ param = key_hash[key]
109
+ v = store.init(key, param, ignore: options['ignore'])
110
+ @log.debug("Create new key: #{param['name']}")
111
+ res.push_data v
112
+ rescue => e
113
+ res.push_error e
114
+ end
115
+ end
116
+
117
+ if options['random']
118
+ @mutex_hash.synchronize_keys(*(key_hash.keys), &do_init)
119
+ else
120
+ @mutex_hash.synchronize(*(key_hash.keys), &do_init)
121
+ end
122
+
123
+ res.to_hash
124
+ end
125
+
126
+ def delete(params, scope, options)
127
+ validator = Fluent::Counter::ArrayValidator.new(:empty, :key)
128
+ valid_params, errors = validator.call(params)
129
+ res = Response.new(errors)
130
+ keys = valid_params.map { |vp| Store.gen_key(scope, vp) }
131
+
132
+ do_delete = lambda do |store, key|
133
+ begin
134
+ v = store.delete(key)
135
+ @log.debug("delete a key: #{key}")
136
+ res.push_data v
137
+ rescue => e
138
+ res.push_error e
139
+ end
140
+ end
141
+
142
+ if options['random']
143
+ @mutex_hash.synchronize_keys(*keys, &do_delete)
144
+ else
145
+ @mutex_hash.synchronize(*keys, &do_delete)
146
+ end
147
+
148
+ res.to_hash
149
+ end
150
+
151
+ def inc(params, scope, options)
152
+ validate_param = [:empty, :name, :value]
153
+ validate_param << :reset_interval if options['force']
154
+ validator = Fluent::Counter::HashValidator.new(*validate_param)
155
+ valid_params, errors = validator.call(params)
156
+ res = Response.new(errors)
157
+ key_hash = valid_params.reduce({}) do |acc, vp|
158
+ acc.merge(Store.gen_key(scope, vp['name']) => vp)
159
+ end
160
+
161
+ do_inc = lambda do |store, key|
162
+ begin
163
+ param = key_hash[key]
164
+ v = store.inc(key, param, force: options['force'])
165
+ @log.debug("Increment #{key} by #{param['value']}")
166
+ res.push_data v
167
+ rescue => e
168
+ res.push_error e
169
+ end
170
+ end
171
+
172
+ if options['random']
173
+ @mutex_hash.synchronize_keys(*(key_hash.keys), &do_inc)
174
+ else
175
+ @mutex_hash.synchronize(*(key_hash.keys), &do_inc)
176
+ end
177
+
178
+ res.to_hash
179
+ end
180
+
181
+ def reset(params, scope, options)
182
+ validator = Fluent::Counter::ArrayValidator.new(:empty, :key)
183
+ valid_params, errors = validator.call(params)
184
+ res = Response.new(errors)
185
+ keys = valid_params.map { |vp| Store.gen_key(scope, vp) }
186
+
187
+ do_reset = lambda do |store, key|
188
+ begin
189
+ v = store.reset(key)
190
+ @log.debug("Reset #{key}'s' counter value")
191
+ res.push_data v
192
+ rescue => e
193
+ res.push_error e
194
+ end
195
+ end
196
+
197
+ if options['random']
198
+ @mutex_hash.synchronize_keys(*keys, &do_reset)
199
+ else
200
+ @mutex_hash.synchronize(*keys, &do_reset)
201
+ end
202
+
203
+ res.to_hash
204
+ end
205
+
206
+ def get(params, scope, _options)
207
+ validator = Fluent::Counter::ArrayValidator.new(:empty, :key)
208
+ valid_params, errors = validator.call(params)
209
+ res = Response.new(errors)
210
+
211
+ keys = valid_params.map { |vp| Store.gen_key(scope, vp) }
212
+ keys.each do |key|
213
+ begin
214
+ v = @store.get(key, raise_error: true)
215
+ @log.debug("Get counter value: #{key}")
216
+ res.push_data v
217
+ rescue => e
218
+ res.push_error e
219
+ end
220
+ end
221
+ res.to_hash
222
+ end
223
+
224
+ def safe_run
225
+ yield
226
+ rescue => e
227
+ {
228
+ 'errors' => [InternalServerError.new(e).to_hash],
229
+ 'data' => []
230
+ }
231
+ end
232
+
233
+ class Response
234
+ def initialize(errors = [], data = [])
235
+ @errors = errors
236
+ @data = data
237
+ end
238
+
239
+ def push_error(error)
240
+ @errors << error
241
+ end
242
+
243
+ def push_data(data)
244
+ @data << data
245
+ end
246
+
247
+ def to_hash
248
+ if @errors.empty?
249
+ { 'data' => @data }
250
+ else
251
+ errors = @errors.map do |e|
252
+ error = e.respond_to?(:to_hash) ? e : InternalServerError.new(e.to_s)
253
+ error.to_hash
254
+ end
255
+ { 'data' => @data, 'errors' => errors }
256
+ end
257
+ end
258
+ end
259
+ end
260
+
261
+ class Handler < Fluent::Counter::BaseSocket
262
+ def initialize(io, on_message)
263
+ super(io)
264
+ @on_message = on_message
265
+ end
266
+
267
+ def on_message(data)
268
+ res = @on_message.call(data)
269
+ packed_write res if res
270
+ end
271
+ end
272
+ end
273
+ end