fluent-plugin-numeric-counter 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.travis.yml +4 -0
- data/README.md +48 -1
- data/fluent-plugin-numeric-counter.gemspec +3 -1
- data/lib/fluent/plugin/out_numeric_counter.rb +82 -4
- data/test/helper.rb +1 -0
- data/test/plugin/test_out_numeric_counter.rb +58 -0
- metadata +29 -19
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: bf34aed7a1bdec3baef3baca72cccc6ee8e858f3
|
4
|
+
data.tar.gz: e77034978932949f7f41d84ac426d6b8152d5d0e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 748cbafe875ef82c2069db03b8fb24e8ff9bd008abb75dde27d1853b5379a155156f7ec627a0c2adafecdb73a07b4590c78998fa7f5fd1af8440867e062cc13e
|
7
|
+
data.tar.gz: 468015aeee1dffdad2679eb85b829789569728d84f36a7c179d915cc883f6fc7dbabd4c24b016eba7ef14dd5eb4f55e49bef050a77ce226dcc5cfe024f315c58
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
### NumericCounterOutput
|
6
6
|
|
7
|
-
Fluentd plugin to count messages, matches for numeric range patterns, and emits its result (like fluent-plugin-datacounter).
|
7
|
+
[Fluentd](http://fluentd.org) plugin to count messages, matches for numeric range patterns, and emits its result (like fluent-plugin-datacounter).
|
8
8
|
|
9
9
|
- Counts per min/hour/day
|
10
10
|
- Counts per second (average every min/hour/day)
|
@@ -113,6 +113,53 @@ And you can get tested messages count with 'output\_messages' option:
|
|
113
113
|
# => tag: 'num.foo' or 'num.bar'
|
114
114
|
# message: {'messages' => xxx, 'SMALL_count' => 100, ... }
|
115
115
|
|
116
|
+
## Parameters
|
117
|
+
|
118
|
+
* count\_key (required)
|
119
|
+
|
120
|
+
The key to count in the event record.
|
121
|
+
|
122
|
+
* tag
|
123
|
+
|
124
|
+
The output tag. Default is `numcount`.
|
125
|
+
|
126
|
+
* tag\_prefix
|
127
|
+
|
128
|
+
The prefix string which will be added to the input tag. `output_per_tag yes` must be specified together.
|
129
|
+
|
130
|
+
* input\_tag\_remove\_prefix
|
131
|
+
|
132
|
+
The prefix string which will be removed from the input tag.
|
133
|
+
|
134
|
+
* count\_interval
|
135
|
+
|
136
|
+
The interval time to count in seconds. Default is `60`.
|
137
|
+
|
138
|
+
* unit
|
139
|
+
|
140
|
+
The interval time to monitor specified an unit (either of `minute`, `hour`, or `day`).
|
141
|
+
Use either of `count_interval` or `unit`.
|
142
|
+
|
143
|
+
* aggregate
|
144
|
+
|
145
|
+
Calculate in each input `tag` separetely, or `all` records in a mass. Default is `tag`.
|
146
|
+
|
147
|
+
* ouput\_per\_tag
|
148
|
+
|
149
|
+
Emit for each input tag. `tag_prefix` must be specified together. Default is `no`.
|
150
|
+
|
151
|
+
* outcast\_unmatched
|
152
|
+
|
153
|
+
Specify `yes` if you do not want to include 'unmatched' counts into percentage. Default is `no`.
|
154
|
+
|
155
|
+
* output\_messages
|
156
|
+
|
157
|
+
Specify `yes` if you want to get tested messages. Default is `no`.
|
158
|
+
|
159
|
+
* store\_file
|
160
|
+
|
161
|
+
Store internal data into a file of the given path on shutdown, and load on starting.
|
162
|
+
|
116
163
|
## TODO
|
117
164
|
|
118
165
|
* more tests
|
@@ -2,12 +2,13 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-numeric-counter"
|
5
|
-
gem.version = "0.2.
|
5
|
+
gem.version = "0.2.2"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.description = %q{Counts messages, with specified key and numeric value in specified range}
|
9
9
|
gem.summary = %q{Fluentd plugin to count messages with specified numeric values}
|
10
10
|
gem.homepage = "https://github.com/tagomoris/fluent-plugin-numeric-counter"
|
11
|
+
gem.license = "APLv2"
|
11
12
|
|
12
13
|
gem.files = `git ls-files`.split($\)
|
13
14
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
@@ -15,5 +16,6 @@ Gem::Specification.new do |gem|
|
|
15
16
|
gem.require_paths = ["lib"]
|
16
17
|
|
17
18
|
gem.add_development_dependency "rake"
|
19
|
+
gem.add_development_dependency "delorean"
|
18
20
|
gem.add_runtime_dependency "fluentd"
|
19
21
|
end
|
@@ -1,6 +1,11 @@
|
|
1
1
|
class Fluent::NumericCounterOutput < Fluent::Output
|
2
2
|
Fluent::Plugin.register_output('numeric_counter', self)
|
3
3
|
|
4
|
+
def initialize
|
5
|
+
super
|
6
|
+
require 'pathname'
|
7
|
+
end
|
8
|
+
|
4
9
|
PATTERN_MAX_NUM = 20
|
5
10
|
|
6
11
|
config_param :count_interval, :time, :default => 60
|
@@ -13,6 +18,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
13
18
|
config_param :count_key, :string
|
14
19
|
config_param :outcast_unmatched, :bool, :default => false
|
15
20
|
config_param :output_messages, :bool, :default => false
|
21
|
+
config_param :store_file, :string, :default => nil
|
16
22
|
|
17
23
|
# pattern0 reserved as unmatched counts
|
18
24
|
config_param :pattern1, :string # string: NAME LOW HIGH
|
@@ -20,11 +26,18 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
20
26
|
(2..PATTERN_MAX_NUM).each do |i|
|
21
27
|
config_param ('pattern' + i.to_s).to_sym, :string, :default => nil
|
22
28
|
end
|
23
|
-
|
24
|
-
attr_accessor :counts, :last_checked
|
25
29
|
|
30
|
+
attr_accessor :counts
|
31
|
+
attr_accessor :last_checked
|
32
|
+
attr_accessor :saved_duration
|
33
|
+
attr_accessor :saved_at
|
26
34
|
attr_accessor :patterns
|
27
35
|
|
36
|
+
# Define `log` method for v0.10.42 or earlier
|
37
|
+
unless method_defined?(:log)
|
38
|
+
define_method("log") { $log }
|
39
|
+
end
|
40
|
+
|
28
41
|
def parse_num(str)
|
29
42
|
if str.nil?
|
30
43
|
nil
|
@@ -58,7 +71,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
58
71
|
|
59
72
|
invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
|
60
73
|
if invalids.size > 0
|
61
|
-
|
74
|
+
log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):" + invalids.join(",")
|
62
75
|
end
|
63
76
|
(1..PATTERN_MAX_NUM).each do |i|
|
64
77
|
next unless conf["pattern#{i}"]
|
@@ -88,12 +101,20 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
88
101
|
@removed_length = @removed_prefix_string.length
|
89
102
|
end
|
90
103
|
|
104
|
+
if @store_file
|
105
|
+
f = Pathname.new(@store_file)
|
106
|
+
if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
|
107
|
+
raise Fluent::ConfigError, "#{@store_file} is not writable"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
91
111
|
@counts = count_initialized
|
92
112
|
@mutex = Mutex.new
|
93
113
|
end
|
94
114
|
|
95
115
|
def start
|
96
116
|
super
|
117
|
+
load_status(@store_file, @count_interval) if @store_file
|
97
118
|
start_watch
|
98
119
|
end
|
99
120
|
|
@@ -101,6 +122,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
101
122
|
super
|
102
123
|
@watcher.terminate
|
103
124
|
@watcher.join
|
125
|
+
save_status(@store_file) if @store_file
|
104
126
|
end
|
105
127
|
|
106
128
|
def count_initialized(keys=nil)
|
@@ -217,7 +239,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
217
239
|
end
|
218
240
|
|
219
241
|
def watch
|
220
|
-
@last_checked
|
242
|
+
@last_checked ||= Fluent::Engine.now
|
221
243
|
while true
|
222
244
|
sleep 0.5
|
223
245
|
if Fluent::Engine.now - @last_checked >= @count_interval
|
@@ -249,4 +271,60 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
249
271
|
|
250
272
|
chain.next
|
251
273
|
end
|
274
|
+
|
275
|
+
# Store internal status into a file
|
276
|
+
#
|
277
|
+
# @param [String] file_path
|
278
|
+
def save_status(file_path)
|
279
|
+
begin
|
280
|
+
Pathname.new(file_path).open('wb') do |f|
|
281
|
+
@saved_at = Fluent::Engine.now
|
282
|
+
@saved_duration = @saved_at - @last_checked
|
283
|
+
Marshal.dump({
|
284
|
+
:counts => @counts,
|
285
|
+
:saved_at => @saved_at,
|
286
|
+
:saved_duration => @saved_duration,
|
287
|
+
:aggregate => @aggregate,
|
288
|
+
:count_key => @count_key,
|
289
|
+
:patterns => @patterns,
|
290
|
+
}, f)
|
291
|
+
end
|
292
|
+
rescue => e
|
293
|
+
log.warn "out_datacounter: Can't write store_file #{e.class} #{e.message}"
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
# Load internal status from a file
|
298
|
+
#
|
299
|
+
# @param [String] file_path
|
300
|
+
# @param [Interger] count_interval
|
301
|
+
def load_status(file_path, count_interval)
|
302
|
+
return unless (f = Pathname.new(file_path)).exist?
|
303
|
+
|
304
|
+
begin
|
305
|
+
f.open('rb') do |f|
|
306
|
+
stored = Marshal.load(f)
|
307
|
+
if stored[:aggregate] == @aggregate and
|
308
|
+
stored[:count_key] == @count_key and
|
309
|
+
stored[:patterns] == @patterns
|
310
|
+
|
311
|
+
if Fluent::Engine.now <= stored[:saved_at] + count_interval
|
312
|
+
@counts = stored[:counts]
|
313
|
+
@saved_at = stored[:saved_at]
|
314
|
+
@saved_duration = stored[:saved_duration]
|
315
|
+
|
316
|
+
# skip the saved duration to continue counting
|
317
|
+
@last_checked = Fluent::Engine.now - @saved_duration
|
318
|
+
else
|
319
|
+
log.warn "out_datacounter: stored data is outdated. ignore stored data"
|
320
|
+
end
|
321
|
+
else
|
322
|
+
log.warn "out_datacounter: configuration param was changed. ignore stored data"
|
323
|
+
end
|
324
|
+
end
|
325
|
+
rescue => e
|
326
|
+
log.warn "out_datacounter: Can't load store_file #{e.class} #{e.message}"
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
252
330
|
end
|
data/test/helper.rb
CHANGED
@@ -496,4 +496,62 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
496
496
|
d.instance.flush_emit(60)
|
497
497
|
assert_equal 2, d.emits.size # +0
|
498
498
|
end
|
499
|
+
|
500
|
+
def test_store_file
|
501
|
+
dir = "test/tmp"
|
502
|
+
Dir.mkdir dir unless Dir.exist? dir
|
503
|
+
file = "#{dir}/test.dat"
|
504
|
+
File.unlink file if File.exist? file
|
505
|
+
|
506
|
+
# test store
|
507
|
+
d = create_driver(CONFIG + %[store_file #{file}])
|
508
|
+
d.run do
|
509
|
+
d.instance.flush_emit(60)
|
510
|
+
d.emit({'target' => 1})
|
511
|
+
d.emit({'target' => 1})
|
512
|
+
d.emit({'target' => 1})
|
513
|
+
d.instance.shutdown
|
514
|
+
end
|
515
|
+
stored_counts = d.instance.counts
|
516
|
+
stored_saved_at = d.instance.saved_at
|
517
|
+
stored_saved_duration = d.instance.saved_duration
|
518
|
+
assert File.exist? file
|
519
|
+
|
520
|
+
# test load
|
521
|
+
d = create_driver(CONFIG + %[store_file #{file}])
|
522
|
+
d.run do
|
523
|
+
loaded_counts = d.instance.counts
|
524
|
+
loaded_saved_at = d.instance.saved_at
|
525
|
+
loaded_saved_duration = d.instance.saved_duration
|
526
|
+
assert_equal stored_counts, loaded_counts
|
527
|
+
assert_equal stored_saved_at, loaded_saved_at
|
528
|
+
assert_equal stored_saved_duration, loaded_saved_duration
|
529
|
+
end
|
530
|
+
|
531
|
+
# test not to load if config is changed
|
532
|
+
d = create_driver(CONFIG + %[count_key foobar store_file #{file}])
|
533
|
+
d.run do
|
534
|
+
loaded_counts = d.instance.counts
|
535
|
+
loaded_saved_at = d.instance.saved_at
|
536
|
+
loaded_saved_duration = d.instance.saved_duration
|
537
|
+
assert_equal({}, loaded_counts)
|
538
|
+
assert_equal(nil, loaded_saved_at)
|
539
|
+
assert_equal(nil, loaded_saved_duration)
|
540
|
+
end
|
541
|
+
|
542
|
+
# test not to load if stored data is outdated.
|
543
|
+
Delorean.jump 61 # jump more than count_interval
|
544
|
+
d = create_driver(CONFIG + %[store_file #{file}])
|
545
|
+
d.run do
|
546
|
+
loaded_counts = d.instance.counts
|
547
|
+
loaded_saved_at = d.instance.saved_at
|
548
|
+
loaded_saved_duration = d.instance.saved_duration
|
549
|
+
assert_equal({}, loaded_counts)
|
550
|
+
assert_equal(nil, loaded_saved_at)
|
551
|
+
assert_equal(nil, loaded_saved_duration)
|
552
|
+
end
|
553
|
+
Delorean.back_to_the_present
|
554
|
+
|
555
|
+
File.unlink file
|
556
|
+
end
|
499
557
|
end
|
metadata
CHANGED
@@ -1,46 +1,55 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-numeric-counter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
5
|
-
prerelease:
|
4
|
+
version: 0.2.2
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- TAGOMORI Satoshi
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2014-03-07 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: rake
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ">="
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :development
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: delorean
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
28
39
|
- !ruby/object:Gem::Version
|
29
40
|
version: '0'
|
30
41
|
- !ruby/object:Gem::Dependency
|
31
42
|
name: fluentd
|
32
43
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
44
|
requirements:
|
35
|
-
- -
|
45
|
+
- - ">="
|
36
46
|
- !ruby/object:Gem::Version
|
37
47
|
version: '0'
|
38
48
|
type: :runtime
|
39
49
|
prerelease: false
|
40
50
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
51
|
requirements:
|
43
|
-
- -
|
52
|
+
- - ">="
|
44
53
|
- !ruby/object:Gem::Version
|
45
54
|
version: '0'
|
46
55
|
description: Counts messages, with specified key and numeric value in specified range
|
@@ -50,7 +59,8 @@ executables: []
|
|
50
59
|
extensions: []
|
51
60
|
extra_rdoc_files: []
|
52
61
|
files:
|
53
|
-
- .gitignore
|
62
|
+
- ".gitignore"
|
63
|
+
- ".travis.yml"
|
54
64
|
- Gemfile
|
55
65
|
- LICENSE
|
56
66
|
- README.md
|
@@ -60,28 +70,28 @@ files:
|
|
60
70
|
- test/helper.rb
|
61
71
|
- test/plugin/test_out_numeric_counter.rb
|
62
72
|
homepage: https://github.com/tagomoris/fluent-plugin-numeric-counter
|
63
|
-
licenses:
|
73
|
+
licenses:
|
74
|
+
- APLv2
|
75
|
+
metadata: {}
|
64
76
|
post_install_message:
|
65
77
|
rdoc_options: []
|
66
78
|
require_paths:
|
67
79
|
- lib
|
68
80
|
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
-
none: false
|
70
81
|
requirements:
|
71
|
-
- -
|
82
|
+
- - ">="
|
72
83
|
- !ruby/object:Gem::Version
|
73
84
|
version: '0'
|
74
85
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
-
none: false
|
76
86
|
requirements:
|
77
|
-
- -
|
87
|
+
- - ">="
|
78
88
|
- !ruby/object:Gem::Version
|
79
89
|
version: '0'
|
80
90
|
requirements: []
|
81
91
|
rubyforge_project:
|
82
|
-
rubygems_version:
|
92
|
+
rubygems_version: 2.2.2
|
83
93
|
signing_key:
|
84
|
-
specification_version:
|
94
|
+
specification_version: 4
|
85
95
|
summary: Fluentd plugin to count messages with specified numeric values
|
86
96
|
test_files:
|
87
97
|
- test/helper.rb
|