fluent-plugin-anomalydetect 0.0 → 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +56 -20
- data/fluent-plugin-anormalydetect.gemspec +1 -1
- data/lib/fluent/plugin/change_finder.rb +10 -9
- data/lib/fluent/plugin/out_anomalydetect.rb +19 -29
- data/test/plugin/test_out_anomalydetect.rb +5 -4
- metadata +18 -8
- data/LICENSE.txt +0 -22
data/README.rdoc
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
|
1
|
+
= Fluent::Plugin::Anomalydetect
|
2
2
|
|
3
3
|
To detect anomaly for log stream, use this plugin.
|
4
4
|
Then you can find changes in logs casually.
|
5
5
|
|
6
|
-
|
6
|
+
= Installation
|
7
7
|
|
8
8
|
Add this line to your application's Gemfile:
|
9
9
|
|
@@ -17,31 +17,67 @@ Or install it yourself as:
|
|
17
17
|
|
18
18
|
$ gem install fluent-plugin-anomalydetect
|
19
19
|
|
20
|
-
|
20
|
+
== Usage
|
21
21
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
</source>
|
22
|
+
<source>
|
23
|
+
type file
|
24
|
+
...
|
25
|
+
tag access.log
|
26
|
+
</source>
|
28
27
|
|
29
|
-
<match access.**>
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
</match>
|
28
|
+
<match access.**>
|
29
|
+
type anomalydetect
|
30
|
+
tag anomaly.access
|
31
|
+
tick 86400
|
32
|
+
</match>
|
34
33
|
|
35
|
-
<match anomaly.access>
|
36
|
-
|
37
|
-
|
38
|
-
</match>
|
39
|
-
```
|
34
|
+
<match anomaly.access>
|
35
|
+
type file
|
36
|
+
...
|
37
|
+
</match>
|
40
38
|
|
41
39
|
Then the plugin output anomaly log counts in each day.
|
42
40
|
|
43
|
-
|
41
|
+
This plugin watches a value of input record number in the interval set with `tick`.
|
42
|
+
|
43
|
+
If you want to watch a value for a target field <fieldname> in data, write below:
|
44
|
+
|
45
|
+
<match access.**>
|
46
|
+
type anomalydetect
|
47
|
+
tag anomaly.access
|
48
|
+
tick 86400
|
49
|
+
target fieldname
|
50
|
+
</match>
|
51
|
+
|
52
|
+
== more configuration
|
53
|
+
|
54
|
+
<match access.**>
|
55
|
+
type anomalydetect
|
56
|
+
tag anomaly.access
|
57
|
+
tick 86400
|
58
|
+
target fieldname
|
59
|
+
outlier_term 7
|
60
|
+
outlier_discount 0.5
|
61
|
+
smooth_term 7
|
62
|
+
score_term 28
|
63
|
+
score_discount 0.01
|
64
|
+
</match>
|
65
|
+
|
66
|
+
If you want to know detail of these parameters, see "Theory".
|
67
|
+
|
68
|
+
|
69
|
+
== Theory
|
44
70
|
"データマイニングによる異常検知" http://amzn.to/XHXNun
|
45
71
|
|
72
|
+
= TODO
|
73
|
+
|
74
|
+
== threshold
|
75
|
+
|
76
|
+
fluentd outputs value when the outlier value over threshold
|
77
|
+
|
78
|
+
== FFT algorithms
|
46
79
|
|
80
|
+
= Copyright
|
47
81
|
|
82
|
+
Copyright:: Copyright (c) 2013- Muddy Dixon
|
83
|
+
License:: Apache License, Version 2.0
|
@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-anomalydetect"
|
6
|
-
gem.version = 0.0
|
6
|
+
gem.version = "0.0.1"
|
7
7
|
gem.authors = ["Muddy Dixon"]
|
8
8
|
gem.email = ["muddydixon@gmail.com"]
|
9
9
|
gem.description = %q{detect anomal sequential input casually}
|
@@ -9,8 +9,9 @@ module Fluent
|
|
9
9
|
@data = []
|
10
10
|
@mu = 0
|
11
11
|
@sigma = 0
|
12
|
-
@c = (0..@term - 1).map
|
12
|
+
@c = (0..@term - 1).map { |i| rand }
|
13
13
|
end
|
14
|
+
|
14
15
|
def next(x)
|
15
16
|
len = @data.size
|
16
17
|
|
@@ -19,7 +20,7 @@ module Fluent
|
|
19
20
|
|
20
21
|
# update @c
|
21
22
|
c = @sigma
|
22
|
-
for j in 0
|
23
|
+
for j in 0..(@term - 1)
|
23
24
|
if @data[len - 1 - j]
|
24
25
|
@c[j] = (1 - @r) * @c[j] + @r * (x - @mu) * (@data[len - 1 - j] - @mu)
|
25
26
|
end
|
@@ -27,7 +28,7 @@ module Fluent
|
|
27
28
|
|
28
29
|
cc = Matrix.zero(@term).to_a
|
29
30
|
for j in 0..(@term - 1)
|
30
|
-
for i in j
|
31
|
+
for i in j..(@term - 1)
|
31
32
|
cc[j][i] = cc[i][j] = @c[i - j]
|
32
33
|
end
|
33
34
|
end
|
@@ -45,13 +46,13 @@ module Fluent
|
|
45
46
|
score(prob xt, @sigma, x)
|
46
47
|
end
|
47
48
|
|
48
|
-
def prob
|
49
|
-
return 0 if sigma
|
50
|
-
|
49
|
+
def prob(mu, sigma, v)
|
50
|
+
return 0 if sigma.zero?
|
51
|
+
|
51
52
|
Math.exp( - 0.5 * (v - mu) ** 2 / sigma) / ((2 * Math::PI) ** 0.5 * sigma ** 0.5)
|
52
53
|
end
|
53
54
|
|
54
|
-
def score
|
55
|
+
def score(p)
|
55
56
|
return 0 if p <= 0
|
56
57
|
-Math.log(p)
|
57
58
|
end
|
@@ -59,10 +60,10 @@ module Fluent
|
|
59
60
|
def smooth(size)
|
60
61
|
_end = @data.size
|
61
62
|
_begin = [_end - size, 0].max
|
62
|
-
@data.slice(_begin, _end).inject(0.0)
|
63
|
+
@data.slice(_begin, _end).inject(0.0) { |sum, v| sum += v } / (_end - _begin)
|
63
64
|
end
|
64
65
|
|
65
|
-
def
|
66
|
+
def show_status
|
66
67
|
{:sigma => @sigma, :mu => @mu, :data => @data, :c => @c}
|
67
68
|
end
|
68
69
|
end
|
@@ -1,4 +1,3 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
1
|
module Fluent
|
3
2
|
class AnomalyDetectOutput < Output
|
4
3
|
Fluent::Plugin.register_output('anomalydetect', self)
|
@@ -12,11 +11,11 @@ module Fluent
|
|
12
11
|
config_param :score_discount, :float, :default => 0.1
|
13
12
|
config_param :tick, :integer, :default => 60 * 5
|
14
13
|
config_param :tag, :string, :default => "anomaly"
|
15
|
-
config_param :target, :string, :default =>
|
14
|
+
config_param :target, :string, :default => nil
|
16
15
|
|
17
16
|
attr_accessor :outlier
|
18
17
|
attr_accessor :score
|
19
|
-
attr_accessor :
|
18
|
+
attr_accessor :record_count
|
20
19
|
|
21
20
|
attr_accessor :outliers
|
22
21
|
|
@@ -47,14 +46,9 @@ module Fluent
|
|
47
46
|
@outlier = ChangeFinder.new(@outlier_term, @outlier_discount)
|
48
47
|
@score = ChangeFinder.new(@score_term, @score_discount)
|
49
48
|
|
50
|
-
@records = []
|
51
49
|
@mutex = Mutex.new
|
52
50
|
|
53
|
-
|
54
|
-
@recordCount = true
|
55
|
-
else
|
56
|
-
@recordCount = false
|
57
|
-
end
|
51
|
+
@record_count = @target.nil?
|
58
52
|
end
|
59
53
|
|
60
54
|
def start
|
@@ -78,14 +72,14 @@ module Fluent
|
|
78
72
|
def watch
|
79
73
|
|
80
74
|
@last_checked = Fluent::Engine.now
|
81
|
-
|
75
|
+
loop {
|
82
76
|
sleep 0.5
|
83
|
-
|
84
|
-
|
77
|
+
now = Fluent::Engine.now
|
78
|
+
if now - @last_checked >= @tick
|
85
79
|
flush_emit(now - @last_checked)
|
86
80
|
@last_checked = now
|
87
81
|
end
|
88
|
-
|
82
|
+
}
|
89
83
|
end
|
90
84
|
|
91
85
|
def init_records
|
@@ -100,38 +94,34 @@ module Fluent
|
|
100
94
|
def flush
|
101
95
|
flushed, @records = @records, init_records
|
102
96
|
|
103
|
-
val =
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
end
|
97
|
+
val = if @record_count
|
98
|
+
flushed.size
|
99
|
+
else
|
100
|
+
flushed.inject(0.0) { |sum, record| sum += record[@target].to_f if record[@target] } / flushed.size
|
101
|
+
end
|
109
102
|
|
110
103
|
outlier = @outlier.next(val)
|
111
104
|
@outliers.push outlier
|
112
|
-
@outliers.shift
|
113
|
-
score = @score.next(@outliers.inject(0)
|
105
|
+
@outliers.shift if @outliers.size > @smooth_term
|
106
|
+
score = @score.next(@outliers.inject(0) { |sum, v| sum += v } / @outliers.size)
|
114
107
|
|
115
108
|
{"outlier" => outlier, "score" => score, "target" => val}
|
116
109
|
|
117
110
|
end
|
118
111
|
|
119
|
-
def
|
112
|
+
def tick_time(time)
|
120
113
|
(time - time % @tick).to_s
|
121
114
|
end
|
122
115
|
|
123
|
-
def
|
116
|
+
def push_records(records)
|
124
117
|
@mutex.synchronize do
|
125
118
|
@records.concat(records)
|
126
119
|
end
|
127
120
|
end
|
128
121
|
|
129
|
-
def emit
|
130
|
-
records =
|
131
|
-
|
132
|
-
records.push record
|
133
|
-
end
|
134
|
-
pushRecords records
|
122
|
+
def emit(tag, es, chain)
|
123
|
+
records = es.map { |time, record| record }
|
124
|
+
push_records records
|
135
125
|
|
136
126
|
chain.next
|
137
127
|
end
|
@@ -28,8 +28,9 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
|
|
28
28
|
assert_equal 14, d.instance.score_term
|
29
29
|
assert_equal 0.1, d.instance.score_discount
|
30
30
|
assert_equal 300, d.instance.tick
|
31
|
-
|
32
|
-
assert_equal
|
31
|
+
assert_nil d.instance.target
|
32
|
+
assert_equal 'anomaly', d.instance.tag
|
33
|
+
assert d.instance.record_count
|
33
34
|
|
34
35
|
d = create_driver
|
35
36
|
assert_equal 28, d.instance.outlier_term
|
@@ -40,7 +41,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
|
|
40
41
|
assert_equal 10, d.instance.tick
|
41
42
|
assert_equal "y", d.instance.target
|
42
43
|
assert_equal 'test.anomaly', d.instance.tag
|
43
|
-
|
44
|
+
assert !d.instance.record_count
|
44
45
|
|
45
46
|
assert_raise(Fluent::ConfigError) {
|
46
47
|
d = create_driver %[
|
@@ -87,7 +88,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
|
|
87
88
|
def test_array_init
|
88
89
|
d = create_driver
|
89
90
|
assert_equal [], d.instance.outliers
|
90
|
-
|
91
|
+
assert_nil d.instance.records # @records is initialized at start, not configure
|
91
92
|
end
|
92
93
|
|
93
94
|
def test_sdar
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-anomalydetect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 0.0.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
16
|
-
requirement:
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,15 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements:
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
25
30
|
- !ruby/object:Gem::Dependency
|
26
31
|
name: fluentd
|
27
|
-
requirement:
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
28
33
|
none: false
|
29
34
|
requirements:
|
30
35
|
- - ! '>='
|
@@ -32,7 +37,12 @@ dependencies:
|
|
32
37
|
version: '0'
|
33
38
|
type: :runtime
|
34
39
|
prerelease: false
|
35
|
-
version_requirements:
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
36
46
|
description: detect anomal sequential input casually
|
37
47
|
email:
|
38
48
|
- muddydixon@gmail.com
|
@@ -42,7 +52,6 @@ extra_rdoc_files: []
|
|
42
52
|
files:
|
43
53
|
- .gitignore
|
44
54
|
- Gemfile
|
45
|
-
- LICENSE.txt
|
46
55
|
- README.rdoc
|
47
56
|
- Rakefile
|
48
57
|
- fluent-plugin-anormalydetect.gemspec
|
@@ -71,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
71
80
|
version: '0'
|
72
81
|
requirements: []
|
73
82
|
rubyforge_project:
|
74
|
-
rubygems_version: 1.8.
|
83
|
+
rubygems_version: 1.8.24
|
75
84
|
signing_key:
|
76
85
|
specification_version: 3
|
77
86
|
summary: detect anomal sequential input casually
|
@@ -79,3 +88,4 @@ test_files:
|
|
79
88
|
- test/helper.rb
|
80
89
|
- test/plugin/test_out_anomalydetect.rb
|
81
90
|
- test/stock.2432.csv
|
91
|
+
has_rdoc:
|
data/LICENSE.txt
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
Copyright (c) 2012 muddydixon
|
2
|
-
|
3
|
-
MIT License
|
4
|
-
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
-
a copy of this software and associated documentation files (the
|
7
|
-
"Software"), to deal in the Software without restriction, including
|
8
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
-
permit persons to whom the Software is furnished to do so, subject to
|
11
|
-
the following conditions:
|
12
|
-
|
13
|
-
The above copyright notice and this permission notice shall be
|
14
|
-
included in all copies or substantial portions of the Software.
|
15
|
-
|
16
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|