fluent-plugin-anomalydetect 0.0 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +56 -20
- data/fluent-plugin-anormalydetect.gemspec +1 -1
- data/lib/fluent/plugin/change_finder.rb +10 -9
- data/lib/fluent/plugin/out_anomalydetect.rb +19 -29
- data/test/plugin/test_out_anomalydetect.rb +5 -4
- metadata +18 -8
- data/LICENSE.txt +0 -22
data/README.rdoc
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
|
1
|
+
= Fluent::Plugin::Anomalydetect
|
2
2
|
|
3
3
|
To detect anomaly for log stream, use this plugin.
|
4
4
|
Then you can find changes in logs casually.
|
5
5
|
|
6
|
-
|
6
|
+
= Installation
|
7
7
|
|
8
8
|
Add this line to your application's Gemfile:
|
9
9
|
|
@@ -17,31 +17,67 @@ Or install it yourself as:
|
|
17
17
|
|
18
18
|
$ gem install fluent-plugin-anomalydetect
|
19
19
|
|
20
|
-
|
20
|
+
== Usage
|
21
21
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
</source>
|
22
|
+
<source>
|
23
|
+
type file
|
24
|
+
...
|
25
|
+
tag access.log
|
26
|
+
</source>
|
28
27
|
|
29
|
-
<match access.**>
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
</match>
|
28
|
+
<match access.**>
|
29
|
+
type anomalydetect
|
30
|
+
tag anomaly.access
|
31
|
+
tick 86400
|
32
|
+
</match>
|
34
33
|
|
35
|
-
<match anomaly.access>
|
36
|
-
|
37
|
-
|
38
|
-
</match>
|
39
|
-
```
|
34
|
+
<match anomaly.access>
|
35
|
+
type file
|
36
|
+
...
|
37
|
+
</match>
|
40
38
|
|
41
39
|
Then the plugin output anomaly log counts in each day.
|
42
40
|
|
43
|
-
|
41
|
+
This plugin watches a value of input record number in the interval set with `tick`.
|
42
|
+
|
43
|
+
If you want to watch a value for a target field <fieldname> in data, write below:
|
44
|
+
|
45
|
+
<match access.**>
|
46
|
+
type anomalydetect
|
47
|
+
tag anomaly.access
|
48
|
+
tick 86400
|
49
|
+
target fieldname
|
50
|
+
</match>
|
51
|
+
|
52
|
+
== more configuration
|
53
|
+
|
54
|
+
<match access.**>
|
55
|
+
type anomalydetect
|
56
|
+
tag anomaly.access
|
57
|
+
tick 86400
|
58
|
+
target fieldname
|
59
|
+
outlier_term 7
|
60
|
+
outlier_discount 0.5
|
61
|
+
smooth_term 7
|
62
|
+
score_term 28
|
63
|
+
score_discount 0.01
|
64
|
+
</match>
|
65
|
+
|
66
|
+
If you want to know detail of these parameters, see "Theory".
|
67
|
+
|
68
|
+
|
69
|
+
== Theory
|
44
70
|
"データマイニングによる異常検知" http://amzn.to/XHXNun
|
45
71
|
|
72
|
+
= TODO
|
73
|
+
|
74
|
+
== threshold
|
75
|
+
|
76
|
+
fluentd outputs value when the outlier value over threshold
|
77
|
+
|
78
|
+
== FFT algorithms
|
46
79
|
|
80
|
+
= Copyright
|
47
81
|
|
82
|
+
Copyright:: Copyright (c) 2013- Muddy Dixon
|
83
|
+
License:: Apache License, Version 2.0
|
@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-anomalydetect"
|
6
|
-
gem.version = 0.0
|
6
|
+
gem.version = "0.0.1"
|
7
7
|
gem.authors = ["Muddy Dixon"]
|
8
8
|
gem.email = ["muddydixon@gmail.com"]
|
9
9
|
gem.description = %q{detect anomal sequential input casually}
|
@@ -9,8 +9,9 @@ module Fluent
|
|
9
9
|
@data = []
|
10
10
|
@mu = 0
|
11
11
|
@sigma = 0
|
12
|
-
@c = (0..@term - 1).map
|
12
|
+
@c = (0..@term - 1).map { |i| rand }
|
13
13
|
end
|
14
|
+
|
14
15
|
def next(x)
|
15
16
|
len = @data.size
|
16
17
|
|
@@ -19,7 +20,7 @@ module Fluent
|
|
19
20
|
|
20
21
|
# update @c
|
21
22
|
c = @sigma
|
22
|
-
for j in 0
|
23
|
+
for j in 0..(@term - 1)
|
23
24
|
if @data[len - 1 - j]
|
24
25
|
@c[j] = (1 - @r) * @c[j] + @r * (x - @mu) * (@data[len - 1 - j] - @mu)
|
25
26
|
end
|
@@ -27,7 +28,7 @@ module Fluent
|
|
27
28
|
|
28
29
|
cc = Matrix.zero(@term).to_a
|
29
30
|
for j in 0..(@term - 1)
|
30
|
-
for i in j
|
31
|
+
for i in j..(@term - 1)
|
31
32
|
cc[j][i] = cc[i][j] = @c[i - j]
|
32
33
|
end
|
33
34
|
end
|
@@ -45,13 +46,13 @@ module Fluent
|
|
45
46
|
score(prob xt, @sigma, x)
|
46
47
|
end
|
47
48
|
|
48
|
-
def prob
|
49
|
-
return 0 if sigma
|
50
|
-
|
49
|
+
def prob(mu, sigma, v)
|
50
|
+
return 0 if sigma.zero?
|
51
|
+
|
51
52
|
Math.exp( - 0.5 * (v - mu) ** 2 / sigma) / ((2 * Math::PI) ** 0.5 * sigma ** 0.5)
|
52
53
|
end
|
53
54
|
|
54
|
-
def score
|
55
|
+
def score(p)
|
55
56
|
return 0 if p <= 0
|
56
57
|
-Math.log(p)
|
57
58
|
end
|
@@ -59,10 +60,10 @@ module Fluent
|
|
59
60
|
def smooth(size)
|
60
61
|
_end = @data.size
|
61
62
|
_begin = [_end - size, 0].max
|
62
|
-
@data.slice(_begin, _end).inject(0.0)
|
63
|
+
@data.slice(_begin, _end).inject(0.0) { |sum, v| sum += v } / (_end - _begin)
|
63
64
|
end
|
64
65
|
|
65
|
-
def
|
66
|
+
def show_status
|
66
67
|
{:sigma => @sigma, :mu => @mu, :data => @data, :c => @c}
|
67
68
|
end
|
68
69
|
end
|
@@ -1,4 +1,3 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
1
|
module Fluent
|
3
2
|
class AnomalyDetectOutput < Output
|
4
3
|
Fluent::Plugin.register_output('anomalydetect', self)
|
@@ -12,11 +11,11 @@ module Fluent
|
|
12
11
|
config_param :score_discount, :float, :default => 0.1
|
13
12
|
config_param :tick, :integer, :default => 60 * 5
|
14
13
|
config_param :tag, :string, :default => "anomaly"
|
15
|
-
config_param :target, :string, :default =>
|
14
|
+
config_param :target, :string, :default => nil
|
16
15
|
|
17
16
|
attr_accessor :outlier
|
18
17
|
attr_accessor :score
|
19
|
-
attr_accessor :
|
18
|
+
attr_accessor :record_count
|
20
19
|
|
21
20
|
attr_accessor :outliers
|
22
21
|
|
@@ -47,14 +46,9 @@ module Fluent
|
|
47
46
|
@outlier = ChangeFinder.new(@outlier_term, @outlier_discount)
|
48
47
|
@score = ChangeFinder.new(@score_term, @score_discount)
|
49
48
|
|
50
|
-
@records = []
|
51
49
|
@mutex = Mutex.new
|
52
50
|
|
53
|
-
|
54
|
-
@recordCount = true
|
55
|
-
else
|
56
|
-
@recordCount = false
|
57
|
-
end
|
51
|
+
@record_count = @target.nil?
|
58
52
|
end
|
59
53
|
|
60
54
|
def start
|
@@ -78,14 +72,14 @@ module Fluent
|
|
78
72
|
def watch
|
79
73
|
|
80
74
|
@last_checked = Fluent::Engine.now
|
81
|
-
|
75
|
+
loop {
|
82
76
|
sleep 0.5
|
83
|
-
|
84
|
-
|
77
|
+
now = Fluent::Engine.now
|
78
|
+
if now - @last_checked >= @tick
|
85
79
|
flush_emit(now - @last_checked)
|
86
80
|
@last_checked = now
|
87
81
|
end
|
88
|
-
|
82
|
+
}
|
89
83
|
end
|
90
84
|
|
91
85
|
def init_records
|
@@ -100,38 +94,34 @@ module Fluent
|
|
100
94
|
def flush
|
101
95
|
flushed, @records = @records, init_records
|
102
96
|
|
103
|
-
val =
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
end
|
97
|
+
val = if @record_count
|
98
|
+
flushed.size
|
99
|
+
else
|
100
|
+
flushed.inject(0.0) { |sum, record| sum += record[@target].to_f if record[@target] } / flushed.size
|
101
|
+
end
|
109
102
|
|
110
103
|
outlier = @outlier.next(val)
|
111
104
|
@outliers.push outlier
|
112
|
-
@outliers.shift
|
113
|
-
score = @score.next(@outliers.inject(0)
|
105
|
+
@outliers.shift if @outliers.size > @smooth_term
|
106
|
+
score = @score.next(@outliers.inject(0) { |sum, v| sum += v } / @outliers.size)
|
114
107
|
|
115
108
|
{"outlier" => outlier, "score" => score, "target" => val}
|
116
109
|
|
117
110
|
end
|
118
111
|
|
119
|
-
def
|
112
|
+
def tick_time(time)
|
120
113
|
(time - time % @tick).to_s
|
121
114
|
end
|
122
115
|
|
123
|
-
def
|
116
|
+
def push_records(records)
|
124
117
|
@mutex.synchronize do
|
125
118
|
@records.concat(records)
|
126
119
|
end
|
127
120
|
end
|
128
121
|
|
129
|
-
def emit
|
130
|
-
records =
|
131
|
-
|
132
|
-
records.push record
|
133
|
-
end
|
134
|
-
pushRecords records
|
122
|
+
def emit(tag, es, chain)
|
123
|
+
records = es.map { |time, record| record }
|
124
|
+
push_records records
|
135
125
|
|
136
126
|
chain.next
|
137
127
|
end
|
@@ -28,8 +28,9 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
|
|
28
28
|
assert_equal 14, d.instance.score_term
|
29
29
|
assert_equal 0.1, d.instance.score_discount
|
30
30
|
assert_equal 300, d.instance.tick
|
31
|
-
|
32
|
-
assert_equal
|
31
|
+
assert_nil d.instance.target
|
32
|
+
assert_equal 'anomaly', d.instance.tag
|
33
|
+
assert d.instance.record_count
|
33
34
|
|
34
35
|
d = create_driver
|
35
36
|
assert_equal 28, d.instance.outlier_term
|
@@ -40,7 +41,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
|
|
40
41
|
assert_equal 10, d.instance.tick
|
41
42
|
assert_equal "y", d.instance.target
|
42
43
|
assert_equal 'test.anomaly', d.instance.tag
|
43
|
-
|
44
|
+
assert !d.instance.record_count
|
44
45
|
|
45
46
|
assert_raise(Fluent::ConfigError) {
|
46
47
|
d = create_driver %[
|
@@ -87,7 +88,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
|
|
87
88
|
def test_array_init
|
88
89
|
d = create_driver
|
89
90
|
assert_equal [], d.instance.outliers
|
90
|
-
|
91
|
+
assert_nil d.instance.records # @records is initialized at start, not configure
|
91
92
|
end
|
92
93
|
|
93
94
|
def test_sdar
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-anomalydetect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 0.0.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
16
|
-
requirement:
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,15 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements:
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
25
30
|
- !ruby/object:Gem::Dependency
|
26
31
|
name: fluentd
|
27
|
-
requirement:
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
28
33
|
none: false
|
29
34
|
requirements:
|
30
35
|
- - ! '>='
|
@@ -32,7 +37,12 @@ dependencies:
|
|
32
37
|
version: '0'
|
33
38
|
type: :runtime
|
34
39
|
prerelease: false
|
35
|
-
version_requirements:
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
36
46
|
description: detect anomal sequential input casually
|
37
47
|
email:
|
38
48
|
- muddydixon@gmail.com
|
@@ -42,7 +52,6 @@ extra_rdoc_files: []
|
|
42
52
|
files:
|
43
53
|
- .gitignore
|
44
54
|
- Gemfile
|
45
|
-
- LICENSE.txt
|
46
55
|
- README.rdoc
|
47
56
|
- Rakefile
|
48
57
|
- fluent-plugin-anormalydetect.gemspec
|
@@ -71,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
71
80
|
version: '0'
|
72
81
|
requirements: []
|
73
82
|
rubyforge_project:
|
74
|
-
rubygems_version: 1.8.
|
83
|
+
rubygems_version: 1.8.24
|
75
84
|
signing_key:
|
76
85
|
specification_version: 3
|
77
86
|
summary: detect anomal sequential input casually
|
@@ -79,3 +88,4 @@ test_files:
|
|
79
88
|
- test/helper.rb
|
80
89
|
- test/plugin/test_out_anomalydetect.rb
|
81
90
|
- test/stock.2432.csv
|
91
|
+
has_rdoc:
|
data/LICENSE.txt
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
Copyright (c) 2012 muddydixon
|
2
|
-
|
3
|
-
MIT License
|
4
|
-
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
-
a copy of this software and associated documentation files (the
|
7
|
-
"Software"), to deal in the Software without restriction, including
|
8
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
-
permit persons to whom the Software is furnished to do so, subject to
|
11
|
-
the following conditions:
|
12
|
-
|
13
|
-
The above copyright notice and this permission notice shall be
|
14
|
-
included in all copies or substantial portions of the Software.
|
15
|
-
|
16
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|