fluent-plugin-anomalydetect 0.0 → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.rdoc CHANGED
@@ -1,9 +1,9 @@
1
- # Fluent::Plugin::Anomalydetect
1
+ = Fluent::Plugin::Anomalydetect
2
2
 
3
3
  To detect anomaly for log stream, use this plugin.
4
4
  Then you can find changes in logs casually.
5
5
 
6
- ## Installation
6
+ = Installation
7
7
 
8
8
  Add this line to your application's Gemfile:
9
9
 
@@ -17,31 +17,67 @@ Or install it yourself as:
17
17
 
18
18
  $ gem install fluent-plugin-anomalydetect
19
19
 
20
- ## Usage
20
+ == Usage
21
21
 
22
- ```
23
- <source>
24
- type file
25
- ...
26
- tag access.log
27
- </source>
22
+ <source>
23
+ type file
24
+ ...
25
+ tag access.log
26
+ </source>
28
27
 
29
- <match access.**>
30
- type anomalydetect
31
- tag anomaly.access
32
- tick 86400
33
- </match>
28
+ <match access.**>
29
+ type anomalydetect
30
+ tag anomaly.access
31
+ tick 86400
32
+ </match>
34
33
 
35
- <match anomaly.access>
36
- type file
37
- ...
38
- </match>
39
- ```
34
+ <match anomaly.access>
35
+ type file
36
+ ...
37
+ </match>
40
38
 
41
39
  Then the plugin output anomaly log counts in each day.
42
40
 
43
- ## Theory
41
+ This plugin watches a value of input record number in the interval set with `tick`.
42
+
43
+ If you want to watch a value for a target field <fieldname> in data, write below:
44
+
45
+ <match access.**>
46
+ type anomalydetect
47
+ tag anomaly.access
48
+ tick 86400
49
+ target fieldname
50
+ </match>
51
+
52
+ == more configuration
53
+
54
+ <match access.**>
55
+ type anomalydetect
56
+ tag anomaly.access
57
+ tick 86400
58
+ target fieldname
59
+ outlier_term 7
60
+ outlier_discount 0.5
61
+ smooth_term 7
62
+ score_term 28
63
+ score_discount 0.01
64
+ </match>
65
+
66
+ If you want to know detail of these parameters, see "Theory".
67
+
68
+
69
+ == Theory
44
70
  "データマイニングによる異常検知" http://amzn.to/XHXNun
45
71
 
72
+ = TODO
73
+
74
+ == threshold
75
+
76
+ fluentd outputs value when the outlier value over threshold
77
+
78
+ == FFT algorithms
46
79
 
80
+ = Copyright
47
81
 
82
+ Copyright:: Copyright (c) 2013- Muddy Dixon
83
+ License:: Apache License, Version 2.0
@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-anomalydetect"
6
- gem.version = 0.0
6
+ gem.version = "0.0.1"
7
7
  gem.authors = ["Muddy Dixon"]
8
8
  gem.email = ["muddydixon@gmail.com"]
9
9
  gem.description = %q{detect anomal sequential input casually}
@@ -9,8 +9,9 @@ module Fluent
9
9
  @data = []
10
10
  @mu = 0
11
11
  @sigma = 0
12
- @c = (0..@term - 1).map do |i| rand end
12
+ @c = (0..@term - 1).map { |i| rand }
13
13
  end
14
+
14
15
  def next(x)
15
16
  len = @data.size
16
17
 
@@ -19,7 +20,7 @@ module Fluent
19
20
 
20
21
  # update @c
21
22
  c = @sigma
22
- for j in 0..@term - 1
23
+ for j in 0..(@term - 1)
23
24
  if @data[len - 1 - j]
24
25
  @c[j] = (1 - @r) * @c[j] + @r * (x - @mu) * (@data[len - 1 - j] - @mu)
25
26
  end
@@ -27,7 +28,7 @@ module Fluent
27
28
 
28
29
  cc = Matrix.zero(@term).to_a
29
30
  for j in 0..(@term - 1)
30
- for i in j..@term - 1
31
+ for i in j..(@term - 1)
31
32
  cc[j][i] = cc[i][j] = @c[i - j]
32
33
  end
33
34
  end
@@ -45,13 +46,13 @@ module Fluent
45
46
  score(prob xt, @sigma, x)
46
47
  end
47
48
 
48
- def prob (mu, sigma, v)
49
- return 0 if sigma == 0
50
-
49
+ def prob(mu, sigma, v)
50
+ return 0 if sigma.zero?
51
+
51
52
  Math.exp( - 0.5 * (v - mu) ** 2 / sigma) / ((2 * Math::PI) ** 0.5 * sigma ** 0.5)
52
53
  end
53
54
 
54
- def score (p)
55
+ def score(p)
55
56
  return 0 if p <= 0
56
57
  -Math.log(p)
57
58
  end
@@ -59,10 +60,10 @@ module Fluent
59
60
  def smooth(size)
60
61
  _end = @data.size
61
62
  _begin = [_end - size, 0].max
62
- @data.slice(_begin, _end).inject(0.0) do |sum, v| sum += v end / (_end - _begin)
63
+ @data.slice(_begin, _end).inject(0.0) { |sum, v| sum += v } / (_end - _begin)
63
64
  end
64
65
 
65
- def showStatus
66
+ def show_status
66
67
  {:sigma => @sigma, :mu => @mu, :data => @data, :c => @c}
67
68
  end
68
69
  end
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  module Fluent
3
2
  class AnomalyDetectOutput < Output
4
3
  Fluent::Plugin.register_output('anomalydetect', self)
@@ -12,11 +11,11 @@ module Fluent
12
11
  config_param :score_discount, :float, :default => 0.1
13
12
  config_param :tick, :integer, :default => 60 * 5
14
13
  config_param :tag, :string, :default => "anomaly"
15
- config_param :target, :string, :default => ""
14
+ config_param :target, :string, :default => nil
16
15
 
17
16
  attr_accessor :outlier
18
17
  attr_accessor :score
19
- attr_accessor :recordCount
18
+ attr_accessor :record_count
20
19
 
21
20
  attr_accessor :outliers
22
21
 
@@ -47,14 +46,9 @@ module Fluent
47
46
  @outlier = ChangeFinder.new(@outlier_term, @outlier_discount)
48
47
  @score = ChangeFinder.new(@score_term, @score_discount)
49
48
 
50
- @records = []
51
49
  @mutex = Mutex.new
52
50
 
53
- if @target == ""
54
- @recordCount = true
55
- else
56
- @recordCount = false
57
- end
51
+ @record_count = @target.nil?
58
52
  end
59
53
 
60
54
  def start
@@ -78,14 +72,14 @@ module Fluent
78
72
  def watch
79
73
 
80
74
  @last_checked = Fluent::Engine.now
81
- while true
75
+ loop {
82
76
  sleep 0.5
83
- if Fluent::Engine.now - @last_checked >= @tick
84
- now = Fluent::Engine.now
77
+ now = Fluent::Engine.now
78
+ if now - @last_checked >= @tick
85
79
  flush_emit(now - @last_checked)
86
80
  @last_checked = now
87
81
  end
88
- end
82
+ }
89
83
  end
90
84
 
91
85
  def init_records
@@ -100,38 +94,34 @@ module Fluent
100
94
  def flush
101
95
  flushed, @records = @records, init_records
102
96
 
103
- val = 0
104
- if @recordCount
105
- val = flushed.size
106
- else
107
- val = flushed.inject(0.0) do |sum, record| sum += record[@target].to_f if record[@target]; end / flushed.size
108
- end
97
+ val = if @record_count
98
+ flushed.size
99
+ else
100
+ flushed.inject(0.0) { |sum, record| sum += record[@target].to_f if record[@target] } / flushed.size
101
+ end
109
102
 
110
103
  outlier = @outlier.next(val)
111
104
  @outliers.push outlier
112
- @outliers.shift() if @outliers.size > @smooth_term
113
- score = @score.next(@outliers.inject(0) do |sum, v| sum += v end / @outliers.size)
105
+ @outliers.shift if @outliers.size > @smooth_term
106
+ score = @score.next(@outliers.inject(0) { |sum, v| sum += v } / @outliers.size)
114
107
 
115
108
  {"outlier" => outlier, "score" => score, "target" => val}
116
109
 
117
110
  end
118
111
 
119
- def tickTime (time)
112
+ def tick_time(time)
120
113
  (time - time % @tick).to_s
121
114
  end
122
115
 
123
- def pushRecords (records)
116
+ def push_records(records)
124
117
  @mutex.synchronize do
125
118
  @records.concat(records)
126
119
  end
127
120
  end
128
121
 
129
- def emit (tag, es, chain)
130
- records = []
131
- es.each do |time, record|
132
- records.push record
133
- end
134
- pushRecords records
122
+ def emit(tag, es, chain)
123
+ records = es.map { |time, record| record }
124
+ push_records records
135
125
 
136
126
  chain.next
137
127
  end
@@ -28,8 +28,9 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
28
28
  assert_equal 14, d.instance.score_term
29
29
  assert_equal 0.1, d.instance.score_discount
30
30
  assert_equal 300, d.instance.tick
31
- assert_equal "", d.instance.target
32
- assert_equal true, d.instance.recordCount
31
+ assert_nil d.instance.target
32
+ assert_equal 'anomaly', d.instance.tag
33
+ assert d.instance.record_count
33
34
 
34
35
  d = create_driver
35
36
  assert_equal 28, d.instance.outlier_term
@@ -40,7 +41,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
40
41
  assert_equal 10, d.instance.tick
41
42
  assert_equal "y", d.instance.target
42
43
  assert_equal 'test.anomaly', d.instance.tag
43
- assert_equal false, d.instance.recordCount
44
+ assert !d.instance.record_count
44
45
 
45
46
  assert_raise(Fluent::ConfigError) {
46
47
  d = create_driver %[
@@ -87,7 +88,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
87
88
  def test_array_init
88
89
  d = create_driver
89
90
  assert_equal [], d.instance.outliers
90
- assert_equal [], d.instance.records
91
+ assert_nil d.instance.records # @records is initialized at start, not configure
91
92
  end
92
93
 
93
94
  def test_sdar
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-anomalydetect
3
3
  version: !ruby/object:Gem::Version
4
- version: '0.0'
4
+ version: 0.0.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-01-11 00:00:00.000000000 Z
12
+ date: 2013-01-13 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
16
- requirement: &2152223980 !ruby/object:Gem::Requirement
16
+ requirement: !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
21
21
  version: '0'
22
22
  type: :development
23
23
  prerelease: false
24
- version_requirements: *2152223980
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
25
30
  - !ruby/object:Gem::Dependency
26
31
  name: fluentd
27
- requirement: &2152223420 !ruby/object:Gem::Requirement
32
+ requirement: !ruby/object:Gem::Requirement
28
33
  none: false
29
34
  requirements:
30
35
  - - ! '>='
@@ -32,7 +37,12 @@ dependencies:
32
37
  version: '0'
33
38
  type: :runtime
34
39
  prerelease: false
35
- version_requirements: *2152223420
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
36
46
  description: detect anomal sequential input casually
37
47
  email:
38
48
  - muddydixon@gmail.com
@@ -42,7 +52,6 @@ extra_rdoc_files: []
42
52
  files:
43
53
  - .gitignore
44
54
  - Gemfile
45
- - LICENSE.txt
46
55
  - README.rdoc
47
56
  - Rakefile
48
57
  - fluent-plugin-anormalydetect.gemspec
@@ -71,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
71
80
  version: '0'
72
81
  requirements: []
73
82
  rubyforge_project:
74
- rubygems_version: 1.8.11
83
+ rubygems_version: 1.8.24
75
84
  signing_key:
76
85
  specification_version: 3
77
86
  summary: detect anomal sequential input casually
@@ -79,3 +88,4 @@ test_files:
79
88
  - test/helper.rb
80
89
  - test/plugin/test_out_anomalydetect.rb
81
90
  - test/stock.2432.csv
91
+ has_rdoc:
data/LICENSE.txt DELETED
@@ -1,22 +0,0 @@
1
- Copyright (c) 2012 muddydixon
2
-
3
- MIT License
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining
6
- a copy of this software and associated documentation files (the
7
- "Software"), to deal in the Software without restriction, including
8
- without limitation the rights to use, copy, modify, merge, publish,
9
- distribute, sublicense, and/or sell copies of the Software, and to
10
- permit persons to whom the Software is furnished to do so, subject to
11
- the following conditions:
12
-
13
- The above copyright notice and this permission notice shall be
14
- included in all copies or substantial portions of the Software.
15
-
16
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.