fluent-plugin-anomalydetect 0.1.0 → 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 00950f80d0252742704254c65f1f20a145fe967d
+  data.tar.gz: 7c9fd9003e576cbb8a51006ef7c5fad2a8b8d626
+SHA512:
+  metadata.gz: e0ca5a907d709856442ba77c26315bcb3d8777484e1920389f0cd1c20889bbd8625a4ad9087dd6ddaf8b03cf38815f64f84a368d2577d9a7c3e02451e4c16da8
+  data.tar.gz: 89b8175c1797a2800e92710fd9f84b0177239e8ab40dd27b7e9624a3786bb81bc813b98b0011d33503f3f09fac5c1301ed9f17fb401194417bfc922cb86668fc

data/README.rdoc

@@ -73,6 +73,15 @@ If you want to know detail of these parameters, see "Theory".
 
 If "store_file" option was specified, a historical stat will be stored to the file at shutdown, and it will be restored on started.
 
+
+    <match access.**>
+      type anomalydetect
+      ...
+      threshold 3
+    </match>
+
+If "threshold" option was specified, plugin only ouput when the anomalyscore is more than threshold.
+
 == Theory
 "データマイニングによる異常検知" http://amzn.to/XHXNun
 

data/fluent-plugin-anormalydetect.gemspec

@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-anomalydetect"
-  gem.version       = "0.1.0"
+  gem.version       = "0.1.1"
   gem.authors       = ["Muddy Dixon"]
   gem.email         = ["muddydixon@gmail.com"]
   gem.description   = %q{detect anomal sequential input casually}
@@ -16,6 +16,6 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
   
-  gem.add_development_dependency "fluentd"
+  gem.add_development_dependency "rake"
   gem.add_runtime_dependency "fluentd"
 end

data/lib/fluent/plugin/change_finder.rb

@@ -43,7 +43,10 @@ module Fluent
         @data.shift
       end
 
-      score(prob xt, @sigma, x)
+      p = prob(xt, @sigma, x)
+      s = score(p)
+      $log.debug "change_finder:#{Thread.current.object_id} x:#{x} xt:#{xt} p:#{p} s:#{s} term:#{@term} r:#{@r} data:#{@data} mu:#{@mu} sigma:#{@sigma} c:#{@c}"
+      s
     end
 
     def prob(mu, sigma, v)
@@ -60,7 +63,7 @@ module Fluent
     def smooth(size)
       _end = @data.size
       _begin = [_end - size, 0].max
-      @data.slice(_begin, _end).inject(0.0) { |sum, v| sum += v } / (_end - _begin)
+      (_size = (_end - _begin)) == 0 ? 0.0 : @data.slice(_begin, _end).inject(:+).to_f / _size
     end
 
     def show_status

data/lib/fluent/plugin/out_anomalydetect.rb

@@ -2,7 +2,7 @@ module Fluent
   class AnomalyDetectOutput < Output
     Fluent::Plugin.register_output('anomalydetect', self)
     
-    require 'fluent/plugin/change_finder'
+    require_relative 'change_finder'
     require 'pathname'
 
     config_param :outlier_term, :integer, :default => 28
@@ -64,6 +64,8 @@ module Fluent
       load_from_file
       init_records
       start_watch
+    rescue => e
+      $log.warn "anomalydetect: #{e.class} #{e.message} #{e.backtrace.first}"
     end
 
     def shutdown
@@ -73,6 +75,8 @@ module Fluent
         @watcher.join
       end
       store_to_file
+    rescue => e
+      $log.warn "anomalydetect: #{e.class} #{e.message} #{e.backtrace.first}"
     end
 
     def load_from_file
@@ -93,11 +97,11 @@ module Fluent
             @outlier_buf = stored[:outlier_buf]
             @score    = stored[:score]
           else
-            $log.warn "configuration param was changed. ignore stored data"
+            $log.warn "anomalydetect: configuration param was changed. ignore stored data"
           end
         end
       rescue => e
-        $log.warn "Can't load store_file #{e}"
+        $log.warn "anomalydetect: Can't load store_file #{e}"
       end
     end
 
@@ -117,7 +121,7 @@ module Fluent
           }, f)
         end
       rescue => e
-        $log.warn "Can't write store_file #{e}"
+        $log.warn "anomalydetect: Can't write store_file #{e}"
       end
     end
 
@@ -126,16 +130,19 @@ module Fluent
     end
 
     def watch
-
       @last_checked = Fluent::Engine.now
-      loop {
-        sleep 0.5
-        now = Fluent::Engine.now
-        if now - @last_checked >= @tick
-          flush_emit(now - @last_checked)
-          @last_checked = now
+      loop do
+        begin
+          sleep 0.5
+          now = Fluent::Engine.now
+          if now - @last_checked >= @tick
+            flush_emit(now - @last_checked)
+            @last_checked = now
+          end
+        rescue => e
+          $log.warn "anomalydetect: #{e.class} #{e.message} #{e.backtrace.first}"
         end
-      }
+      end
     end
 
     def init_records
@@ -155,14 +162,20 @@ module Fluent
       val = if @record_count
               flushed.size
             else
-              flushed.inject(0.0) { |sum, record| sum += record[@target].to_f if record[@target] } / flushed.size
+              filtered = flushed.map {|record| record[@target] }.compact
+              return nil if filtered.empty?
+              filtered.inject(:+).to_f / filtered.size
             end
 
       outlier = @outlier.next(val)
+
       @outlier_buf.push outlier
       @outlier_buf.shift if @outlier_buf.size > @smooth_term
-      score = @score.next(@outlier_buf.inject(0) { |sum, v| sum += v } / @outlier_buf.size)
+      outlier_avg = @outlier_buf.empty? ? 0.0 : @outlier_buf.inject(:+).to_f / @outlier_buf.size
+
+      score = @score.next(outlier_avg)
 
+      $log.debug "out_anomalydetect:#{Thread.current.object_id} flushed:#{flushed} val:#{val} outlier:#{outlier} outlier_buf:#{@outlier_buf} score:#{score}"
       if @threshold < 0 or (@threshold >= 0 and score > @threshold)
         {"outlier" => outlier, "score" => score, "target" => val}
       else
@@ -185,6 +198,8 @@ module Fluent
       push_records records
 
       chain.next
+    rescue => e
+      $log.warn "anomalydetect: #{e.class} #{e.message} #{e.backtrace.first}"
     end
   end
 end

data/test/plugin/test_out_anomalydetect.rb

@@ -108,8 +108,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
       smooth_term 3
     ]
 
-    data = (0..10).map do || (rand * 100).to_i end
-    data.push 0
+    data = 10.times.map { (rand * 100).to_i } + [0]
     d.run do 
       data.each do |val|
         (0..val - 1).each do ||
@@ -121,6 +120,49 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_emit_target
+    d = create_driver %[
+      tag test.anomaly
+      outlier_term 28
+      outlier_discount 0.05
+      score_term 28
+      score_discount 0.05
+      tick 10
+      smooth_term 3
+      target y
+    ]
+
+    data = 10.times.map { (rand * 100).to_i } + [0]
+    d.run do
+      data.each do |val|
+        d.emit({'y' => val})
+        r = d.instance.flush
+        assert_equal val, r['target']
+      end
+    end
+  end
+
+  def test_emit_when_target_does_not_exist
+    d = create_driver %[
+      tag test.anomaly
+      outlier_term 28
+      outlier_discount 0.05
+      score_term 28
+      score_discount 0.05
+      tick 10
+      smooth_term 3
+      target y
+    ]
+
+    d.run do
+      10.times do
+        d.emit({'foobar' => 999.99})
+        r = d.instance.flush
+        assert_equal nil, r
+      end
+    end
+  end
+
   def test_emit_stock_data
     require 'csv'
     reader = CSV.open("test/stock.2432.csv", "r")

metadata

@@ -1,46 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anomalydetect
 version: !ruby/object:Gem::Version
-  version: 0.1.0
-  prerelease: 
+  version: 0.1.1
 platform: ruby
 authors:
 - Muddy Dixon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-21 00:00:00.000000000 Z
+date: 2013-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: detect anomal sequential input casually
@@ -62,30 +57,28 @@ files:
 - test/stock.2432.csv
 homepage: https://github.com/muddydixon/fluent-plugin-anomalydetect
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: detect anomal sequential input casually
 test_files:
 - test/helper.rb
 - test/plugin/test_out_anomalydetect.rb
 - test/stock.2432.csv
-has_rdoc: