fluent-plugin-anomalydetect 0.0.1 → 0.1.0

data/README.rdoc

@@ -65,6 +65,13 @@ If you want to watch a value for a target field <fieldname> in data, write below
 
 If you want to know detail of these parameters, see "Theory".
     
+    <match access.**>
+      type anomalydetect
+      ...
+      store_file /path/to/anomalydetect.dat
+    </match>
+
+If "store_file" option was specified, a historical stat will be stored to the file at shutdown, and it will be restored on started.
 
 == Theory
 "データマイニングによる異常検知" http://amzn.to/XHXNun

data/fluent-plugin-anormalydetect.gemspec

@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-anomalydetect"
-  gem.version       = "0.0.1"
+  gem.version       = "0.1.0"
   gem.authors       = ["Muddy Dixon"]
   gem.email         = ["muddydixon@gmail.com"]
   gem.description   = %q{detect anomal sequential input casually}

data/lib/fluent/plugin/out_anomalydetect.rb

@@ -3,6 +3,7 @@ module Fluent
     Fluent::Plugin.register_output('anomalydetect', self)
     
     require 'fluent/plugin/change_finder'
+    require 'pathname'
 
     config_param :outlier_term, :integer, :default => 28
     config_param :outlier_discount, :float, :default => 0.05
@@ -12,12 +13,14 @@ module Fluent
     config_param :tick, :integer, :default => 60 * 5
     config_param :tag, :string, :default => "anomaly"
     config_param :target, :string, :default => nil
+    config_param :store_file, :string, :default => nil
+    config_param :threshold, :float, :default => -1.0
 
     attr_accessor :outlier
     attr_accessor :score
     attr_accessor :record_count
 
-    attr_accessor :outliers
+    attr_accessor :outlier_buf
 
     attr_accessor :records
 
@@ -41,8 +44,13 @@ module Fluent
       if @tick < 1
         raise Fluent::ConfigError, "tick timer should be greater than 1 sec"
       end
-      
-      @outliers = []
+      if @store_file
+        f = Pathname.new(@store_file)
+        if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
+          raise Fluent::ConfigError, "#{@store_file} is not writable"
+        end
+      end
+      @outlier_buf = []
       @outlier  = ChangeFinder.new(@outlier_term, @outlier_discount)
       @score    = ChangeFinder.new(@score_term, @score_discount)
 
@@ -53,6 +61,7 @@ module Fluent
 
     def start
       super
+      load_from_file
       init_records
       start_watch
     end
@@ -63,6 +72,53 @@ module Fluent
         @watcher.terminate
         @watcher.join
       end
+      store_to_file
+    end
+
+    def load_from_file
+      return unless @store_file
+      f = Pathname.new(@store_file)
+      return unless f.exist?
+
+      begin
+        f.open('rb') do |f|
+          stored = Marshal.load(f)
+          if (( stored[:outlier_term]     == @outlier_term ) &&
+              ( stored[:outlier_discount] == @outlier_discount ) &&
+              ( stored[:score_term]       == @score_term ) &&
+              ( stored[:score_discount]   == @score_discount ) &&
+              ( stored[:smooth_term]      == @smooth_term ))
+          then
+            @outlier  = stored[:outlier]
+            @outlier_buf = stored[:outlier_buf]
+            @score    = stored[:score]
+          else
+            $log.warn "configuration param was changed. ignore stored data"
+          end
+        end
+      rescue => e
+        $log.warn "Can't load store_file #{e}"
+      end
+    end
+
+    def store_to_file
+      return unless @store_file
+      begin
+        Pathname.new(@store_file).open('wb') do |f|
+          Marshal.dump({
+            :outlier          => @outlier,
+            :outlier_buf         => @outlier_buf,
+            :score            => @score,
+            :outlier_term     => @outlier_term,
+            :outlier_discount => @outlier_discount,
+            :score_term       => @score_term,
+            :score_discount   => @score_discount,
+            :smooth_term      => @smooth_term,
+          }, f)
+        end
+      rescue => e
+        $log.warn "Can't write store_file #{e}"
+      end
     end
 
     def start_watch
@@ -88,7 +144,9 @@ module Fluent
 
     def flush_emit(step)
       output = flush
-      Fluent::Engine.emit(@tag, Fluent::Engine.now, output)
+      if output
+        Fluent::Engine.emit(@tag, Fluent::Engine.now, output)
+      end
     end
 
     def flush
@@ -101,12 +159,15 @@ module Fluent
             end
 
       outlier = @outlier.next(val)
-      @outliers.push outlier
-      @outliers.shift if @outliers.size > @smooth_term
-      score = @score.next(@outliers.inject(0) { |sum, v| sum += v } / @outliers.size)
-
-      {"outlier" => outlier, "score" => score, "target" => val}
-
+      @outlier_buf.push outlier
+      @outlier_buf.shift if @outlier_buf.size > @smooth_term
+      score = @score.next(@outlier_buf.inject(0) { |sum, v| sum += v } / @outlier_buf.size)
+
+      if @threshold < 0 or (@threshold >= 0 and score > @threshold)
+        {"outlier" => outlier, "score" => score, "target" => val}
+      else
+        nil
+      end
     end
 
     def tick_time(time)

data/test/plugin/test_out_anomalydetect.rb

@@ -87,7 +87,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
 
   def test_array_init
     d = create_driver
-    assert_equal [], d.instance.outliers
+    assert_equal [], d.instance.outlier_buf
     assert_nil d.instance.records  # @records is initialized at start, not configure
   end
 
@@ -137,4 +137,71 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  def test_store_file
+    dir = "test/tmp"
+    Dir.mkdir dir unless Dir.exist? dir
+    file = "#{dir}/test.dat"
+    File.unlink file if File.exist? file
+
+    d = create_driver %[
+      store_file #{file}
+    ]
+
+    d.run do
+      assert_equal [], d.instance.outlier_buf
+      d.emit({'x' => 1})
+      d.emit({'x' => 1})
+      d.emit({'x' => 1})
+      d.instance.flush
+      d.emit({'x' => 1})
+      d.emit({'x' => 1})
+      d.emit({'x' => 1})
+      d.instance.flush
+    end
+    assert File.exist? file
+
+    d2 = create_driver %[
+      store_file #{file}
+    ]
+    d2.run do
+      assert_equal 2, d2.instance.outlier_buf.size
+    end
+
+    File.unlink file
+  end
+
+  def test_set_large_threshold
+    require 'csv'
+    reader = CSV.open("test/stock.2432.csv", "r")
+    header = reader.take(1)[0]
+    d = create_driver %[
+      threshold 1000
+    ]
+    d.run do 
+      reader.each_with_index do |row, idx|
+        break if idx > 5
+        d.emit({'y' => row[4].to_i})
+        r = d.instance.flush
+        assert_equal nil, r
+      end
+    end
+  end
+
+  def test_set_small_threshold
+    require 'csv'
+    reader = CSV.open("test/stock.2432.csv", "r")
+    header = reader.take(1)[0]
+    d = create_driver %[
+      threshold 1
+    ]
+    d.run do 
+      reader.each_with_index do |row, idx|
+        break if idx > 5
+        d.emit({'y' => row[4].to_i})
+        r = d.instance.flush
+        assert_not_equal nil, r
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anomalydetect
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-13 00:00:00.000000000 Z
+date: 2013-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd