fluent-plugin-anomalydetect 0.0 → 0.0.1

data/README.rdoc

@@ -1,9 +1,9 @@
-# Fluent::Plugin::Anomalydetect
+= Fluent::Plugin::Anomalydetect
 
 To detect anomaly for log stream, use this plugin.
 Then you can find changes in logs casually.
 
-## Installation
+= Installation
 
 Add this line to your application's Gemfile:
 
@@ -17,31 +17,67 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-anomalydetect
 
-## Usage
+== Usage
 
-```
-<source>
-  type file
-  ...
-  tag access.log
-</source>
+    <source>
+      type file
+      ...
+      tag access.log
+    </source>
 
-<match access.**>
-  type anomalydetect
-  tag anomaly.access
-  tick 86400
-</match>
+    <match access.**>
+      type anomalydetect
+      tag anomaly.access
+      tick 86400
+    </match>
 
-<match anomaly.access>
-  type file
-  ...
-</match>
-```
+    <match anomaly.access>
+      type file
+      ...
+    </match>
 
 Then the plugin output anomaly log counts in each day.
 
-## Theory
+This plugin watches a value of input record number in the interval set with `tick`.
+
+If you want to watch a value for a target field <fieldname> in data, write below:
+
+    <match access.**>
+      type anomalydetect
+      tag anomaly.access
+      tick 86400
+      target fieldname
+    </match>
+
+== more configuration
+
+    <match access.**>
+      type anomalydetect
+      tag anomaly.access
+      tick 86400
+      target fieldname
+      outlier_term 7
+      outlier_discount 0.5
+      smooth_term 7
+      score_term 28
+      score_discount 0.01
+    </match>
+
+If you want to know detail of these parameters, see "Theory".
+    
+
+== Theory
 "データマイニングによる異常検知" http://amzn.to/XHXNun
 
+= TODO
+
+== threshold
+
+fluentd outputs value when the outlier value over threshold
+
+== FFT algorithms
 
+= Copyright
 
+Copyright:: Copyright (c) 2013- Muddy Dixon
+License::   Apache License, Version 2.0

data/fluent-plugin-anormalydetect.gemspec

@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-anomalydetect"
-  gem.version       = 0.0
+  gem.version       = "0.0.1"
   gem.authors       = ["Muddy Dixon"]
   gem.email         = ["muddydixon@gmail.com"]
   gem.description   = %q{detect anomal sequential input casually}

data/lib/fluent/plugin/change_finder.rb

@@ -9,8 +9,9 @@ module Fluent
       @data = []
       @mu = 0
       @sigma = 0
-      @c = (0..@term - 1).map do |i| rand end
+      @c = (0..@term - 1).map { |i| rand }
     end
+
     def next(x)
       len = @data.size
 
@@ -19,7 +20,7 @@ module Fluent
 
       # update @c
       c = @sigma
-      for j in 0..@term - 1
+      for j in 0..(@term - 1)
         if @data[len - 1 - j]
           @c[j] = (1 - @r) * @c[j] + @r * (x - @mu) * (@data[len - 1 - j] - @mu) 
         end
@@ -27,7 +28,7 @@ module Fluent
       
       cc = Matrix.zero(@term).to_a
       for j in 0..(@term - 1)
-        for i in j..@term - 1
+        for i in j..(@term - 1)
           cc[j][i] = cc[i][j] = @c[i - j]
         end
       end
@@ -45,13 +46,13 @@ module Fluent
       score(prob xt, @sigma, x)
     end
 
-    def prob (mu, sigma, v)
-      return 0 if sigma == 0
-        
+    def prob(mu, sigma, v)
+      return 0 if sigma.zero?
+
       Math.exp( - 0.5 * (v - mu) ** 2 / sigma) / ((2 * Math::PI) ** 0.5 * sigma ** 0.5)
     end
 
-    def score (p)
+    def score(p)
       return 0 if p <= 0
       -Math.log(p)
     end
@@ -59,10 +60,10 @@ module Fluent
     def smooth(size)
       _end = @data.size
       _begin = [_end - size, 0].max
-      @data.slice(_begin, _end).inject(0.0) do |sum, v| sum += v end / (_end - _begin)
+      @data.slice(_begin, _end).inject(0.0) { |sum, v| sum += v } / (_end - _begin)
     end
 
-    def showStatus
+    def show_status
       {:sigma => @sigma, :mu => @mu, :data => @data, :c => @c}
     end
   end

data/lib/fluent/plugin/out_anomalydetect.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Fluent
   class AnomalyDetectOutput < Output
     Fluent::Plugin.register_output('anomalydetect', self)
@@ -12,11 +11,11 @@ module Fluent
     config_param :score_discount, :float, :default => 0.1
     config_param :tick, :integer, :default => 60 * 5
     config_param :tag, :string, :default => "anomaly"
-    config_param :target, :string, :default => ""
+    config_param :target, :string, :default => nil
 
     attr_accessor :outlier
     attr_accessor :score
-    attr_accessor :recordCount
+    attr_accessor :record_count
 
     attr_accessor :outliers
 
@@ -47,14 +46,9 @@ module Fluent
       @outlier  = ChangeFinder.new(@outlier_term, @outlier_discount)
       @score    = ChangeFinder.new(@score_term, @score_discount)
 
-      @records = []
       @mutex = Mutex.new
 
-      if @target == ""
-        @recordCount = true 
-      else
-        @recordCount = false
-      end
+      @record_count = @target.nil?
     end
 
     def start
@@ -78,14 +72,14 @@ module Fluent
     def watch
 
       @last_checked = Fluent::Engine.now
-      while true
+      loop {
         sleep 0.5
-        if Fluent::Engine.now - @last_checked >= @tick
-          now = Fluent::Engine.now
+        now = Fluent::Engine.now
+        if now - @last_checked >= @tick
           flush_emit(now - @last_checked)
           @last_checked = now
         end
-      end
+      }
     end
 
     def init_records
@@ -100,38 +94,34 @@ module Fluent
     def flush
       flushed, @records = @records, init_records
 
-      val = 0
-      if @recordCount
-        val = flushed.size
-      else
-        val = flushed.inject(0.0) do |sum, record| sum += record[@target].to_f if record[@target]; end / flushed.size
-      end
+      val = if @record_count
+              flushed.size
+            else
+              flushed.inject(0.0) { |sum, record| sum += record[@target].to_f if record[@target] } / flushed.size
+            end
 
       outlier = @outlier.next(val)
       @outliers.push outlier
-      @outliers.shift() if @outliers.size > @smooth_term
-      score = @score.next(@outliers.inject(0) do |sum, v| sum += v end / @outliers.size)
+      @outliers.shift if @outliers.size > @smooth_term
+      score = @score.next(@outliers.inject(0) { |sum, v| sum += v } / @outliers.size)
 
       {"outlier" => outlier, "score" => score, "target" => val}
 
     end
 
-    def tickTime (time)
+    def tick_time(time)
       (time - time % @tick).to_s
     end
 
-    def pushRecords (records)
+    def push_records(records)
       @mutex.synchronize do
         @records.concat(records)
       end
     end
 
-    def emit (tag, es, chain)
-      records = []
-      es.each do |time, record|
-        records.push record
-      end
-      pushRecords records
+    def emit(tag, es, chain)
+      records = es.map { |time, record| record }
+      push_records records
 
       chain.next
     end

data/test/plugin/test_out_anomalydetect.rb

@@ -28,8 +28,9 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
     assert_equal 14, d.instance.score_term
     assert_equal 0.1, d.instance.score_discount
     assert_equal 300, d.instance.tick
-    assert_equal "", d.instance.target
-    assert_equal true, d.instance.recordCount
+    assert_nil d.instance.target
+    assert_equal 'anomaly', d.instance.tag
+    assert d.instance.record_count
 
     d = create_driver
     assert_equal 28, d.instance.outlier_term
@@ -40,7 +41,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
     assert_equal 10, d.instance.tick
     assert_equal "y", d.instance.target
     assert_equal 'test.anomaly', d.instance.tag
-    assert_equal false, d.instance.recordCount
+    assert !d.instance.record_count
 
     assert_raise(Fluent::ConfigError) {
       d = create_driver %[
@@ -87,7 +88,7 @@ class AnomalyDetectOutputTest < Test::Unit::TestCase
   def test_array_init
     d = create_driver
     assert_equal [], d.instance.outliers
-    assert_equal [], d.instance.records
+    assert_nil d.instance.records  # @records is initialized at start, not configure
   end
 
   def test_sdar

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anomalydetect
 version: !ruby/object:Gem::Version
-  version: '0.0'
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-11 00:00:00.000000000 Z
+date: 2013-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2152223980 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152223980
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2152223420 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +37,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2152223420
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: detect anomal sequential input casually
 email:
 - muddydixon@gmail.com
@@ -42,7 +52,6 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
-- LICENSE.txt
 - README.rdoc
 - Rakefile
 - fluent-plugin-anormalydetect.gemspec
@@ -71,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: detect anomal sequential input casually
@@ -79,3 +88,4 @@ test_files:
 - test/helper.rb
 - test/plugin/test_out_anomalydetect.rb
 - test/stock.2432.csv
+has_rdoc: 

data/LICENSE.txt

@@ -1,22 +0,0 @@
-Copyright (c) 2012 muddydixon
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.