fluent-plugin-numeric-counter 0.2.1 → 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bf34aed7a1bdec3baef3baca72cccc6ee8e858f3
+  data.tar.gz: e77034978932949f7f41d84ac426d6b8152d5d0e
+SHA512:
+  metadata.gz: 748cbafe875ef82c2069db03b8fb24e8ff9bd008abb75dde27d1853b5379a155156f7ec627a0c2adafecdb73a07b4590c78998fa7f5fd1af8440867e062cc13e
+  data.tar.gz: 468015aeee1dffdad2679eb85b829789569728d84f36a7c179d915cc883f6fc7dbabd4c24b016eba7ef14dd5eb4f55e49bef050a77ce226dcc5cfe024f315c58

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0

data/README.md

@@ -4,7 +4,7 @@
 
 ### NumericCounterOutput
 
-Fluentd plugin to count messages, matches for numeric range patterns, and emits its result (like fluent-plugin-datacounter).
+[Fluentd](http://fluentd.org) plugin to count messages, matches for numeric range patterns, and emits its result (like fluent-plugin-datacounter).
 
 - Counts per min/hour/day
 - Counts per second (average every min/hour/day)
@@ -113,6 +113,53 @@ And you can get tested messages count with 'output\_messages' option:
     # => tag: 'num.foo' or 'num.bar'
     #    message: {'messages' => xxx, 'SMALL_count' => 100, ... }
 
+## Parameters
+
+* count\_key (required)
+
+    The key to count in the event record.
+
+* tag
+
+    The output tag. Default is `numcount`.
+
+* tag\_prefix
+
+    The prefix string which will be added to the input tag. `output_per_tag yes` must be specified together. 
+
+* input\_tag\_remove\_prefix
+
+    The prefix string which will be removed from the input tag.
+
+* count\_interval
+
+    The interval time to count in seconds. Default is `60`.
+
+* unit
+
+    The interval time to monitor specified an unit (either of `minute`, `hour`, or `day`).
+    Use either of `count_interval` or `unit`.
+
+* aggregate
+
+    Calculate in each input `tag` separetely, or `all` records in a mass. Default is `tag`.
+
+* ouput\_per\_tag
+
+    Emit for each input tag. `tag_prefix` must be specified together. Default is `no`.
+
+* outcast\_unmatched
+
+    Specify `yes` if you do not want to include 'unmatched' counts into percentage. Default is `no`.
+
+* output\_messages
+
+    Specify `yes` if you want to get tested messages. Default is `no`.
+
+* store\_file
+
+    Store internal data into a file of the given path on shutdown, and load on starting.
+
 ## TODO
 
 * more tests

data/fluent-plugin-numeric-counter.gemspec

@@ -2,12 +2,13 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-numeric-counter"
-  gem.version       = "0.2.1"
+  gem.version       = "0.2.2"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{Counts messages, with specified key and numeric value in specified range}
   gem.summary       = %q{Fluentd plugin to count messages with specified numeric values}
   gem.homepage      = "https://github.com/tagomoris/fluent-plugin-numeric-counter"
+  gem.license       = "APLv2"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -15,5 +16,6 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "delorean"
   gem.add_runtime_dependency "fluentd"
 end

data/lib/fluent/plugin/out_numeric_counter.rb

@@ -1,6 +1,11 @@
 class Fluent::NumericCounterOutput < Fluent::Output
   Fluent::Plugin.register_output('numeric_counter', self)
 
+  def initialize
+    super
+    require 'pathname'
+  end
+
   PATTERN_MAX_NUM = 20
 
   config_param :count_interval, :time, :default => 60
@@ -13,6 +18,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
   config_param :count_key, :string
   config_param :outcast_unmatched, :bool, :default => false
   config_param :output_messages, :bool, :default => false
+  config_param :store_file, :string, :default => nil
 
   # pattern0 reserved as unmatched counts
   config_param :pattern1, :string # string: NAME LOW HIGH
@@ -20,11 +26,18 @@ class Fluent::NumericCounterOutput < Fluent::Output
   (2..PATTERN_MAX_NUM).each do |i|
     config_param ('pattern' + i.to_s).to_sym, :string, :default => nil
   end
-  
-  attr_accessor :counts, :last_checked
 
+  attr_accessor :counts
+  attr_accessor :last_checked
+  attr_accessor :saved_duration
+  attr_accessor :saved_at
   attr_accessor :patterns
 
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
   def parse_num(str)
     if str.nil?
       nil
@@ -58,7 +71,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
 
     invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
     if invalids.size > 0
-      $log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):" + invalids.join(",")
+      log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):" + invalids.join(",")
     end
     (1..PATTERN_MAX_NUM).each do |i|
       next unless conf["pattern#{i}"]
@@ -88,12 +101,20 @@ class Fluent::NumericCounterOutput < Fluent::Output
       @removed_length = @removed_prefix_string.length
     end
 
+    if @store_file
+      f = Pathname.new(@store_file)
+      if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
+        raise Fluent::ConfigError, "#{@store_file} is not writable"
+      end
+    end
+
     @counts = count_initialized
     @mutex = Mutex.new
   end
 
   def start
     super
+    load_status(@store_file, @count_interval) if @store_file
     start_watch
   end
 
@@ -101,6 +122,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
     super
     @watcher.terminate
     @watcher.join
+    save_status(@store_file) if @store_file
   end
 
   def count_initialized(keys=nil)
@@ -217,7 +239,7 @@ class Fluent::NumericCounterOutput < Fluent::Output
   end
 
   def watch
-    @last_checked = Fluent::Engine.now
+    @last_checked ||= Fluent::Engine.now
     while true
       sleep 0.5
       if Fluent::Engine.now - @last_checked >= @count_interval
@@ -249,4 +271,60 @@ class Fluent::NumericCounterOutput < Fluent::Output
 
     chain.next
   end
+
+  # Store internal status into a file
+  #
+  # @param [String] file_path
+  def save_status(file_path)
+    begin
+      Pathname.new(file_path).open('wb') do |f|
+        @saved_at = Fluent::Engine.now
+        @saved_duration = @saved_at - @last_checked
+        Marshal.dump({
+          :counts           => @counts,
+          :saved_at         => @saved_at,
+          :saved_duration   => @saved_duration,
+          :aggregate        => @aggregate,
+          :count_key        => @count_key,
+          :patterns         => @patterns,
+        }, f)
+      end
+    rescue => e
+      log.warn "out_datacounter: Can't write store_file #{e.class} #{e.message}"
+    end
+  end
+
+  # Load internal status from a file
+  #
+  # @param [String] file_path
+  # @param [Interger] count_interval
+  def load_status(file_path, count_interval)
+    return unless (f = Pathname.new(file_path)).exist?
+
+    begin
+      f.open('rb') do |f|
+        stored = Marshal.load(f)
+        if stored[:aggregate] == @aggregate and
+          stored[:count_key] == @count_key and
+          stored[:patterns]  == @patterns
+
+          if Fluent::Engine.now <= stored[:saved_at] + count_interval
+            @counts = stored[:counts]
+            @saved_at = stored[:saved_at]
+            @saved_duration = stored[:saved_duration]
+
+            # skip the saved duration to continue counting
+            @last_checked = Fluent::Engine.now - @saved_duration
+          else
+            log.warn "out_datacounter: stored data is outdated. ignore stored data"
+          end
+        else
+          log.warn "out_datacounter: configuration param was changed. ignore stored data"
+        end
+      end
+    rescue => e
+      log.warn "out_datacounter: Can't load store_file #{e.class} #{e.message}"
+    end
+  end
+
 end

data/test/helper.rb

@@ -22,6 +22,7 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
+require 'delorean'
 require 'fluent/plugin/out_numeric_counter'
 
 class Test::Unit::TestCase

data/test/plugin/test_out_numeric_counter.rb

@@ -496,4 +496,62 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     d.instance.flush_emit(60)
     assert_equal 2, d.emits.size # +0
   end
+
+  def test_store_file
+    dir = "test/tmp"
+    Dir.mkdir dir unless Dir.exist? dir
+    file = "#{dir}/test.dat"
+    File.unlink file if File.exist? file
+
+    # test store
+    d = create_driver(CONFIG + %[store_file #{file}])
+    d.run do
+      d.instance.flush_emit(60)
+      d.emit({'target' => 1})
+      d.emit({'target' => 1})
+      d.emit({'target' => 1})
+      d.instance.shutdown
+    end
+    stored_counts = d.instance.counts
+    stored_saved_at = d.instance.saved_at
+    stored_saved_duration = d.instance.saved_duration
+    assert File.exist? file
+
+    # test load
+    d = create_driver(CONFIG + %[store_file #{file}])
+    d.run do
+      loaded_counts = d.instance.counts
+      loaded_saved_at = d.instance.saved_at
+      loaded_saved_duration = d.instance.saved_duration
+      assert_equal stored_counts, loaded_counts
+      assert_equal stored_saved_at, loaded_saved_at
+      assert_equal stored_saved_duration, loaded_saved_duration
+    end
+
+    # test not to load if config is changed
+    d = create_driver(CONFIG + %[count_key foobar store_file #{file}])
+    d.run do
+      loaded_counts = d.instance.counts
+      loaded_saved_at = d.instance.saved_at
+      loaded_saved_duration = d.instance.saved_duration
+      assert_equal({}, loaded_counts)
+      assert_equal(nil, loaded_saved_at)
+      assert_equal(nil, loaded_saved_duration)
+    end
+
+    # test not to load if stored data is outdated.
+    Delorean.jump 61 # jump more than count_interval
+    d = create_driver(CONFIG + %[store_file #{file}])
+    d.run do
+      loaded_counts = d.instance.counts
+      loaded_saved_at = d.instance.saved_at
+      loaded_saved_duration = d.instance.saved_duration
+      assert_equal({}, loaded_counts)
+      assert_equal(nil, loaded_saved_at)
+      assert_equal(nil, loaded_saved_duration)
+    end
+    Delorean.back_to_the_present
+
+    File.unlink file
+  end
 end

metadata

@@ -1,46 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-numeric-counter
 version: !ruby/object:Gem::Version
-  version: 0.2.1
-  prerelease: 
+  version: 0.2.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-08 00:00:00.000000000 Z
+date: 2014-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: delorean
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Counts messages, with specified key and numeric value in specified range
@@ -50,7 +59,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -60,28 +70,28 @@ files:
 - test/helper.rb
 - test/plugin/test_out_numeric_counter.rb
 homepage: https://github.com/tagomoris/fluent-plugin-numeric-counter
-licenses: []
+licenses:
+- APLv2
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fluentd plugin to count messages with specified numeric values
 test_files:
 - test/helper.rb