fluent-plugin-datacounter 0.4.1 → 0.4.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1a2ccf7898022261c52e17eb5aa9375df17fcac1
+  data.tar.gz: 19dbfaf01f24fe21fcfc76429d9be3c77d3c4167
+SHA512:
+  metadata.gz: 4739c24aabcd790e73038e60956a9bc7fdd00434d910f07e2f6cfbe55b28790fbc1859fd2027c9a8d01c9170f040726c1bdbb8fd394b6d38b6e57582d9011c46
+  data.tar.gz: 14729172930a3b75453916c757c2be4415f428a372bdce4c1ae0120152389b7e845bdc942b06f48f5c9848eaa4045799f918a7884504ce67554f9c7abefd5d37

data/{README.rdoc → README.md}

@@ -1,8 +1,8 @@
-= fluent-plugin-datacounter
+# fluent-plugin-datacounter
 
-== Component
+## Component
 
-=== DataCounterOutput
+### DataCounterOutput
 
 Count messages with data matches any of specified regexp patterns in specified attribute.
 
@@ -17,9 +17,9 @@ DataCounterOutput emits messages contains results data, so you can output these
 
 'input_tag_remove_prefix' option available if you want to remove tag prefix from output field names.
 
-== Configuration
+## Configuration
 
-=== DataCounterOutput
+### DataCounterOutput
 
 Count messages that have attribute 'referer' as 'google.com', 'yahoo.com' and 'facebook.com' from all messages matched, per minutes.
 
@@ -116,12 +116,59 @@ And you can get tested messages count with 'output_messages' option:
     # => tag: 'datacount.baz'
     #    message: {'messages' => xxx, 'OK_count' => ...}
 
-== TODO
+## Parameters
+
+* count\_key (required)
+
+    The key to count in the event record.
+
+* tag
+
+    The output tag. Default is `datacount`.
+
+* tag\_prefix
+
+    The prefix string which will be added to the input tag. `output_per_tag yes` must be specified together. 
+
+* input\_tag\_remove\_prefix
+
+    The prefix string which will be removed from the input tag.
+
+* count\_interval
+
+    The interval time to count in seconds. Default is `60`.
+
+* unit
+
+    The interval time to monitor specified an unit (either of `minute`, `hour`, or `day`).
+    Use either of `count_interval` or `unit`.
+
+* aggregate
+
+    Calculate in each input `tag` separetely, or `all` records in a mass. Default is `tag`.
+
+* ouput\_per\_tag
+
+    Emit for each input tag. `tag_prefix` must be specified together. Default is `no`.
+
+* outcast\_unmatched
+
+    Specify `yes` if you do not want to include 'unmatched' counts into percentage. Default is `no`.
+
+* output\_messages
+
+    Specify `yes` if you want to get tested messages. Default is `no`.
+
+* store\_file
+
+    Store internal data into a file of the given path on shutdown, and load on starting.
+
+## TODO
 
 - consider what to do next
 - patches welcome!
 
-== Copyright
+## Copyright
 
 Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
 License::   Apache License, Version 2.0

data/fluent-plugin-datacounter.gemspec

@@ -1,22 +1,22 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-datacounter"
+  gem.version     = "0.4.2"
+  gem.authors     = ["TAGOMORI Satoshi"]
+  gem.email       = ["tagomoris@gmail.com"]
+  gem.homepage    = "https://github.com/tagomoris/fluent-plugin-datacounter"
+  gem.summary     = %q{Fluentd plugin to count records with specified regexp patterns}
+  gem.description = %q{To count records with string fields by regexps (To count records with numbers, use numeric-counter)}
+  gem.license     = "APLv2"
 
-Gem::Specification.new do |s|
-  s.name        = "fluent-plugin-datacounter"
-  s.version     = "0.4.1"
-  s.authors     = ["TAGOMORI Satoshi"]
-  s.email       = ["tagomoris@gmail.com"]
-  s.homepage    = "https://github.com/tagomoris/fluent-plugin-datacounter"
-  s.summary     = %q{Output filter plugin to count messages that matches specified conditions}
-  s.description = %q{Output filter plugin to count messages that matches specified conditions}
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ["lib"]
 
-  s.rubyforge_project = "fluent-plugin-datacounter"
+  gem.add_runtime_dependency "fluentd"
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
-
-  s.add_development_dependency "rake"
-  s.add_runtime_dependency "fluentd"
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "delorean"
 end

data/lib/fluent/plugin/out_datacounter.rb

@@ -1,6 +1,11 @@
 class Fluent::DataCounterOutput < Fluent::Output
   Fluent::Plugin.register_output('datacounter', self)
 
+  def initialize
+    super
+    require 'pathname'
+  end
+
   PATTERN_MAX_NUM = 20
 
   config_param :count_interval, :time, :default => nil
@@ -13,6 +18,7 @@ class Fluent::DataCounterOutput < Fluent::Output
   config_param :count_key, :string
   config_param :outcast_unmatched, :bool, :default => false
   config_param :output_messages, :bool, :default => false
+  config_param :store_file, :string, :default => nil
 
   # pattern0 reserved as unmatched counts
   config_param :pattern1, :string # string: NAME REGEXP
@@ -22,6 +28,8 @@ class Fluent::DataCounterOutput < Fluent::Output
 
   attr_accessor :tick
   attr_accessor :counts
+  attr_accessor :saved_duration
+  attr_accessor :saved_at
   attr_accessor :last_checked
 
   def configure(conf)
@@ -78,12 +86,20 @@ class Fluent::DataCounterOutput < Fluent::Output
       @removed_length = @removed_prefix_string.length
     end
 
+    if @store_file
+      f = Pathname.new(@store_file)
+      if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
+        raise Fluent::ConfigError, "#{@store_file} is not writable"
+      end
+    end
+
     @counts = count_initialized
     @mutex = Mutex.new
   end
 
   def start
     super
+    load_status(@store_file, @tick) if @store_file
     start_watch
   end
 
@@ -91,6 +107,7 @@ class Fluent::DataCounterOutput < Fluent::Output
     super
     @watcher.terminate
     @watcher.join
+    save_status(@store_file) if @store_file
   end
 
   def count_initialized(keys=nil)
@@ -210,7 +227,7 @@ class Fluent::DataCounterOutput < Fluent::Output
   
   def watch
     # instance variable, and public accessable, for test
-    @last_checked = Fluent::Engine.now
+    @last_checked ||= Fluent::Engine.now
     while true
       sleep 0.5
       if Fluent::Engine.now - @last_checked >= @tick
@@ -242,4 +259,60 @@ class Fluent::DataCounterOutput < Fluent::Output
 
     chain.next
   end
+
+  # Store internal status into a file
+  #
+  # @param [String] file_path
+  def save_status(file_path)
+    begin
+      Pathname.new(file_path).open('wb') do |f|
+        @saved_at = Fluent::Engine.now
+        @saved_duration = @saved_at - @last_checked
+        Marshal.dump({
+          :counts           => @counts,
+          :saved_at        => @saved_at,
+          :saved_duration  => @saved_duration,
+          :aggregate        => @aggregate,
+          :count_key        => @count_key,
+          :patterns         => @patterns,
+        }, f)
+      end
+    rescue => e
+      $log.warn "out_datacounter: Can't write store_file #{e.class} #{e.message}"
+    end
+  end
+
+  # Load internal status from a file
+  #
+  # @param [String] file_path
+  # @param [Interger] tick The count interval
+  def load_status(file_path, tick)
+    return unless (f = Pathname.new(file_path)).exist?
+
+    begin
+      f.open('rb') do |f|
+        stored = Marshal.load(f)
+        if stored[:aggregate] == @aggregate and
+          stored[:count_key] == @count_key and
+          stored[:patterns]  == @patterns
+
+          if Fluent::Engine.now <= stored[:saved_at] + tick
+            @counts = stored[:counts]
+            @saved_at = stored[:saved_at]
+            @saved_duration = stored[:saved_duration]
+
+            # skip the saved duration to continue counting
+            @last_checked = Fluent::Engine.now - @saved_duration
+          else
+            $log.warn "out_datacounter: stored data is outdated. ignore stored data"
+          end
+        else
+          $log.warn "out_datacounter: configuration param was changed. ignore stored data"
+        end
+      end
+    rescue => e
+      $log.warn "out_datacounter: Can't load store_file #{e.class} #{e.message}"
+    end
+  end
+
 end

data/test/helper.rb

@@ -22,6 +22,7 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
+require 'delorean'
 require 'fluent/plugin/out_datacounter'
 
 class Test::Unit::TestCase

data/test/plugin/test_out_datacounter.rb

@@ -653,4 +653,62 @@ class DataCounterOutputTest < Test::Unit::TestCase
     d.instance.flush_emit(60)
     assert_equal 2, d.emits.size # +0
   end
+
+  def test_store_file
+    dir = "test/tmp"
+    Dir.mkdir dir unless Dir.exist? dir
+    file = "#{dir}/test.dat"
+    File.unlink file if File.exist? file
+
+    # test store
+    d = create_driver(CONFIG + %[store_file #{file}])
+    d.run do
+      d.instance.flush_emit(60)
+      d.emit({'target' => 1})
+      d.emit({'target' => 1})
+      d.emit({'target' => 1})
+      d.instance.shutdown
+    end
+    stored_counts = d.instance.counts
+    stored_saved_at = d.instance.saved_at
+    stored_saved_duration = d.instance.saved_duration
+    assert File.exist? file
+
+    # test load
+    d = create_driver(CONFIG + %[store_file #{file}])
+    d.run do
+      loaded_counts = d.instance.counts
+      loaded_saved_at = d.instance.saved_at
+      loaded_saved_duration = d.instance.saved_duration
+      assert_equal stored_counts, loaded_counts
+      assert_equal stored_saved_at, loaded_saved_at
+      assert_equal stored_saved_duration, loaded_saved_duration
+    end
+
+    # test not to load if config is changed
+    d = create_driver(CONFIG + %[count_key foobar store_file #{file}])
+    d.run do
+      loaded_counts = d.instance.counts
+      loaded_saved_at = d.instance.saved_at
+      loaded_saved_duration = d.instance.saved_duration
+      assert_equal({}, loaded_counts)
+      assert_equal(nil, loaded_saved_at)
+      assert_equal(nil, loaded_saved_duration)
+    end
+
+    # test not to load if stored data is outdated.
+    Delorean.jump 61 # jump more than count_interval
+    d = create_driver(CONFIG + %[store_file #{file}])
+    d.run do
+      loaded_counts = d.instance.counts
+      loaded_saved_at = d.instance.saved_at
+      loaded_saved_duration = d.instance.saved_duration
+      assert_equal({}, loaded_counts)
+      assert_equal(nil, loaded_saved_at)
+      assert_equal(nil, loaded_saved_duration)
+    end
+    Delorean.back_to_the_present
+
+    File.unlink file
+  end
 end

metadata

@@ -1,49 +1,73 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datacounter
 version: !ruby/object:Gem::Version
-  version: 0.4.1
-  prerelease: 
+  version: 0.4.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-08 00:00:00.000000000 Z
+date: 2013-08-27 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: delorean
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Output filter plugin to count messages that matches specified conditions
+description: To count records with string fields by regexps (To count records with
+  numbers, use numeric-counter)
 email:
 - tagomoris@gmail.com
 executables: []
@@ -53,7 +77,7 @@ files:
 - .gitignore
 - Gemfile
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - example.conf
 - fluent-plugin-datacounter.gemspec
@@ -61,29 +85,30 @@ files:
 - test/helper.rb
 - test/plugin/test_out_datacounter.rb
 homepage: https://github.com/tagomoris/fluent-plugin-datacounter
-licenses: []
+licenses:
+- APLv2
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: fluent-plugin-datacounter
-rubygems_version: 1.8.23
+rubyforge_project: 
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
-summary: Output filter plugin to count messages that matches specified conditions
+specification_version: 4
+summary: Fluentd plugin to count records with specified regexp patterns
 test_files:
 - test/helper.rb
 - test/plugin/test_out_datacounter.rb
+has_rdoc: