fluent-plugin-http-record-modifier 0.0.0 → 0.0.1

data/fluent-plugin-http-record-modifier.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-http-record-modifier"
-  spec.version       = "0.0.0"
+  spec.version       = "0.0.1"
   spec.authors       = ["Augusto Nishi"]
   spec.email         = ["augusto.nishi@gmail.com"]
   spec.summary       = %q{fluentd filter plugin for modifing record based on a HTTP request}

data/lib/fluent/plugin/filter_http_record_modifier.rb

@@ -22,6 +22,9 @@ module Fluent
     config_param :username, :string, :default => ''
     config_param :password, :string, :default => ''
     config_param :raise_on_error, :bool, :default => true
+    config_param :cache, :bool, :default => true
+    config_param :expire, :integer, :default => 600 #10 min
+    config_param :renew_time_key, :string, :default => nil
 
     def configure(conf)
       super
@@ -53,6 +56,8 @@ module Fluent
         @keep_keys = @keep_keys.split(',')
       end
 
+      @request_cache = Cache.new(@cache, @expire)
+
       @placeholder_expander = PlaceholderExpander.new({
         :log           => log,
         :auto_typecast => @auto_typecast,
@@ -77,8 +82,12 @@ module Fluent
       es.each do |time, record|
         last_record = record # for debug log
         req, uri = create_request(tag, time, record)
-        res = send_request(req, uri)
-        body = deserialize_body(res)
+        body = @request_cache.get(uri.to_s)
+        if body.nil?
+          res = send_request(req, uri)
+          body = deserialize_body(res)
+          @request_cache.set(uri.to_s, body)
+        end
         new_record = reform(time, record, placeholders, body)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
           time = new_record[@renew_time_key].to_i
@@ -220,6 +229,34 @@ module Fluent
       rev_tag_suffix.reverse!
     end
 
+    class Cache
+      def initialize(cache, expire)
+        @data = {}
+        @cache = cache
+        @expire = expire
+      end
+
+      def get(key)
+        unless @data.has_key?(key) and @cache
+          return nil
+        end
+        if Time.now.to_i > @data[key]['time'] + @expire
+          @data.delete(key)
+          return nil
+        end
+        return @data[key]['value']
+      end
+
+      def set(key, value)
+        if @cache
+          @data[key] = {
+            'time' => Time.now.to_i,
+            'value' => value
+          }
+        end
+      end
+    end
+
     class PlaceholderExpander
       attr_reader :placeholders, :log
 
@@ -230,8 +267,7 @@ module Fluent
 
       def prepare_placeholders(time, record, opts)
         placeholders = { '${time}' => Time.at(time).to_s }
-        record.each {|key, value| crawl_placeholder(value, placeholders, "#{key}")
-}
+        record.each {|key, value| crawl_placeholder(value, placeholders, "#{key}")}
         opts.each do |key, value|
           if value.kind_of?(Array) # tag_parts, etc
             size = value.size

metadata

@@ -1,55 +1,62 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-http-record-modifier
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
+  prerelease: 
 platform: ruby
 authors:
 - Augusto Nishi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-30 00:00:00.000000000 Z
+date: 2015-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: fluentd filter plugin for modifing record based on a HTTP request
@@ -59,36 +66,36 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - fluent-plugin-http-record-modifier.gemspec
-- lib/fluent/plugin/.filter_http_record_modifier.rb.swp
 - lib/fluent/plugin/filter_http_record_modifier.rb
 homepage: http://github.com/DEVTecnologia/fluent-plugin-http-record-modifier
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.1
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: fluentd filter plugin for modifing record based on a HTTP request
 test_files: []

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 2ea15085273d646c9d75d934aa5b2e90eeef89af
-  data.tar.gz: 693781df064fceb076c7d99a8718a89724898aa8
-SHA512:
-  metadata.gz: d938f24e8a733b6a9161f94e09d066d2eed540d29a5487684c14e226867a92f8fb6f7fa229ab49a09d25a89d178c2379245c65d77ad4e2ecd251d651e4a40122
-  data.tar.gz: 859c43d550ef56cc379bf899c6fbf7e43964103d4369fe85d9eb70e2c87b4e245eb50f8edb9ce0bec993b31600968a06fb13cfcbd66e04f6910ecdeba79cebac