fluent-plugin-http-record-modifier 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2ea15085273d646c9d75d934aa5b2e90eeef89af
+  data.tar.gz: 693781df064fceb076c7d99a8718a89724898aa8
+SHA512:
+  metadata.gz: d938f24e8a733b6a9161f94e09d066d2eed540d29a5487684c14e226867a92f8fb6f7fa229ab49a09d25a89d178c2379245c65d77ad4e2ecd251d651e4a40122
+  data.tar.gz: 859c43d550ef56cc379bf899c6fbf7e43964103d4369fe85d9eb70e2c87b4e245eb50f8edb9ce0bec993b31600968a06fb13cfcbd66e04f6910ecdeba79cebac

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-mqtt.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Augusto Nishi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+# Fluent::Plugin::Simple Value to Hash
+
+Fluent plugin for converting simple value variables to hash to be usable to others plugins
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-http-record-modifier'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-http-record-modifier
+
+## Usage
+
+```
+
+<filter tag>
+  type http_record_modifier
+  method (defaults to GET)
+  renew_record (defaults to false)
+  endpoint_url yourapi.com/api/something/
+  serializer (form or json, just used on POST)
+  authentication (none or basic, defaults to none)
+  username (default to '')
+  password (default to '')
+  <params>
+    var ${tag_parts[1]}
+    var2 ${record_attr}
+  </params>
+  <record>
+    name ${body.name}
+    body ${body}
+    array ${body.array}
+    second ${body.array[1]}
+    last ${body.array[-1]}
+    deep ${body.array[0].attr}
+  </record>
+</filter>
+
+```
+
+## Contributing
+
+1. Fork it ( http://github.com/DEVTecnologia/fluent-plugin-http-record-modifier/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-http-record-modifier.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-http-record-modifier"
+  spec.version       = "0.0.0"
+  spec.authors       = ["Augusto Nishi"]
+  spec.email         = ["augusto.nishi@gmail.com"]
+  spec.summary       = %q{fluentd filter plugin for modifing record based on a HTTP request}
+  spec.description   = %q{fluentd filter plugin for modifing record based on a HTTP request}
+  spec.homepage      = "http://github.com/DEVTecnologia/fluent-plugin-http-record-modifier"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "fluentd"
+
+end

data/lib/fluent/plugin/filter_http_record_modifier.rb

@@ -0,0 +1,288 @@
+module Fluent
+  class HttpRecordModifier < Filter
+    Plugin.register_filter('http_record_modifier', self)
+
+    def initialize
+      require 'socket'
+      require 'yajl'
+      require 'net/http'
+      require 'uri'
+      super
+    end
+
+    #Based on Filter Record Tranformer that is built-in on Fluent
+    config_param :remove_keys, :string, :default => nil
+    config_param :keep_keys, :string, :default => nil
+    config_param :method, :string, :default => :get
+    config_param :rene_record, :bool, :default => false
+    config_param :auto_typecast, :bool, :default => false # false for lower version compatibility
+    config_param :endpoint_url, :string
+    config_param :serializer, :string, :default => :form
+    config_param :authentication, :string, :default => nil 
+    config_param :username, :string, :default => ''
+    config_param :password, :string, :default => ''
+    config_param :raise_on_error, :bool, :default => true
+
+    def configure(conf)
+      super
+      @method ||= conf['method']
+      @map = {}
+      # <record></record> directive
+      conf.elements.select { |element| element.name == 'record' }.each do |element|
+        element.each_pair do |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          @map[k] = parse_value(v)
+        end
+      end
+
+      @maped_params = {}
+      # <params></params> directive
+      conf.elements.select { |element| element.name == 'params' }.each do |element|
+        element.each_pair do |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          @maped_params[k] = parse_value(v)
+        end
+      end
+
+      if @remove_keys
+        @remove_keys = @remove_keys.split(',')
+      end
+
+      if @keep_keys
+        raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
+        @keep_keys = @keep_keys.split(',')
+      end
+
+      @placeholder_expander = PlaceholderExpander.new({
+        :log           => log,
+        :auto_typecast => @auto_typecast,
+      })
+
+      @hostname = Socket.gethostname
+    end
+
+    def filter_stream(tag, es)
+      new_es = MultiEventStream.new
+      tag_parts = tag.split('.')
+      tag_prefix = tag_prefix(tag_parts)
+      tag_suffix = tag_suffix(tag_parts)
+      placeholders = {
+        'tag' => tag,
+        'tag_parts' => tag_parts,
+        'tag_prefix' => tag_prefix,
+        'tag_suffix' => tag_suffix,
+        'hostname' => @hostname,
+      }
+      last_record = nil
+      es.each do |time, record|
+        last_record = record # for debug log
+        req, uri = create_request(tag, time, record)
+        res = send_request(req, uri)
+        body = deserialize_body(res)
+        new_record = reform(time, record, placeholders, body)
+        if @renew_time_key && new_record.has_key?(@renew_time_key)
+          time = new_record[@renew_time_key].to_i
+        end
+        new_es.add(time, new_record)
+      end
+      new_es
+    rescue => e
+      log.warn "failed to reform records", :error_class => e.class, :error => e.message
+      log.warn_backtrace
+      log.debug "map:#{@map} record:#{last_record} placeholders:#{placeholders}"
+    end
+
+    def deserialize_body(res)
+      clean = {}
+      body = res.body
+      if res.content_type == 'application/json'
+        body = Yajl.load(body)
+        if body.is_a? Hash
+          clean = Yajl.load(res.body)
+        end
+      end
+      clean['body'] = body
+      clean
+    end
+
+    def format_params(tag, time, record)
+      tag_parts = tag.split('.')
+      tag_prefix = tag_prefix(tag_parts)
+      tag_suffix = tag_suffix(tag_parts)
+      placeholders = {
+        'tag' => tag,
+        'tag_parts' => tag_parts,
+        'tag_prefix' => tag_prefix,
+        'tag_suffix' => tag_suffix,
+        'hostname' => @hostname,
+      }
+      @placeholder_expander.prepare_placeholders(time, record, placeholders)
+
+      expand_placeholders(@maped_params)
+    end
+
+    def set_body(req, tag, time, record)
+      if @serializer == :json
+        req['Content-Type'] = 'application/json'
+      else
+        req.set_form_data(record)
+      end
+      req
+    end
+
+
+    def create_request(tag, time, record)
+      url = URI.encode(@endpoint_url.to_s)
+      uri = URI.parse(url)
+      params = format_params(tag, time, record)
+      uri.query = URI.encode_www_form(params)
+      req = Net::HTTP.const_get(@method.to_s.capitalize).new(uri)
+      unless @method.to_s.capitalize == 'Get'
+        set_body(req, tag, time, record)
+      end
+      return req, uri
+    end
+
+    def send_request(req, uri)    
+      res = nil
+
+      begin
+        if @auth and @auth == :basic
+          req.basic_auth(@username, @password)
+        end
+        res = Net::HTTP.new(uri.host, uri.port).start {|http| http.request(req) }
+      rescue => e # rescue all StandardErrors
+        # server didn't respond
+        $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
+        raise e if @raise_on_error
+      end # end begin
+    end # end send_request
+
+    private
+
+    def parse_value(value_str)
+      if value_str.start_with?('{', '[')
+        JSON.parse(value_str)
+      else
+        value_str
+      end
+    rescue => e
+      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", :error_class => e.class, :error => e.message
+      value_str # emit as string
+    end
+
+    def reform(time, record, opts, res)
+      @placeholder_expander.prepare_placeholders(time, res, opts)
+
+      new_record = @renew_record ? {} : record.dup
+      @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
+      new_record.merge!(expand_placeholders(@map))
+      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
+
+      new_record
+    end
+
+    def expand_placeholders(value)
+      if value.is_a?(String)
+        new_value = @placeholder_expander.expand(value)
+      elsif value.is_a?(Hash)
+        new_value = {}
+        value.each_pair do |k, v|
+          new_value[@placeholder_expander.expand(k, true)] = expand_placeholders(v)
+        end
+      elsif value.is_a?(Array)
+        new_value = []
+        value.each_with_index do |v, i|
+          new_value[i] = expand_placeholders(v)
+        end
+      else
+        new_value = value
+      end
+      new_value
+    end
+
+    def tag_prefix(tag_parts)
+      return [] if tag_parts.empty?
+      tag_prefix = [tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
+      end
+      tag_prefix
+    end
+
+    def tag_suffix(tag_parts)
+      return [] if tag_parts.empty?
+      rev_tag_parts = tag_parts.reverse
+      rev_tag_suffix = [rev_tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
+      end
+      rev_tag_suffix.reverse!
+    end
+
+    class PlaceholderExpander
+      attr_reader :placeholders, :log
+
+      def initialize(params)
+        @log = params[:log]
+        @auto_typecast = params[:auto_typecast]
+      end
+
+      def prepare_placeholders(time, record, opts)
+        placeholders = { '${time}' => Time.at(time).to_s }
+        record.each {|key, value| crawl_placeholder(value, placeholders, "#{key}")
+}
+        opts.each do |key, value|
+          if value.kind_of?(Array) # tag_parts, etc
+            size = value.size
+            value.each_with_index { |v, idx|
+              placeholders.store("${#{key}[#{idx}]}", v)
+              placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
+            }
+          else # string, interger, float, and others?
+            placeholders.store("${#{key}}", value)
+          end
+        end
+
+        @placeholders = placeholders
+      end
+
+      def crawl_placeholder (value, placeholder, before, limit = 50)
+        if limit >= 0
+          if value.kind_of?(Hash) 
+            value.each {|key, v| crawl_placeholder(v, placeholder, "#{before}.#{key}", limit - 1)}
+          elsif value.kind_of?(Array) # tag_parts, etc
+            size = value.size
+            value.each_with_index { |v, idx|
+              crawl_placeholder(v, placeholder, "#{before}[#{idx}]", limit - 1)
+              crawl_placeholder(v, placeholder, "#{before}[#{idx-size}]", limit - 1) #suport [-1]
+            }
+          end
+        end
+        # string, interger, float, and others?
+        placeholder.store("${#{before}}", value)
+      end
+
+      def expand(str, force_stringify=false)
+        if @auto_typecast and !force_stringify
+          single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
+          if single_placeholder_matched
+            log_unknown_placeholder($1)
+            return @placeholders[single_placeholder_matched[1]]
+          end
+        end
+        str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
+          log_unknown_placeholder($1)
+          @placeholders[$1]
+        }
+      end
+
+      private
+      def log_unknown_placeholder(placeholder)
+        unless @placeholders.include?(placeholder)
+          log.warn "unknown placeholder `#{placeholder}` found"
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-http-record-modifier
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Augusto Nishi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: fluentd filter plugin for modifing record based on a HTTP request
+email:
+- augusto.nishi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-http-record-modifier.gemspec
+- lib/fluent/plugin/.filter_http_record_modifier.rb.swp
+- lib/fluent/plugin/filter_http_record_modifier.rb
+homepage: http://github.com/DEVTecnologia/fluent-plugin-http-record-modifier
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: fluentd filter plugin for modifing record based on a HTTP request
+test_files: []