fluent-plugin-https-client 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3b59b6107b871dd3b8d36e7df137961cd9f1a68e
+  data.tar.gz: 63f34236d611c5e7968e298a696a2d934b66aa88
+SHA512:
+  metadata.gz: 3da69d57d183d3fac60a0f40e0388633c2efe78ffa525d6651e58dba39ac65a5c7724fc6b20f34346df1889e5fb1712465fdd2e60f86a37ca9d971369eafa1c6
+  data.tar.gz: 745b017399dc6383bc9669953cf9d0a5964926ef27fcb9414226f8fbd83280523d165e5d7705d5c9c5d31cba408f631286c4d237c80deccdbeaf9c9c528682cc

data/README.md

@@ -0,0 +1,44 @@
+## Fluent::Plugin::HTTPSClient
+
+Output plugin for [Fluentd](http://fluentd.org), for sending records to an HTTP or HTTPS endpoint, with SSL, Proxy,
+ and Header implementation.
+ 
+## Configuration Guide
+
+    <match *>
+      type              https_client           
+      endpoint_url      # endpoint_url 
+      http_method       # get / post / put / delete defaults to post
+      serializer        # json / form defaults to form
+      include_timestamp # true / false defaults to false
+      rate_limit_msec   # limit the rate in ms, defaults to 10 ms
+      auth              # basic / none
+      use_ssl           # true / false
+      proxy_addr        # proxy url
+      proxy_port        # proxy port
+      username          # user name if auth is basic
+      password          # password if auth is basic
+      <header>          # HTTP headers (see examples below)
+        Accept          application/json
+        auth_token      my_secret
+      </header>
+    </match>
+    
+    
+ ### Use Cases
+ * send records to HTTP endpoints
+ * send records to HTTPS endpoints
+ * send events through proxy
+ * send events by setting custom headers (header-based authentication, etc)
+ 
+
+### Credits
+* Majority of the code is cloned from  [fluent-plugin-out-http][2]
+* SSL implementation from  [fluent-plugin-out-https][3]
+
+### Exception Handling
+* For retries in case of exceptions, use
+ [fluent-plugin-bufferize](https://github.com/kazegusuri/fluent-plugin-bufferize) as a wrapper.
+
+
+

data/fluent-plugin-https-client.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-https-client'
+  gem.version       = '0.0.1'
+  gem.authors       = ['Arash Vatanpoor']
+  gem.email         = ['arash@a-venture.org']
+  gem.summary       = %q{A generic Fluentd output plugin to send records to HTTP / HTTPS endpoint}
+  gem.description   = %q{A generic Fluentd output plugin to send records to HTTP / HTTPS endpoint, with SSL, Proxy, and Header implementation}
+  gem.homepage      = 'https://github.com/a-venture/fluent-plugin-https-client'
+  gem.license       = 'MIT'
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+  gem.required_ruby_version = '>= 2.1.2'
+
+  gem.add_runtime_dependency 'yajl-ruby', '~> 1.0'
+  gem.add_runtime_dependency 'fluentd', '~> 0.12'
+end

data/lib/fluent/plugin/out_https_client.rb

@@ -0,0 +1,174 @@
+class Fluent::HTTPSOutput < Fluent::Output
+  Fluent::Plugin.register_output('https_client', self)
+
+  def initialize
+    super
+    require 'net/https'
+    require 'openssl'
+    require 'uri'
+    require 'yajl'
+  end
+
+  config_param :use_ssl, :bool, :default => false
+  config_param :include_tag, :bool, :default => false
+  config_param :include_timestamp, :bool, :default => false
+  config_param :endpoint_url, :string
+  config_param :http_method, :string, :default => :post
+  config_param :serializer, :string, :default => :form
+  config_param :rate_limit_msec, :integer, :default => 10
+  config_param :auth, :string, :default => nil
+  config_param :username, :string, :default => ''
+  config_param :password, :string, :default => ''
+  config_param :proxy_addr, :string, :default => ''
+  config_param :proxy_port, :integer, :default => -1
+
+  
+
+ def configure(conf)
+    super
+
+    @use_ssl = conf['use_ssl']
+    @include_tag = conf['include_tag']
+    @include_timestamp = conf['include_timestamp']
+
+
+    @use_proxy = false
+    if @proxy_port and @proxy_addr
+      # check for proxy settings
+      if @proxy_port > 0 and @proxy_addr.empty?
+        raise Fluent::ConfigError, 'HTTPS Output :: provide a valid proxy address'
+      elsif  @proxy_port <= 0 and !@proxy_addr.empty?
+        raise Fluent::ConfigError, 'HTTPS Output :: provide a valid proxy port number'
+      elsif @proxy_port == 0
+        raise Fluent::ConfigError, 'HTTPS Output :: provide a valid proxy port number'
+      elsif  @proxy_port > 0 and !@proxy_addr.empty?
+        @use_proxy = true
+      end
+    end
+
+    serializers = [:json, :form]
+    @serializer = if serializers.include? @serializer.intern
+                    @serializer.intern
+                  else
+                    :form
+                  end
+
+    http_methods = [:get, :put, :post, :delete]
+    @http_method = if http_methods.include? @http_method.intern
+                    @http_method.intern
+                  else
+                    :post
+                  end
+
+    @auth = case @auth
+            when 'basic' then :basic
+            else
+              :none
+            end
+
+    # create the headers hash
+    @headers = {}
+    conf.elements.each do |elem|
+        elem.keys.each do |key|
+          @headers[key] = elem[key]
+        end
+    end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url
+     @endpoint_url
+  end
+
+  def set_body(req, tag, record)
+    if @include_tag
+      record['tag'] = tag
+    end
+    if @include_timestamp
+      record['timestamp'] = Time.now.to_i
+    end 
+    if @serializer == :json
+      set_json_body(req, record)
+    else
+      req.set_form_data(record)
+    end
+     req
+  end
+
+  def set_header(req)
+    @headers.each do |key, value|
+      req[key] = value
+    end
+  end
+
+  def set_json_body(req, data)
+    req.body = Yajl.dump(data)
+    req['Content-Type'] = 'application/json'
+  end
+
+  def create_request(tag, time, record)
+    url = format_url()
+    uri = URI.parse(url)
+    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+    set_body(req, tag, record)
+    set_header(req)
+    return req, uri
+  end
+
+  def send_request(req, uri, record)
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      $log.info('Dropped request due to rate limiting')
+      return
+    end
+    
+    res = nil
+    begin
+      if @auth and @auth == :basic
+        req.basic_auth(@username, @password)
+      end
+      @last_request_time = Time.now.to_f
+      if @use_proxy
+        https = Net::HTTP.new(uri.host, uri.port, @proxy_addr, @proxy_port)
+      else
+        https = Net::HTTP.new(uri.host, uri.port)
+      end
+      https.use_ssl = @use_ssl
+      https.ca_file = OpenSSL::X509::DEFAULT_CERT_FILE 
+      https.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      res = https.start {|http| http.request(req) }
+    rescue IOError, EOFError, SystemCallError
+      $log.warn "HTTPS Output :: Net::HTTP.#{req.method.capitalize} raises exception: #{$!.class}, '#{$!.message}'"
+    end
+    unless res and res.is_a?(Net::HTTPSuccess)
+      res_summary = if res
+                      "#{res.code} #{res.message} #{res.body}"
+                    else
+                      "res=nil"
+                    end
+      $log.warn "HTTPS Output :: failed to #{req.method} #{uri} (#{res_summary})"
+      $log.warn "HTTPS Output :: record failed to send : #{record}"
+    else 
+      $log.info "HTTPS Output :: emitted record : #{record}"
+    end
+  end
+
+  def handle_record(tag, time, record)
+    req, uri = create_request(tag, time, record)
+    send_request(req, uri, record)
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+    chain.next
+  end
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-https-client
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Arash Vatanpoor
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+description: A generic Fluentd output plugin to send records to HTTP / HTTPS endpoint,
+  with SSL, Proxy, and Header implementation
+email:
+- arash@a-venture.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- fluent-plugin-https-client.gemspec
+- lib/fluent/plugin/out_https_client.rb
+homepage: https://github.com/a-venture/fluent-plugin-https-client
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: A generic Fluentd output plugin to send records to HTTP / HTTPS endpoint
+test_files: []