fluent-plugin-gimbal-out-http 1.1.5.pre.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4f82ebc471a2855aed2ad3462e5df3a6ceac5656
+  data.tar.gz: a85ae2faa3b718edd6002b3b27aaa7d3ffd20fcc
+SHA512:
+  metadata.gz: 5eb1ae203708cf68069d06a88a0dc7a10b9d3f76265d90613552551e0731f577ae41525e394a17d87fcc5479b868015a3b42c48488a28b190f2315034a3e557f
+  data.tar.gz: 1e3e5bb18dde2371a9883d814d765b52a3fe2cdf8ee8966d10cc9ec7bf296f839367d13b45a0934fe88d281999cf497b1ac87fe95491380c6afa3f43da571575

data/.gitignore

@@ -0,0 +1,8 @@
+*~
+\#*
+.\#*
+*.gem
+.bundle
+.ruby-version
+Gemfile.lock
+vendor

data/.travis.yml

@@ -0,0 +1,25 @@
+rvm:
+  - 2.2
+  - 2.3
+  - 2.4.5
+  - 2.5.3
+  - 2.6.0
+  - ruby-head
+
+os:
+  - linux
+  - osx
+
+dist: trusty
+
+gemfile:
+  - Gemfile
+
+before_install:
+  - gem update --system=2.7.8
+
+script: bundle exec rake test
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/CHANGELOG.md

@@ -0,0 +1,58 @@
+# Changelog
+
+### 1.1.5
+* Add :raw serializer
+
+### 1.1.4
+* Add custom formatter feature
+* Tweak Travis CI tasks
+
+### 1.1.3
+* Send query_string to endpoint_url
+
+### 1.1.2
+* Added custom headers feature
+
+## 1.1.1
+* Added plain text transport capability
+* Added specify cacert file for ssl verify
+
+## 1.1.0
+* Support for jwt token authentication
+
+## 1.0.1
+* Added endpoint_url placeholder support
+
+## 1.0.0
+* Use Fluentd v1 API
+
+## 0.2.0
+### Added
+* SSL is now supported if `endpoint_url` uses the `https` scheme (uses ruby-2.1 syntax internally)
+* New config: set `ssl_no_verify` to `true` to bypass SSL certificate verification.
+  Use at your own risk.
+### Changed
+* Fixed tests:
+  * Removed some warnings
+  * Fixed failing binary test to use UTF-8
+### Removed
+* Dropped support of Ruby 1.9-2.0
+
+## 0.1.4
+* #11 Updated Fluentd dependency to:  [">= 0.10.0", "< 2"]
+* #10 `password` is now marked as a [secret option](https://github.com/fluent/fluentd/pull/604)
+
+## 0.1.3
+* Added a new configuration option: `raise_on_error` (default: true)
+  * In order to let the plugin raise exceptions like it did in 0.1.1: keep using your configuration as-is
+  * In order to suppress all exceptions: add `raise_on_error false` to your configuration
+
+## 0.1.2
+* #6 Catch all `StandardError`s during HTTP request to prevent td-agent from freezing
+
+## 0.1.1
+* #2 Use yajl instead of json as json serializer
+* #1 Fix a bug where a nil HTTP response caused the plugin to stop working
+
+## 0.1.0
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-out-http.gemspec
+gemspec

data/ISSUE_TEMPLATE.md

@@ -0,0 +1,21 @@
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Provide example config and message
+
+#### Expected Behavior or What you need to ask
+
+...
+
+#### Using Fluentd and out_http plugin versions
+
+* OS version
+* Fluentd v0.12 or v0.14/v1.0
+  * paste result of ``fluentd --version`` or ``td-agent --version``
+* out_http plugin 1.x.y or 0.x.y
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
+* Bear Metal or Within Docker or Kubernetes or others? (optional)

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,35 @@
+# fluent-plugin-out-http, a plugin for [Fluentd](http://fluentd.org)
+
+A generic [fluentd][1] output plugin for sending logs to an HTTP endpoint.
+
+[![Build Status](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-out-http.svg?branch=master)](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-out-http)
+
+## Configuration options
+
+    <match *>
+      @type http
+      endpoint_url    http://localhost.local/api/
+      http_method     put    # default: post
+      serializer      json   # default: form
+      rate_limit_msec 100    # default: 0 = no rate limiting
+      raise_on_error  false  # default: true
+      authentication  basic  # default: none
+      username        alice  # default: ''
+      password        bobpop # default: '', secret: true
+      buffered        true   # default: false. Switch non-buffered/buffered mode
+      cacert_file     /etc/ssl/endpoint1.cert # default: ''
+      token           tokent # default: ''
+      custom_headers  {"token":"arbitrary"} # default: nil
+    </match>
+
+## Usage notes
+
+If you'd like to retry failed requests, consider using [fluent-plugin-bufferize][3].
+
+----
+
+Heavily based on [fluent-plugin-growthforecast][2]
+
+  [1]: http://fluentd.org/
+  [2]: https://github.com/tagomoris/fluent-plugin-growthforecast
+  [3]: https://github.com/sabottenda/fluent-plugin-bufferize

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-gimbal-out-http.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-gimbal-out-http"
+  gem.version       = "1.1.5-1"
+  gem.authors       = ["Marica Odagaki"]
+  gem.email         = ["ento.entotto@gmail.com"]
+  gem.summary       = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint forked from fluent-plugin-out-http}
+  gem.description   = gem.summary
+  gem.homepage      = "https://github.com/PaeDae/fluent-plugin-gimbal-out-http"
+  gem.licenses      = ["Apache-2.0"]
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.required_ruby_version  = '>= 2.1.0'
+
+  gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
+  gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
+  gem.add_runtime_dependency "faraday", "0.15.4"
+  gem.add_runtime_dependency "net-http-persistent", "2.9.4"
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", ">= 3.1.0"
+end

data/lib/fluent/plugin/out_http.rb

@@ -0,0 +1,229 @@
+require 'net/http/persistent'
+require 'uri'
+require 'yajl'
+require 'fluent/plugin/output'
+require 'faraday'
+
+class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
+  Fluent::Plugin.register_output('http', self)
+
+  helpers :compat_parameters, :formatter
+
+  DEFAULT_BUFFER_TYPE = "memory"
+  DEFAULT_FORMATTER = "json"
+
+  def initialize
+    super
+  end
+
+  # Endpoint URL ex. http://localhost.local/api/
+  config_param :endpoint_url, :string
+
+  # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
+  config_param :ssl_no_verify, :bool, :default => false
+
+  # HTTP method
+  config_param :http_method, :enum, list: [:get, :put, :post, :delete], :default => :post
+
+  # form | json | text | raw
+  config_param :serializer, :enum, list: [:json, :form, :text, :raw], :default => :form
+
+  # Simple rate limiting: ignore any records within `rate_limit_msec`
+  # since the last one.
+  config_param :rate_limit_msec, :integer, :default => 0
+
+  # Raise errors that were rescued during HTTP requests?
+  config_param :raise_on_error, :bool, :default => true
+
+  # ca file to use for https request
+  config_param :cacert_file, :string, :default => ''
+
+  # custom headers
+  config_param :custom_headers, :hash, :default => nil
+
+  # 'none' | 'basic' | 'jwt' | 'bearer'
+  config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer],  :default => :none
+  config_param :username, :string, :default => ''
+  config_param :password, :string, :default => '', :secret => true
+  config_param :token, :string, :default => ''
+  # Switch non-buffered/buffered plugin
+  config_param :buffered, :bool, :default => false
+
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+    config_set_default :chunk_keys, ['tag']
+  end
+
+  config_section :format do
+    config_set_default :@type, DEFAULT_FORMATTER
+  end
+
+  def configure(conf)
+    compat_parameters_convert(conf, :buffer, :formatter)
+    super
+
+    @ssl_verify_mode = if @ssl_no_verify
+                         OpenSSL::SSL::VERIFY_NONE
+                       else
+                         OpenSSL::SSL::VERIFY_PEER
+                       end
+
+    @ca_file = @cacert_file
+    @last_request_time = nil
+    raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered
+
+    http_methods = [:get, :put, :post, :delete]
+    @http_method = if http_methods.include? @http_method.intern
+                     @http_method.intern
+                   else
+                     :post
+                   end
+    puts @endpoint_url
+    @uri = URI.parse(@endpoint_url)
+    # ssl_verify = @ssl_no_verify
+    url = @uri.scheme  + "://" + @uri.host + ":" + @uri.port.to_s
+    @adapter = Faraday.new(url: url, ssl: {verify:false} ) do |f|
+      f.request :retry, max:                 5,
+        interval:            1,
+        interval_randomness: 0.5,
+        backoff_factor:      2,
+        methods:             @http_method,
+        exceptions:          %w(Errno::ETIMEDOUT
+                                Faraday::TimeoutError
+                                Faraday::Error::TimeoutError
+                                Net::ReadTimeout).freeze
+
+      f.adapter :net_http_persistent
+    end
+
+    if @formatter_config = conf.elements('format').first
+      @formatter = formatter_create
+    end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url(tag, time, record)
+    @endpoint_url
+  end
+
+  def set_body(req, tag, time, record)
+    if @serializer == :json
+      set_json_body(req, record)
+    elsif @serializer == :text
+      set_text_body(req, record)
+    elsif @serializer == :raw
+      set_raw_body(req, record)
+    else
+      req.set_form_data(record)
+    end
+    req
+  end
+
+  def set_header(req, tag, time, record)
+    if @custom_headers
+      @custom_headers.each do |k,v|
+        req[k] = v
+      end
+      req
+    else
+      req
+    end
+  end
+
+  def set_json_body(req, data)
+    req.body = Yajl.dump(data)
+    req['Content-Type'] = 'application/json'
+  end
+
+  def set_text_body(req, data)
+    req.body = data["message"]
+    req['Content-Type'] = 'text/plain'
+  end
+
+  def set_raw_body(req, data)
+    req.body = data.to_s
+    req['Content-Type'] = 'application/octet-stream'
+  end
+
+  def create_request(tag, time, record)
+    url = format_url(tag, time, record)
+    uri = URI.parse(url)
+    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.request_uri)
+    set_body(req, tag, time, record)
+    set_header(req, tag, time, record)
+    return req, uri
+  end
+
+  def http_opts(uri)
+      opts = {
+        :use_ssl => uri.scheme == 'https'
+      }
+      opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
+      opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file)
+      opts
+  end
+
+  def proxies
+    ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
+  end
+
+  def send_request(record)
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      log.info('Dropped request due to rate limiting')
+      return
+    end
+
+    _ = @adapter.post(@uri.path) do |request|
+      request.headers['Content-Type'] = 'application/json'
+      request.body = Yajl.dump(record)
+      request.options.timeout = 60
+      request.options.open_timeout = 60
+    end
+  end # end send_request
+
+  def handle_record(tag, time, record)
+    if @formatter_config
+      record = @formatter.format(tag, time, record)
+    end
+    # req, uri = create_request(tag, time, record)
+    send_request(record)
+  end
+
+  def prefer_buffered_processing
+    @buffered
+  end
+
+  def format(tag, time, record)
+    [time, record].to_msgpack
+  end
+
+  def formatted_to_msgpack_binary?
+    true
+  end
+
+  def multi_workers_ready?
+    true
+  end
+
+  def process(tag, es)
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+  end
+
+  def write(chunk)
+    tag = chunk.metadata.tag
+    @endpoint_url = extract_placeholders(@endpoint_url, chunk.metadata)
+    chunk.msgpack_each do |time, record|
+      handle_record(tag, time, record)
+    end
+  end
+end

data/lib/fluent/test/http_output_test.rb

@@ -0,0 +1,42 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+class Test::Unit::TestCase
+end
+
+require 'webrick'
+require 'webrick/https'
+
+# to handle POST/PUT/DELETE ...
+module WEBrick::HTTPServlet
+  class ProcHandler < AbstractServlet
+    alias do_POST   do_GET
+    alias do_PUT    do_GET
+    alias do_DELETE do_GET
+  end
+end

data/test/plugin/script/plugin/formatter_test.rb

@@ -0,0 +1,21 @@
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class TestFormatter < Formatter
+      Fluent::Plugin.register_formatter('test', self)
+
+      def configure(conf)
+        super
+      end
+
+      def format(tag, time, record)
+        output = {
+          "wrapped" => true,
+          "record" => record
+        }
+        output
+      end
+    end
+  end
+end