fluent-plugin-out-kivera 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ce321115bd4b6032242005171702e374148359e62998ca3a4175ab6ae4aa8add
+  data.tar.gz: 5891e23d0f951706f17314e708fed643761901f60eafe6e046abddc5e9bed907
+SHA512:
+  metadata.gz: 63dc2922e9e0771369c125b3e33105656a9237703e8b5cd0cded8f7808481707af395f3416e8abedb26ffec891a3fee262b6290a841112120d6341c1cd671afa
+  data.tar.gz: f47e150d7a7d920d5b4ada2e5d53a758efd39c4b3da4a896fedf7f42cc6027340837e9c77c350fd216fecf80145e5dcf5ec1e01f9fa443c8ebb4e72c12246ad5

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/macos.yml

@@ -0,0 +1,26 @@
+name: Testing on macOS
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - macOS-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.gitignore

@@ -0,0 +1,8 @@
+*~
+\#*
+.\#*
+*.gem
+.bundle
+.ruby-version
+Gemfile.lock
+vendor

data/CHANGELOG.md

@@ -0,0 +1,80 @@
+# Changelog
+## 1.3.3
+* Revert x-ndjson format payload behavior
+
+## 1.3.2
+* Fix invalid x-ndjson payload format
+
+## 1.3.1
+* Support compression request
+
+## 1.3.0
+* Support all private key types
+* Recoverable error codes
+* Bulk request with x-ndjson
+
+## 1.2.0
+* Support mutual authentication
+
+## 1.1.7
+* Fix dependent Fluentd version
+
+## 1.1.6
+* Pass chunk directly info built-in placeholder instead of chunk.metadata
+
+## 1.1.5
+* Add :raw serializer
+
+## 1.1.4
+* Add custom formatter feature
+* Tweak Travis CI tasks
+
+## 1.1.3
+* Send query_string to endpoint_url
+
+## 1.1.2
+* Added custom headers feature
+
+## 1.1.1
+* Added plain text transport capability
+* Added specify cacert file for ssl verify
+
+## 1.1.0
+* Support for jwt token authentication
+
+## 1.0.1
+* Added endpoint_url placeholder support
+
+## 1.0.0
+* Use Fluentd v1 API
+
+## 0.2.0
+### Added
+* SSL is now supported if `endpoint_url` uses the `https` scheme (uses ruby-2.1 syntax internally)
+* New config: set `ssl_no_verify` to `true` to bypass SSL certificate verification.
+  Use at your own risk.
+### Changed
+* Fixed tests:
+  * Removed some warnings
+  * Fixed failing binary test to use UTF-8
+### Removed
+* Dropped support of Ruby 1.9-2.0
+
+## 0.1.4
+* #11 Updated Fluentd dependency to:  [">= 0.10.0", "< 2"]
+* #10 `password` is now marked as a [secret option](https://github.com/fluent/fluentd/pull/604)
+
+## 0.1.3
+* Added a new configuration option: `raise_on_error` (default: true)
+  * In order to let the plugin raise exceptions like it did in 0.1.1: keep using your configuration as-is
+  * In order to suppress all exceptions: add `raise_on_error false` to your configuration
+
+## 0.1.2
+* #6 Catch all `StandardError`s during HTTP request to prevent td-agent from freezing
+
+## 0.1.1
+* #2 Use yajl instead of json as json serializer
+* #1 Fix a bug where a nil HTTP response caused the plugin to stop working
+
+## 0.1.0
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-out-kivera.gemspec
+gemspec

data/ISSUE_TEMPLATE.md

@@ -0,0 +1,21 @@
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Provide example config and message
+
+#### Expected Behavior or What you need to ask
+
+...
+
+#### Using Fluentd and out_kivera plugin versions
+
+* OS version
+* Fluentd v0.12 or v0.14/v1.0
+  * paste result of ``fluentd --version`` or ``td-agent --version``
+* out_kivera plugin 1.x.y or 0.x.y
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
+* Bear Metal or Within Docker or Kubernetes or others? (optional)

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,58 @@
+# fluent-plugin-out-kivera, a plugin for [Fluentd](http://fluentd.org)
+
+A generic [fluentd][1] output plugin for sending Kivera proxy logs to the Kivera log ingestion service.
+
+## Configuration options
+
+You can specify the path to a config.json file which can contain the following parameters:
+
+  *config.json*
+  {
+    "client_id": "",
+    "client_secret": "",
+    "audience": "",
+    "auth0_domain": "",
+    "auth0_cert": "",
+    "auth0_cert_file": ""
+  }
+
+Note that parameter specified within the config_file will take precendence over parameters specified in your fluent.conf.
+Also note that the auth0_cert parameter will take precedence over the auth0_cert_file parameter.
+
+  *fluent.conf*
+    <match *>
+      @type kivera
+      endpoint_url              http://localhost.local/api/
+      config_file               /path/to/config_file.json
+      client_id                 abc123
+      client_secret             def456
+      audience                  http://api.kivera.io
+      auth0_domain              auth.nonp.kivera.io
+      auth0_cert                -----BEGIN CERTIFICATE-----...
+      auth0_cert_file           /path/to/auth0_cert_file
+      ssl_no_verify             false  # default: false
+      rate_limit_msec           100    # default: 0 = no rate limiting
+      raise_on_error            false  # default: true
+      recoverable_status_codes  503, 400 # default: 503
+      custom_headers            {"token":"arbitrary"} # default: nil
+      buffered                  true   # default: false. Switch non-buffered/buffered mode
+      bulk_request              true  # default: true. Send events as application/x-ndjson
+      compress_request          true  # default: false. Send compressed events
+    </match>
+
+## Usage notes
+
+If you'd like to retry failed requests, consider using [fluent-plugin-bufferize][3].
+Or, specify appropriate `recoverable_status_codes` parameter.
+
+To send events with bulk_request, you should specify `bulk_request` as `true`
+Note that when this parameter as `true`, Fluentd always send events as `application/x-ndjson`.
+Currently, `application/x-ndjson` is only supported MIME type for bulk_request.
+
+----
+
+Heavily based on [fluent-plugin-out-http][2]
+
+  [1]: http://fluentd.org/
+  [2]: https://github.com/fluent-plugins-nursery/fluent-plugin-out-http
+  [3]: https://github.com/sabottenda/fluent-plugin-bufferize

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-out-kivera.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-out-kivera"
+  gem.version       = "1.0.1"
+  gem.authors       = ["Tyler Matheson"]
+  gem.email         = ["support@kivera.io"]
+  gem.summary       = "Fluentd plugin for Kivera"
+  gem.description   = "A Fluentd output plugin for sending Kivera proxy logs to the Kivera log ingestion service"
+  gem.homepage      = "https://github.com/kivera-io/fluent-plugin-out-kivera"
+  gem.licenses      = ["Apache-2.0"]
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.required_ruby_version  = '>= 2.1.0'
+
+  gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
+  gem.add_runtime_dependency "fluentd", [">= 0.14.22", "< 2"]
+  gem.add_runtime_dependency "jwt", '~> 2.2'
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", ">= 3.1.0"
+end

data/lib/fluent/plugin/out_kivera.rb

@@ -0,0 +1,378 @@
+require 'net/http'
+require 'uri'
+require 'yajl'
+require 'fluent/plugin/output'
+require 'tempfile'
+require 'openssl'
+require 'zlib'
+require 'jwt'
+
+class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
+  Fluent::Plugin.register_output('kivera', self)
+
+  class RecoverableResponse < StandardError; end
+
+  helpers :compat_parameters, :formatter, :storage
+
+  DEFAULT_STORAGE_TYPE = "local"
+  DEFAULT_BUFFER_TYPE = "memory"
+  DEFAULT_FORMATTER = "json"
+  TOKEN_EXPIRY_OFFSET = 300
+
+  def initialize
+    super
+  end
+
+  # Endpoint URL ex. http://localhost.local/api/
+  config_param :endpoint_url, :string
+
+  # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
+  config_param :ssl_no_verify, :bool, :default => false
+
+  # Simple rate limiting: ignore any records within `rate_limit_msec`
+  # since the last one.
+  config_param :rate_limit_msec, :integer, :default => 0
+
+  # Raise errors that were rescued during HTTP requests?
+  config_param :raise_on_error, :bool, :default => true
+
+  # Specify recoverable error codes
+  config_param :recoverable_status_codes, :array, value_type: :integer, default: [503]
+
+  # kivera proxy client config file
+  config_param :config_file, :string, default: ""
+
+  # kivera proxy client id
+  config_param :client_id, :string, default: ""
+
+  # kivera proxy client secret
+  config_param :client_secret, :string, default: ""
+
+  # kivera audience api
+  config_param :audience, :string, default: ""
+
+  # explicit Kivera auth0 certificate as a string
+  config_param :auth0_cert, :string, default: ""
+
+  # Kivera auth0 certificate file
+  config_param :auth0_cert_file, :string, default: ""
+
+  # Kivera auth0 domain
+  config_param :auth0_domain, :string, default: ""
+
+  # custom headers
+  config_param :custom_headers, :hash, :default => nil
+
+  # Switch non-buffered/buffered plugin
+  config_param :bulk_request, :bool, :default => true
+  config_param :buffered, :bool, :default => false
+  # Compress with gzip except for form serializer
+  config_param :compress_request, :bool, :default => false
+
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+    config_set_default :chunk_keys, ['tag']
+  end
+
+  config_section :format do
+    config_set_default :@type, DEFAULT_FORMATTER
+  end
+
+  def configure(conf)
+    compat_parameters_convert(conf, :buffer, :formatter)
+    super
+
+    @ssl_verify_mode = if @ssl_no_verify
+                         OpenSSL::SSL::VERIFY_NONE
+                       else
+                         OpenSSL::SSL::VERIFY_PEER
+                       end
+
+    @last_request_time = nil
+    raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered
+
+    if @formatter_config = conf.elements('format').first
+      @formatter = formatter_create
+    end
+
+    if @bulk_request
+      class << self
+        alias_method :format, :bulk_request_format
+      end
+      @formatter = formatter_create(type: :json)
+      @serializer = :x_ndjson # secret settings for bulk_request
+    else
+      class << self
+        alias_method :format, :split_request_format
+      end
+      @serializer = :json
+    end
+
+    # Create local storage for persisting JWT token
+    config = conf.elements(name: 'storage').first
+    @storage = storage_create(usage: 'jwt_token', conf: config, default_type: 'local')
+
+    if ! @config_file.empty?
+      creds =  File.read(@config_file)
+      parsed = Yajl::Parser.new.parse(StringIO.new(creds))
+      @client_id = parsed.fetch("client_id", @client_id)
+      @client_secret = parsed.fetch("client_secret", @client_secret)
+      @audience = parsed.fetch("audience", @audience)
+      @auth0_cert = parsed.fetch("auth0_cert", @auth0_cert)
+      @auth0_cert_file = parsed.fetch("auth0_cert", @auth0_cert_file)
+      @auth0_domain = parsed.fetch("auth0_domain", @auth0_domain)
+    end
+
+    if @auth0_cert.empty? && ! @auth0_cert_file.empty?
+      @auth0_cert = File.read(@auth0_cert_file)
+    end
+
+    if @client_id.empty? && 
+        @client_secret.empty? && 
+        @audience.empty? && 
+        @auth0_cert.empty? && 
+        @auth0_domain.empty?
+      params = "client_id, client_secret, audience, auth0_cert and auth0_domain"
+      log.error "Missing configuration. Either specify a config_file or set the #{params} parameters"
+    end
+
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url(tag, time, record)
+    @endpoint_url
+  end
+
+  def set_body(req, tag, time, record)
+    if @serializer == :json
+      set_json_body(req, record)
+    elsif @serializer == :x_ndjson
+      set_bulk_body(req, record)
+    else
+      req.set_form_data(record)
+    end
+    req
+  end
+
+  def set_header(req, tag, time, record)
+    if @custom_headers
+      @custom_headers.each do |k,v|
+        req[k] = v
+      end
+      req
+    else
+      req
+    end
+  end
+
+  def refresh_jwt_token
+    if @storage.get(:jwt_token)
+      if token_expired
+        @storage.put(:jwt_token, new_jwt_token)
+      end
+    else
+      @storage.put(:jwt_token, new_jwt_token)
+    end
+  end
+
+  def token_expired
+    x509 = OpenSSL::X509::Certificate.new(@auth0_cert)
+    begin
+      decoded_token = JWT.decode @storage.get(:jwt_token), x509.public_key, true, { algorithm: 'RS256' }
+    rescue => e
+        log.info 'JWT token expired'
+        return true
+    else
+      if decoded_token[0]['exp'] - Time.now.to_f < TOKEN_EXPIRY_OFFSET
+        log.info 'JWT token about to expire'
+        return true
+      end
+      return false
+    end
+  end
+
+  def new_jwt_token
+    url = "https://" + @auth0_domain + "/oauth/token"
+    uri = URI.parse(url)
+    req = Net::HTTP::Post.new(uri.to_s)
+    payload = { 
+      "client_id" =>      @client_id,
+			"client_secret" =>  @client_secret,
+			"audience" =>       @audience,
+      "grant_type" =>     "client_credentials"
+    }
+    set_json_body(req, payload)
+    res = https(uri).request(req)
+    case res
+    when Net::HTTPSuccess then
+      parsed = Yajl::Parser.new.parse(StringIO.new(res.body))
+      log.info 'Generated new JWT token'
+      parsed['access_token']
+    else
+      log.warn "Failed to get token for client #{@client_id}"
+    end
+  end
+
+  def https(uri)
+    Net::HTTP.new(uri.host, uri.port).tap { |http|
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    }
+  end
+
+  def compress_body(req, data)
+    return unless @compress_request
+    gz = Zlib::GzipWriter.new(StringIO.new)
+    gz << data
+
+    req['Content-Encoding'] = "gzip"
+    req.body = gz.close.string
+  end
+
+  def set_json_body(req, data)
+    req.body = Yajl.dump(data)
+    req['Content-Type'] = 'application/json'
+    compress_body(req, req.body)
+  end
+
+  def set_bulk_body(req, data)
+    req.body = data.to_s
+    req['Content-Type'] = 'application/x-ndjson'
+    compress_body(req, req.body)
+  end
+
+  def set_jwt_auth(req)
+    refresh_jwt_token
+    req['Authorization'] = "Bearer #{@storage.get(:jwt_token)}"
+  end
+
+  def create_request(tag, time, record)
+    url = format_url(tag, time, record)
+    uri = URI.parse(url)
+    req = Net::HTTP::Put.new(uri.request_uri)
+    set_body(req, tag, time, record)
+    set_header(req, tag, time, record)
+    set_jwt_auth(req)
+    return req, uri
+  end
+
+  def http_opts(uri)
+    opts = {
+      :use_ssl => uri.scheme == 'https'
+    }
+    opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
+    opts
+  end
+
+  def proxies
+    ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
+  end
+
+  def send_request(req, uri)
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      log.info('Dropped request due to rate limiting')
+      return
+    end
+
+    res = nil
+
+    begin
+
+      if proxy = proxies
+        proxy_uri = URI.parse(proxy)
+
+        res = Net::HTTP.start(uri.host, uri.port,
+                              proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password,
+                              **http_opts(uri)) {|http| http.request(req) }
+      else
+        res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
+      end
+
+    rescue => e # rescue all StandardErrors
+      # server didn't respond
+      log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
+      raise e if @raise_on_error
+    else
+       unless res and res.is_a?(Net::HTTPSuccess)
+          res_summary = if res
+                           "#{res.code} #{res.message} #{res.body}"
+                        else
+                           "res=nil"
+                        end
+          if @recoverable_status_codes.include?(res.code.to_i)
+            raise RecoverableResponse, res_summary
+          else
+            log.warn "failed to #{req.method} #{uri} (#{res_summary})"
+          end
+       end #end unless
+    end # end begin
+  end # end send_request
+
+  def handle_record(tag, time, record)
+    if @formatter_config
+      record = @formatter.format(tag, time, record)
+    end
+    req, uri = create_request(tag, time, record)
+    send_request(req, uri)
+  end
+
+  def handle_records(tag, time, chunk)
+    req, uri = create_request(tag, time, chunk.read)
+    send_request(req, uri)
+  end
+
+  def prefer_buffered_processing
+    @buffered
+  end
+
+  def format(tag, time, record)
+    # For safety.
+  end
+
+  def split_request_format(tag, time, record)
+    [time, record].to_msgpack
+  end
+
+  def bulk_request_format(tag, time, record)
+    @formatter.format(tag, time, record)
+  end
+
+  def formatted_to_msgpack_binary?
+    if @bulk_request
+      false
+    else
+      true
+    end
+  end
+
+  def multi_workers_ready?
+    true
+  end
+
+  def process(tag, es)
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+  end
+
+  def write(chunk)
+    tag = chunk.metadata.tag
+    @endpoint_url = extract_placeholders(@endpoint_url, chunk)
+    if @bulk_request
+      time = Fluent::Engine.now
+      handle_records(tag, time, chunk)
+    else
+      chunk.msgpack_each do |time, record|
+        handle_record(tag, time, record)
+      end
+    end
+  end
+end