logstash-output-edge_http 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a076945788c4f08facd2cb2e61717e26e77431552c7ac846ce4bf8401fb156d4
+  data.tar.gz: 0cb3ebf49320c940bfb4ad29f65af21c59c12227b97de1f06391c4b3e2eddea8
+SHA512:
+  metadata.gz: 22d7e935ba950ae5a401524867efefa8f58803f99151af98ae82e439eefc1e45c901bfe0c1245a25a5c90932f16673fd1fb82a92e50f70e1b085614f951fe3d3
+  data.tar.gz: ac400ffed57010d459a6adcdde926a1ae379296c94c03cd77f0148fd0baa9b7f6783983451af7ab54517aab5cd6ba1467384de3f029c0a65f34e9ad2e9a32ed8

data/Gemfile

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "./logstash"
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-mixin-http_client', :path => "#{logstash_path}/vendor/bundle/jruby/2.6.0/gems/logstash-mixin-http_client"
+else
+  raise 'missing logstash vendoring'
+end
+
+gem "oauth2", "~> 2.0.9"
+gem "webmock", "~> 3.8"
+gem 'logstash-devutils', "~> 2"
+gem 'sinatra'
+gem 'webrick'

data/README.md

@@ -0,0 +1,88 @@
+# Contributing to Loki Logstash Output Plugin
+
+For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md).
+
+## Install dependencies
+
+First, make sure you have JDK version `8` or `11` installed and you have set the `JAVA_HOME` environment variable.
+
+You need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
+
+After setting up `rbenv`. Install JRuby
+
+```bash
+rbenv install jruby-9.2.10.0
+rbenv local jruby-9.2.10.0
+```
+
+Check that the environment is configured
+
+```bash
+ruby --version
+jruby 9.2.10
+```
+
+You should make sure you are running `jruby` and not `ruby`. If the command `ruby --version` still shows `ruby` and not `jruby`, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
+
+```bash
+export PATH="$HOME/.rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+```
+
+Then install bundler:
+
+```bash
+gem install bundler:2.1.4
+```
+
+Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
+
+## Build and test the plugin
+
+### Install required packages
+
+```bash
+git clone git@github.com:elastic/logstash.git
+cd logstash
+git checkout tags/v7.16.1
+export LOGSTASH_PATH="$(pwd)"
+export GEM_PATH="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
+export GEM_HOME="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
+./gradlew assemble
+cd ..
+ruby -S bundle /config set --local path "$LOGSTASH_PATH/vendor/bundle"
+ruby -S bundle install
+ruby -S bundle exec rake vendor
+```
+
+### Build the plugin
+
+```bash
+gem build logstash-output-loki.gemspec
+```
+
+### Test
+
+```bash
+ruby -S bundle exec rspec
+```
+
+Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
+
+```bash
+docker build -t logstash-loki ./
+docker run -v  $(pwd)/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
+bundle exec rspec
+```
+
+## Install plugin to local logstash
+
+```bash
+bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem
+```
+
+## Send sample event and check plugin is working
+
+```bash
+bin/logstash -f loki.conf
+```

data/lib/logstash/outputs/edge_http.rb

@@ -0,0 +1,414 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require "uri"
+require "logstash/plugin_mixins/http_client"
+require "zlib"
+require 'oauth2'
+
+class LogStash::Outputs::Http < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+
+  concurrency :shared
+
+  attr_accessor :is_batch
+
+  VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
+
+  RETRYABLE_MANTICORE_EXCEPTIONS = [
+    ::Manticore::Timeout,
+    ::Manticore::SocketException,
+    ::Manticore::ClientProtocolException,
+    ::Manticore::ResolutionFailure,
+    ::Manticore::SocketTimeout
+  ]
+
+  RETRYABLE_UNKNOWN_EXCEPTION_STRINGS = [
+    /Connection reset by peer/i,
+    /Read Timed out/i
+  ]
+
+  class PluginInternalQueueLeftoverError < StandardError; end
+
+  # This output lets you send events to a
+  # generic HTTP(S) endpoint
+  #
+  # This output will execute up to 'pool_max' requests in parallel for performance.
+  # Consider this when tuning this plugin for performance.
+  #
+  # Additionally, note that when parallel execution is used strict ordering of events is not
+  # guaranteed!
+  #
+  # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
+
+  config_name "http"
+
+  # URL to use
+  config :url, :validate => :string, :required => :true
+
+  # The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
+  config :http_method, :validate => VALID_METHODS, :required => :true
+
+  # Custom headers to use
+  # format is `headers => ["X-My-Header", "%{host}"]`
+  config :headers, :validate => :hash, :default => {}
+
+  # Content type
+  #
+  # If not specified, this defaults to the following:
+  #
+  # * if format is "json", "application/json"
+  # * if format is "form", "application/x-www-form-urlencoded"
+  config :content_type, :validate => :string
+
+  # Set this to false if you don't want this output to retry failed requests
+  config :retry_failed, :validate => :boolean, :default => true
+
+  # If encountered as response codes this plugin will retry these requests
+  config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
+
+  # If you would like to consider some non-2xx codes to be successes
+  # enumerate them here. Responses returning these codes will be considered successes
+  config :ignorable_codes, :validate => :number, :list => true
+
+  # This lets you choose the structure and parts of the event that are sent.
+  #
+  #
+  # For example:
+  # [source,ruby]
+  #    mapping => {"foo" => "%{host}"
+  #               "bar" => "%{type}"}
+  config :mapping, :validate => :hash
+
+  # Set the format of the http body.
+  #
+  # If form, then the body will be the mapping (or whole event) converted
+  # into a query parameter string, e.g. `foo=bar&baz=fizz...`
+  #
+  # If message, then the body will be the result of formatting the event according to message
+  #
+  # Otherwise, the event is sent as json.
+  config :format, :validate => ["json", "json_batch", "form", "message"], :default => "json"
+
+  # Set this to true if you want to enable gzip compression for your http requests
+  config :http_compression, :validate => :boolean, :default => false
+
+  config :message, :validate => :string
+
+  def register
+    @http_method = @http_method.to_sym
+
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
+
+    @requests = Array.new
+
+    if @content_type.nil?
+      case @format
+        when "form" ; @content_type = "application/x-www-form-urlencoded"
+        when "json" ; @content_type = "application/json"
+        when "json_batch" ; @content_type = "application/json"
+        when "message" ; @content_type = "text/plain"
+      end
+    end
+
+    @is_batch = @format == "json_batch"
+
+    @headers["Content-Type"] = @content_type
+
+    validate_format!
+
+    # Run named Timer as daemon thread
+    @timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
+  end # def register
+
+  def multi_receive(events)
+    return if events.empty?
+    send_events(events)
+  end
+
+  class RetryTimerTask < java.util.TimerTask
+    def initialize(pending, event, attempt)
+      @pending = pending
+      @event = event
+      @attempt = attempt
+      super()
+    end
+
+    def run
+      @pending << [@event, @attempt]
+    end
+  end
+
+  def log_retryable_response(response)
+    retry_msg = @retry_failed ? 'will retry' : "won't retry"
+    if (response.code == 429)
+      @logger.debug? && @logger.debug("Encountered a 429 response, #{retry_msg}. This is not serious, just flow control via HTTP")
+    else
+      @logger.warn("Encountered a retryable HTTP request in HTTP output, #{retry_msg}", :code => response.code, :body => response.body)
+    end
+  end
+
+  def log_error_response(response, url, event)
+    log_failure(
+              "Encountered non-2xx HTTP code #{response.code}",
+              :response_code => response.code,
+              :url => url,
+              :event => event
+            )
+  end
+
+  def send_events(events)
+    successes = java.util.concurrent.atomic.AtomicInteger.new(0)
+    failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
+    retries = java.util.concurrent.atomic.AtomicInteger.new(0)
+    event_count = @is_batch ? 1 : events.size
+
+    pending = Queue.new
+    if @is_batch
+      pending << [events, 0]
+    else
+      events.each {|e| pending << [e, 0]}
+    end
+
+    while popped = pending.pop
+      break if popped == :done
+
+      event, attempt = popped
+
+      raise PluginInternalQueueLeftoverError.new("Received pipeline shutdown request but http output has unfinished events. " \
+              "If persistent queue is enabled, events will be retried.") if attempt > 2 && pipeline_shutdown_requested?
+
+      action, event, attempt = send_event(event, attempt)
+      begin
+        action = :failure if action == :retry && !@retry_failed
+
+        case action
+        when :success
+          successes.incrementAndGet
+        when :retry
+          retries.incrementAndGet
+
+          next_attempt = attempt+1
+          sleep_for = sleep_for_attempt(next_attempt)
+          @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
+          timer_task = RetryTimerTask.new(pending, event, next_attempt)
+          @timer.schedule(timer_task, sleep_for*1000)
+        when :failure
+          failures.incrementAndGet
+        else
+          raise "Unknown action #{action}"
+        end
+
+        if action == :success || action == :failure
+          if successes.get+failures.get == event_count
+            pending << :done
+          end
+        end
+      rescue => e
+        # This should never happen unless there's a flat out bug in the code
+        @logger.error("Error sending HTTP Request",
+          :class => e.class.name,
+          :message => e.message,
+          :backtrace => e.backtrace)
+        failures.incrementAndGet
+        raise e
+      end
+    end
+  rescue => e
+    @logger.error("Error in http output loop",
+            :class => e.class.name,
+            :message => e.message,
+            :backtrace => e.backtrace)
+    raise e
+  end
+
+  def pipeline_shutdown_requested?
+    return super if defined?(super) # since LS 8.1.0
+    nil
+  end
+
+  def sleep_for_attempt(attempt)
+    sleep_for = attempt**2
+    sleep_for = sleep_for <= 60 ? sleep_for : 60
+    (sleep_for/2) + (rand(0..sleep_for)/2)
+  end
+
+  def send_event(event, attempt)
+    body = event_body(event)
+
+    # Send the request
+    url = @is_batch ? @url : event.sprintf(@url)
+    headers = @is_batch ? @headers : event_headers(event)
+
+    # Compress the body and add appropriate header
+    if @http_compression == true
+      headers["Content-Encoding"] = "gzip"
+      body = gzip(body)
+    end
+
+    # Create an async request
+    response = client.send(@http_method, url, :body => body, :headers => headers).call
+
+    if !response_success?(response)
+      if retryable_response?(response)
+        log_retryable_response(response)
+        return :retry, event, attempt
+      else
+        log_error_response(response, url, event)
+        return :failure, event, attempt
+      end
+    else
+      return :success, event, attempt
+    end
+
+  rescue => exception
+    will_retry = retryable_exception?(exception)
+    log_params = {
+      :url => url,
+      :method => @http_method,
+      :message => exception.message,
+      :class => exception.class,
+      :will_retry => will_retry
+    }
+    if @logger.debug?
+      # backtraces are big
+      log_params[:backtrace] = exception.backtrace
+      # headers can have sensitive data
+      log_params[:headers] = headers
+      # body can be big and may have sensitive data
+      log_params[:body] = body
+    end
+    log_failure("Could not fetch URL", log_params)
+
+    if will_retry
+      return :retry, event, attempt
+    else
+      return :failure, event, attempt
+    end
+  end
+
+  def close
+    @timer.cancel
+    client.close
+  end
+
+  private
+
+  def response_success?(response)
+    code = response.code
+    return true if @ignorable_codes && @ignorable_codes.include?(code)
+    return code >= 200 && code <= 299
+  end
+
+  def retryable_response?(response)
+    @retryable_codes && @retryable_codes.include?(response.code)
+  end
+
+  def retryable_exception?(exception)
+    retryable_manticore_exception?(exception) || retryable_unknown_exception?(exception)
+  end
+
+  def retryable_manticore_exception?(exception)
+    RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me)}
+  end
+
+  def retryable_unknown_exception?(exception)
+    exception.is_a?(::Manticore::UnknownException) &&
+        RETRYABLE_UNKNOWN_EXCEPTION_STRINGS.any? { |snippet| exception.message =~ snippet }
+  end
+
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error(message, opts)
+  end
+
+  # Format the HTTP body
+  def event_body(event)
+    # TODO: Create an HTTP post data codec, use that here
+    if @format == "json"
+      LogStash::Json.dump(map_event(event))
+    elsif @format == "message"
+      event.sprintf(@message)
+    elsif @format == "json_batch"
+      LogStash::Json.dump(event.map {|e| map_event(e) })
+    else
+      encode(map_event(event))
+    end
+  end
+
+  # gzip data
+  def gzip(data)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz)
+    z.write(data)
+    z.close
+    gz.string
+  end
+
+  def convert_mapping(mapping, event)
+    if mapping.is_a?(Hash)
+      mapping.reduce({}) do |acc, kv|
+        k, v = kv
+        acc[k] = convert_mapping(v, event)
+        acc
+      end
+    elsif mapping.is_a?(Array)
+      mapping.map { |elem| convert_mapping(elem, event) }
+    else
+      event.sprintf(mapping)
+    end
+  end
+
+  def map_event(event)
+    if @mapping
+      convert_mapping(@mapping, event)
+    else
+      event.to_hash
+    end
+  end
+
+  def event_headers(event)
+    custom_headers(event) || {}
+  end
+
+  def custom_headers(event)
+    return nil unless @headers
+
+    @headers.reduce({}) do |acc,kv|
+      k,v = kv
+      acc[k] = event.sprintf(v)
+      acc
+    end
+  end
+
+  #TODO Extract this to a codec
+  def encode(hash)
+    return hash.collect do |key, value|
+      CGI.escape(key) + "=" + CGI.escape(value.to_s)
+    end.join("&")
+  end
+
+
+  def validate_format!
+    if @format == "message"
+      if @message.nil?
+        raise "message must be set if message format is used"
+      end
+
+      if @content_type.nil?
+        raise "content_type must be set if message format is used"
+      end
+
+      unless @mapping.nil?
+        @logger.warn "mapping is not supported and will be ignored if message format is used"
+      end
+    end
+  end
+end

data/logstash-output-edge_http.gemspec

@@ -0,0 +1,32 @@
+Gem::Specification.new do |s|
+  s.name    = 'logstash-output-edge_http'
+  s.version = '1.0.0'
+  s.authors = ['Britto Prabhu']
+  s.email   = ['britto.prabhu@apmterminals.com']
+
+  s.summary       = 'Output plugin to ship logs to Loki server with oauth2 token'
+  s.description   = 'This Output plugin to ship logs to Loki server with oauth2 token and proxy support'
+  s.summary         = "Sends events to a generic HTTP or HTTPS endpoint with additional oauth2 token support"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program. This plugin supports oauth2 token."
+  s.homepage      = 'https://github.com/Maersk-Global/apmt-observability-deployment'
+  s.license       = 'Apache-2.0'
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "oauth2", "~> 2.0.9"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.2.0", "< 8.0.0"
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-edge_http
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Britto Prabhu
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.9
+  name: oauth2
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+  name: logstash-mixin-http_client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: sinatra
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program. This plugin supports oauth2 token.
+email:
+- britto.prabhu@apmterminals.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- lib/logstash/outputs/edge_http.rb
+- logstash-output-edge_http.gemspec
+homepage: https://github.com/Maersk-Global/apmt-observability-deployment
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.33
+signing_key:
+specification_version: 4
+summary: Sends events to a generic HTTP or HTTPS endpoint with additional oauth2 token
+  support
+test_files: []