logstash-output-dynatrace 0.6.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9281b9e498b9986e45d25eaadc74358f95547551baf97132483533997edf134d
-  data.tar.gz: 27bc3095ec54575becfaf75eb2b7bca7d186f95b2ab239713e83f8b9f58b363b
+  metadata.gz: 81b4a8c2170a81fb234ff285214b21e531d8b33a304ccbe25b643a10d05a7c28
+  data.tar.gz: 70e9196d107abd6655ea8394fac223713bd8120c5305c5f22075d402f897d65d
 SHA512:
-  metadata.gz: f4863a8fae10fe23afc85d4a6e31290ba3c0628d974c57c8cec6306adc2ba1da5c73905c7597c0954cb095ea1a24c6e182d028d8d7289f05951f77c214255b48
-  data.tar.gz: 53281ec7c6a7a8f6ac5dffae9cde5babe61d0a1b962d5c903e7e60975bb00d0528efc57c21c3b35d6f9bfc18b84c1549221c18dbe8ca3ae75e8cde764a94d7b7
+  metadata.gz: 6c67bc08595859467e6c32da87b60e0da67e99f3df4eb8b59d1ecedd8b96676429fe908b4e0678ea4f74736445fab3880a25461d439ef8fef3ce6100b9bc7ada
+  data.tar.gz: c0c78831f7436a02ae3faafab913a725af81d9bf8a14111b7de5d4e46bac3b473f22b325ef27280f5bb4262279a86dadf6ec2b19250b89819e90b7e7f5b8c32b

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.7.1
+
+- Fix an error where max payload size was calculated using character count instead of bytes
+
+## 0.7.0
+
+- Add new development dependency `rackup` for logstash 8.x compatibility
+- Enable `compression` configuration to compress payloads using `gzip`
+- Document `proxy` configuration
+
 ## 0.6.0
 
 - Disable cookie processing by default

data/README.md

@@ -15,6 +15,7 @@
   - [`ingest_endpoint_url`](#ingest_endpoint_url)
   - [`api_key`](#api_key)
   - [`ssl_verify_none`](#ssl_verify_none)
+  - [`proxy`](#proxy)
   - [`enable_metric`](#enable_metric)
   - [`id`](#id)
 - [Troubleshooting issues](#troubleshooting-issues)
@@ -62,6 +63,7 @@ The following configuration options are supported by the Dynatrace output plugin
 | [`ingest_endpoint_url`](#ingest_endpoint_url) | [String](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string)   | Yes      |
 | [`api_key`](#api_key)                         | [String](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string)   | Yes      |
 | [`ssl_verify_none`](#ssl_verify_none)         | [Boolean](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#boolean) | No       |
+| [`http_compression`](#http_compression)         | [Boolean](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#boolean) | No       |
 
 
 ### Common Options
@@ -70,7 +72,8 @@ The following configuration options are supported by all output plugins:
 
 | Setting                           | Input type                                                                                            | Required |
 | --------------------------------- | ----------------------------------------------------------------------------------------------------- | -------- |
-| [`codec`](#codec)                 | [Codec](https://www.elastic.co/guide/en/logstash/7.16/configuration-file-structure.html#codec)        | No       |
+| [`proxy`](#proxy)                 | [String](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string) or [Hash](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#hash) | No |
+|  `codec`                          | [Codec](https://www.elastic.co/guide/en/logstash/7.16/configuration-file-structure.html#codec)        | No       |
 | [`enable_metric`](#enable_metric) | [Boolean](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#boolean) | No       |
 | [`id`](#id)                       | [String](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string)   | No       |
 
@@ -106,9 +109,26 @@ This option may be required if you are using a self-signed certificate, an expir
 > NOTE: Starting in plugin version `0.5.0`, this option has no effect in versions of Logstash older than `8.1.0`.
 > If this functionality is required, it is recommended to update Logstash or stay at plugin version `0.4.x` or older.
 
+### `proxy`
+
+* Value type is [string](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string) or [hash](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#hash)
+* Optional
+* No default value
+* Introduced in `logstash-output-dynatrace` version `0.5.0` (not available in older versions)
+
+This setting configures an HTTP proxy to route exported data through.
+
+The supported configuration options and their syntax are the same as for the [proxy option](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-http.html#plugins-outputs-http-proxy) in the HTTP output plugin:
+
+1. `proxy => http://example.org:1234`
+2. `proxy => { host => "example.org" port => 80 scheme => 'http' user => 'username@host' password => 'password' }`
+3. `proxy => { url =>  'http://example.org:1234' user => 'username@host' password => 'password' }`
+
+Note that [Hashes](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#hash) in the Logstash configuration file format use spaces as delimiters between entries, not commas.
+
 ### `enable_metric`
 
- * Value type is [boolean](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#boolean)
+* Value type is [boolean](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#boolean)
 * Default value is true
 
 Disable or enable metric logging for this specific plugin instance. By default we record all the metrics we can, but you can disable metrics collection for a specific plugin.
@@ -116,7 +136,7 @@ Disable or enable metric logging for this specific plugin instance. By default w
 ### `id`
 
 * Value type is string
-* There is no default value for this setting.
+* No default value
 
 Add a unique ID to the plugin configuration. If no ID is specified, Logstash will generate one. It is strongly recommended to set this ID in your configuration. This is particularly useful when you have two or more plugins of the same type. For example, if you have 2 dynatrace outputs. Adding a named ID in this case will help in monitoring Logstash when using the monitoring APIs.
 
@@ -128,6 +148,15 @@ output {
 }
 ```
 
+### `http_compression`
+
+* Value type is [boolean](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#boolean)
+* Optional
+* Default value is `false`
+
+Setting `http_compression` to `true` causes the output plugin to compress all requests to the Dynatrace API using `gzip`.
+This can result in a reduction in size and transmission time of network requests at the expense of some additional CPU and memory consumption.
+
 ## Troubleshooting issues
 
 When troubleshooting, always try to reduce the configuration as much as possible.

data/lib/logstash/outputs/dynatrace.rb

@@ -21,6 +21,7 @@ require 'logstash/version'
 require 'dynatrace/version'
 require 'uri'
 require 'logstash/plugin_mixins/http_client'
+require 'zlib'
 
 # These constants came from the http plugin config but we don't want them configurable
 # If encountered as response codes this plugin will retry these requests
@@ -80,6 +81,9 @@ module LogStash
       # Disable cookie support. Overridden default value from LogStash::PluginMixins::HttpClient
       config :cookies, :validate => :boolean, :default => false
 
+      # Set this to true if you want to enable gzip compression for your http requests
+      config :http_compression, :validate => :boolean, :default => false
+
       def register
         # ssl_verification_mode config is from mixin but ssl_verify_none is our documented config
         @ssl_verification_mode = 'none' if @ssl_verify_none
@@ -118,11 +122,11 @@ module LogStash
 
         def offer(serialized_event)
           # 2 square brackets, the length of all previously serialized strings, commas, and the current event size
-          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.length
+          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.bytesize
           return false if batch_size_bytes > @max_batch_size
 
           @serialized_events.push(serialized_event)
-          @batch_events_size += serialized_event.length
+          @batch_events_size += serialized_event.bytesize
           true
         end
 
@@ -174,8 +178,8 @@ module LogStash
 
         events.each do |event|
           serialized_event = LogStash::Json.dump(event.to_hash)
-          if serialized_event.length > @max_payload_size
-            log_params = { size: serialized_event.length }
+          if serialized_event.bytesize > @max_payload_size
+            log_params = { size: serialized_event.bytesize }
             log_params[:body] = serialized_event if @debug_include_body
             log_warning('Event larger than max_payload_size dropped', log_params)
             next
@@ -253,6 +257,12 @@ module LogStash
       def send_event(event, attempt)
         headers = make_headers
 
+        # Compress the body and add appropriate header
+        if @http_compression == true
+          headers["Content-Encoding"] = "gzip"
+          event = gzip(event)
+        end
+
         # Create an async request
         response = client.post(ingest_endpoint_url, body: event, headers: headers)
 
@@ -331,6 +341,16 @@ module LogStash
       def log_warning(message, opts)
         @logger.warn(message, opts)
       end
+
+      # gzip data
+      def gzip(data)
+        gz = StringIO.new
+        gz.set_encoding("BINARY")
+        z = Zlib::GzipWriter.new(gz)
+        z.write(data)
+        z.close
+        gz.string
+      end
     end
   end
 end

data/logstash-output-dynatrace.gemspec

@@ -52,4 +52,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rubocop', '1.9.1'
   s.add_development_dependency 'rubocop-rake', '0.5.1'
+
+  # Pin to avoid using new Fiber-based implementation that breaks tests here
+  s.add_development_dependency 'rackup', "< 2.1.0"
 end

data/spec/outputs/dynatrace_spec.rb

@@ -232,17 +232,38 @@ describe LogStash::Outputs::Dynatrace do
     end
   end
 
-  context 'max_payload_size 2MB' do
-    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => 2_000_000 } }
+  context 'max_payload_size 500' do
+    let(:max_payload_size) { 500 }
+    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => max_payload_size } }
     subject { LogStash::Outputs::Dynatrace.new(config) }
 
     before do
       allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
-      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 1_250_000 }) })
     end
 
-    it 'should split the chunk into multiple requests' do
-      expect(subject).to have_received(:send_event).exactly(2).times
+    it 'should send 2 400B messages in multiple requests' do
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 400 }) })
+      expect(subject).to have_received(:send_event).exactly(2).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
+    end
+
+    it 'should send 2 100B messages in a single request' do
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 100 }) })
+      expect(subject).to have_received(:send_event).exactly(1).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
+    end
+
+    it 'should drop messages larger than max_payload_size' do
+      subject.multi_receive([
+        LogStash::Event.new({ 'event' => '🤣' * 400 }),
+        LogStash::Event.new({ 'event' => 'n' * 600 }),
+        LogStash::Event.new({ 'event' => 'n' * 400 }),
+      ])
+      expect(subject).to have_received(:send_event).exactly(1).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
     end
   end
 
@@ -347,7 +368,7 @@ describe LogStash::Outputs::Dynatrace do
       end
 
       let(:last_request) { TestApp.last_request }
-      let(:body) { last_request.body.read }
+      let(:body) { read_last_request_body(last_request) }
       let(:content_type) { last_request.env['CONTENT_TYPE'] }
 
       it 'should receive the request' do
@@ -409,11 +430,11 @@ describe LogStash::Outputs::Dynatrace do
     include_examples('integration tests')
   end
 
-  # describe "integration test with gzip compression" do
-  #   include_examples("integration tests") do
-  #     let(:base_config) { { "http_compression" => true } }
-  #   end
-  # end
+  describe "integration test with gzip compression" do
+    include_examples("integration tests") do
+      let(:base_config) { { "http_compression" => true } }
+    end
+  end
 
   describe 'retryable error in termination' do
     let(:ingest_endpoint_url) { "http://localhost:#{port - 1}/invalid" }
@@ -437,10 +458,10 @@ describe LogStash::Outputs::Dynatrace do
 end
 
 RSpec.describe LogStash::Outputs::Dynatrace do # different block as we're starting web server with TLS
-  @@default_server_settings = TestApp.server_settings.dup
+  let(:default_server_settings) { TestApp.server_settings.dup }
 
   before do
-    TestApp.server_settings = @@default_server_settings.merge(webrick_config)
+    TestApp.server_settings = default_server_settings.merge(webrick_config)
 
     TestApp.last_request = nil
 
@@ -465,7 +486,7 @@ RSpec.describe LogStash::Outputs::Dynatrace do # different block as we're starti
     rescue StandardError
       nil
     end
-    TestApp.server_settings = @@default_server_settings
+    TestApp.server_settings = default_server_settings
   end
 
   let(:ssl_cert_host) { 'localhost' }
@@ -483,7 +504,7 @@ RSpec.describe LogStash::Outputs::Dynatrace do # different block as we're starti
   after  { subject.close }
 
   let(:last_request) { TestApp.last_request }
-  let(:last_request_body) { last_request.body.read }
+  let(:last_request_body) { read_last_request_body(last_request) }
 
   let(:event) { LogStash::Event.new('message' => 'hello!') }
 
@@ -550,3 +571,10 @@ RSpec.describe LogStash::Outputs::Dynatrace do # different block as we're starti
     end
   end
 end
+
+# Pre-emptively rewind the retrieval of `last_request.body` - for form based endpoints, the body
+# is placed in params, and body is empty, requiring a `rewind` for the body to be available for comparison
+def read_last_request_body(last_request)
+  last_request.body.rewind
+  last_request.body.read
+end

data/spec/spec_helper.rb

@@ -23,6 +23,8 @@ require 'webrick'
 require 'webrick/https'
 require 'openssl'
 
+require "supports/compressed_requests"
+
 PORT = rand(65_535 - 1024) + 1025
 
 module LogStash
@@ -52,6 +54,9 @@ end
 # == Sinatra has ended his set (crowd applauds)
 #
 class TestApp < Sinatra::Base
+  # on the fly uncompress gzip content
+  use CompressedRequests
+
   set :environment, :production
   set :sessions, false
 

data/spec/supports/compressed_requests.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+#
+# based on relistan's rack handler
+# out of the box rack only gives use the rack deflater handler to return compressed
+# response, this gist offer the inverse and should work on all rack based app like sinatra or rails.
+#
+# original source: https://gist.github.com/relistan/2109707
+require "zlib"
+
+class CompressedRequests
+  def initialize(app)
+    @app = app
+  end
+
+  def method_handled?(env)
+    ['POST', 'PUT'].include? env['REQUEST_METHOD']
+  end
+
+  def encoding_handled?(env)
+    ['gzip', 'deflate'].include? env['HTTP_CONTENT_ENCODING']
+  end
+
+  def call(env)
+    if method_handled?(env) && encoding_handled?(env)
+      extracted = decode(env['rack.input'], env['HTTP_CONTENT_ENCODING'])
+
+      env.delete('HTTP_CONTENT_ENCODING')
+      env['CONTENT_LENGTH'] = extracted.bytesize
+      env['rack.input'] = StringIO.new(extracted)
+    end
+
+    status, headers, response = @app.call(env)
+    return [status, headers, response]
+  end
+
+  def decode(input, content_encoding)
+    case content_encoding
+      when 'gzip' then Zlib::GzipReader.new(input).read
+      when 'deflate' then Zlib::Inflate.inflate(input.read)
+    end
+  end
+end

data/version.yaml

@@ -1 +1 @@
-logstash-output-dynatrace: '0.6.0'
+logstash-output-dynatrace: '0.7.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-dynatrace
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Dynatrace Open Source Engineering
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-27 00:00:00.000000000 Z
+date: 2025-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json
@@ -148,6 +148,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.5.1
+- !ruby/object:Gem::Dependency
+  name: rackup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 description: |2
       This gem is a Logstash plugin required to be installed on top of the Logstash
       core pipeline using `$LS_HOME/bin/logstash-plugin install logstash-output-dynatrace`.
@@ -168,6 +182,7 @@ files:
 - logstash-output-dynatrace.gemspec
 - spec/outputs/dynatrace_spec.rb
 - spec/spec_helper.rb
+- spec/supports/compressed_requests.rb
 - version.yaml
 homepage: https://github.com/dynatrace-oss/logstash-output-dynatrace
 licenses:
@@ -175,7 +190,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -191,10 +206,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.6
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A logstash output plugin for sending logs to the Dynatrace Generic log ingest
   API v2
 test_files:
 - spec/outputs/dynatrace_spec.rb
 - spec/spec_helper.rb
+- spec/supports/compressed_requests.rb