logstash-output-datadog_logs 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 474cfba6967ab1b3109dc47be8ed80a292957f09286015fd9506d44d9d72548c
-  data.tar.gz: 17f42ddff7a84cacc24795468ba105a0622f7f5554324d717bf4d5f61618dd6b
+  metadata.gz: 6f81e4d1f9e76634f0ee83cc649ad18a9ee9defc51c54c0cd630e8c9bae69d53
+  data.tar.gz: e15dfeba8935d842bc9db8ed8c1a8f166165376c4b0bc02d6feafca166313432
 SHA512:
-  metadata.gz: f1285fe2fc85df206d20315fd0f549190d856035410bc86267eac3e6da169bbd33b8fa2bc08dd9c7f6612189239ca44b0896ae6390b858d0ecaebfd8ae5a724b
-  data.tar.gz: e387e7e3a92cd452fa4fa57cfe8b1c7fb26093436aada5d8a7e6a368bb302bea789f8e617e376d6ee1a7a8b900c3ccf334705656d2f9540a00151bc7ae00d7ba
+  metadata.gz: 46a621add6073375b653ce158b67d9aec0c1e6544c7d69ce7740b1b7930cb95a28525765048ebcc66975f9fa99b755f10fee32127a14b842ae0be80f034ebf11
+  data.tar.gz: e9276dc886a503ed450a20ce986861835496b5f8f1d991389afd5ece5a0933b18ae2cad57d211a74d068b51024c560eabb7a859ba17ebd6b6f9ca44fe71fb2a2

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.0
+  - Enable HTTP forwarding for logs
+  - Provide an option to disable SSL hostname verification for HTTPS
+
 ## 0.3.1
   - Make sure that we can disable retries
 

data/README.md

@@ -3,6 +3,10 @@
 
 DatadogLogs lets you send logs to Datadog based on LogStash events.
 
+## Requirements
+
+The plugin relies upon the `zlib` library for compressing data.
+
 ## How to install it?
 
 ```bash
@@ -12,13 +16,77 @@ logstash-plugin install logstash-output-datadog_logs
 
 ## How to use it?
 
-Configure `datadog_logs` plugin with your Datadog API key:
+The `datadog_logs` plugin is configured by default to send logs to a US endpoint over an SSL-encrypted HTTP connection.
+The logs are by default batched and compressed.
+ 
+Configure the plugin with your Datadog API key:
+
+```
+output {
+    datadog_logs {
+        api_key => "<DATADOG_API_KEY>"
+    }
+}
+```
+
+To enable TCP forwarding, configure your forwarder with:
+
+```
+output {
+    datadog_logs {
+        api_key => "<DATADOG_API_KEY>"
+        host => "tcp-intake.logs.datadoghq.com"
+        port => 10516
+        use_http => false
+    }
+}
+```
+
+To send logs to the Datadog's EU HTTP endpoint, override the default `host`
 
 ```
 output {
     datadog_logs {
         api_key => "<DATADOG_API_KEY>"
+        host => "http-intake.logs.datadoghq.eu"
+    }
+}
+```
+
+### Configuration properties
+
+|  Property   |  Description                                                             |  Default value |
+|-------------|--------------------------------------------------------------------------|----------------|
+| **api_key** | The API key of your Datadog platform | nil |
+| **host** | Proxy endpoint when logs are not directly forwarded to Datadog | intake.logs.datadoghq.com |
+| **port** | Proxy port when logs are not directly forwarded to Datadog | 443 |
+| **use_ssl** | If true, the agent initializes a secure connection to Datadog. Ensure to update the port if you disable it. | true |
+| **max_retries** | The number of retries before the output plugin stops | 5 |
+| **max_backoff** | The maximum time waited between each retry in seconds | 30 |
+| **use_http** | Enable HTTP forwarding. If you disable it, make sure to update the port to 10516 if use_ssl is enabled or 10514 otherwise.  | true |
+| **use_compression** | Enable log compression for HTTP | true |
+| **compression_level** | Set the log compression level for HTTP (1 to 9, 9 being the best ratio) | 6 |
+| **no_ssl_validation** | Disable SSL validation (useful for proxy forwarding) | false |
+
+
+
+For additional options, see the [Datadog endpoint documentation](https://docs.datadoghq.com/logs/?tab=eusite#datadog-logs-endpoints)
+
+## Add metadata to your logs
+
+In order to get the best use out of your logs in Datadog, it is important to have the proper metadata associated with them (including hostname, service and source). 
+To add those to your logs, add them into your logs with a mutate filter:
+
+```
+filter {
+  mutate {
+    add_field => {
+      "host"     => "<HOST>"
+      "service"  => "<SERVICE>"
+      "ddsource" => "<MY_SOURCE_VALUE>"
+      "ddtags"   => "<KEY1:VALUE1>,<KEY2:VALUE2>"
     }
+  }
 }
 ```
 

data/lib/logstash/outputs/datadog_logs.rb

@@ -6,67 +6,249 @@
 # encoding: utf-8
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "zlib"
+
 
 # DatadogLogs lets you send logs to Datadog
 # based on LogStash events.
 class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
 
+  # Respect limit documented at https://docs.datadoghq.com/api/?lang=bash#logs
+  DD_MAX_BATCH_LENGTH = 500
+  DD_MAX_BATCH_SIZE = 5000000
+  DD_TRUNCATION_SUFFIX = "...TRUNCATED..."
+
   config_name "datadog_logs"
 
   default :codec, "json"
 
   # Datadog configuration parameters
-  config :api_key,     :validate => :string,  :required => true
-  config :host,        :validate => :string,  :required => true, :default  => 'intake.logs.datadoghq.com'
-  config :port,        :validate => :number,  :required => true, :default  => 10516
-  config :use_ssl,     :validate => :boolean, :required => true, :default  => true
-  config :max_backoff, :validate => :number,  :required => true, :default  => 30
-  config :max_retries, :validate => :number,  :required => true, :default  => 5
+  config :api_key, :validate => :string, :required => true
+  config :host, :validate => :string, :required => true, :default => "http-intake.logs.datadoghq.com"
+  config :port, :validate => :number, :required => true, :default => 443
+  config :use_ssl, :validate => :boolean, :required => true, :default => true
+  config :max_backoff, :validate => :number, :required => true, :default => 30
+  config :max_retries, :validate => :number, :required => true, :default => 5
+  config :use_http, :validate => :boolean, :required => false, :default => true
+  config :use_compression, :validate => :boolean, :required => false, :default => true
+  config :compression_level, :validate => :number, :required => false, :default => 6
+  config :no_ssl_validation, :validate => :boolean, :required => false, :default => false
 
+  # Register the plugin to logstash
   public
   def register
-    require "socket"
-    client = nil
-    @codec.on_event do |event, payload|
-      message = "#{@api_key} #{payload}\n"
-      retries = 0
+    @client = new_client(@logger, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @port, @use_compression)
+  end
+
+  # Logstash shutdown hook
+  def close
+    @client.close
+  end
+
+  # Entry point of the plugin, receiving a set of Logstash events
+  public
+  def multi_receive(events)
+    return if events.empty?
+    encoded_events = @codec.multi_encode(events)
+    if @use_http
+      batches = batch_http_events(encoded_events, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
+      batches.each do |batched_event|
+        process_encoded_payload(format_http_event_batch(batched_event))
+      end
+    else
+      encoded_events.each do |encoded_event|
+        process_encoded_payload(format_tcp_event(encoded_event.last, @api_key, DD_MAX_BATCH_SIZE))
+      end
+    end
+  end
+
+  # Process and send each encoded payload
+  def process_encoded_payload(payload)
+    if @use_compression and @use_http
+      payload = gzip_compress(payload, @compression_level)
+    end
+    @client.send_retries(payload, @max_retries, @max_backoff)
+  end
+
+  # Format TCP event
+  def format_tcp_event(payload, api_key, max_request_size)
+    formatted_payload = "#{api_key} #{payload}"
+    if (formatted_payload.bytesize > max_request_size)
+      return truncate(formatted_payload, max_request_size)
+    end
+    formatted_payload
+  end
+
+  # Format HTTP events
+  def format_http_event_batch(batched_events)
+    "[#{batched_events.join(',')}]"
+  end
+
+  # Group HTTP events in batches
+  def batch_http_events(encoded_events, max_batch_length, max_request_size)
+    batches = []
+    current_batch = []
+    current_batch_size = 0
+    encoded_events.each_with_index do |event, i|
+      encoded_event = event.last
+      current_event_size = encoded_event.bytesize
+      # If this unique log size is bigger than the request size, truncate it
+      if current_event_size > max_request_size
+        encoded_event = truncate(encoded_event, max_request_size)
+        current_event_size = encoded_event.bytesize
+      end
+
+      if (i > 0 and i % max_batch_length == 0) or (current_batch_size + current_event_size > max_request_size)
+        batches << current_batch
+        current_batch = []
+        current_batch_size = 0
+      end
+
+      current_batch_size += encoded_event.bytesize
+      current_batch << encoded_event
+    end
+    batches << current_batch
+    batches
+  end
+
+  # Truncate events over the provided max length, appending a marker when truncated
+  def truncate(event, max_length)
+    if event.length > max_length
+      event = event[0..max_length - 1]
+      event[max(0, max_length - DD_TRUNCATION_SUFFIX.length)..max_length - 1] = DD_TRUNCATION_SUFFIX
+      return event
+    end
+    event
+  end
+
+  def max(a, b)
+    a > b ? a : b
+  end
+
+  # Compress logs with GZIP
+  def gzip_compress(payload, compression_level)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz, compression_level)
+    begin
+      z.write(payload)
+    ensure
+      z.close
+    end
+    gz.string
+  end
+
+  # Build a new transport client
+  def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression)
+    if use_http
+      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key
+    else
+      DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, port
+    end
+  end
+
+  class RetryableError < StandardError;
+  end
+
+  class DatadogClient
+    def send_retries(payload, max_retries, max_backoff)
       backoff = 1
+      retries = 0
       begin
-        client ||= new_client
-        client.write(message)
-      rescue => e
-        @logger.warn("Could not send payload", :exception => e, :backtrace => e.backtrace)
-        client.close rescue nil
-        client = nil
+        send(payload)
+      rescue RetryableError => e
         if retries < max_retries || max_retries < 0
+          @logger.warn("Retrying ", :exception => e, :backtrace => e.backtrace)
           sleep backoff
           backoff = 2 * backoff unless backoff > max_backoff
           retries += 1
           retry
         end
-        @logger.warn("Max number of retries reached, dropping the payload", :payload => payload, :max_retries => max_retries)
       end
     end
+
+    def send(payload)
+      raise NotImplementedError, "Datadog transport client should implement the send method"
+    end
+
+    def close
+      raise NotImplementedError, "Datadog transport client should implement the close method"
+    end
   end
 
-  public
-  def receive(event)
-    # handle new event
-    @codec.encode(event)
+  class DatadogHTTPClient < DatadogClient
+    require "manticore"
+
+    def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
+      @logger = logger
+      protocol = use_ssl ? "https" : "http"
+      @url = "#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}"
+      @headers = {"Content-Type" => "application/json"}
+      if use_compression
+        @headers["Content-Encoding"] = "gzip"
+      end
+      logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled"))
+      config = {}
+      config[:ssl][:verify] = :disable if no_ssl_validation
+      @client = Manticore::Client.new(config)
+    end
+
+    def send(payload)
+      response = @client.post(@url, :body => payload, :headers => @headers).call
+      if response.code >= 500
+        raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
+      end
+      if response.code >= 400
+        @logger.error("Unable to send payload due to client error: #{response.code} #{response.body}")
+      end
+    end
+
+    def close
+      @client.close
+    end
   end
 
-  private
-  def new_client
-    # open a secure connection with Datadog
-    if @use_ssl
-      @logger.info("Starting SSL connection", :host => @host, :port => @port)
-      socket = TCPSocket.new @host, @port
-      sslSocket = OpenSSL::SSL::SSLSocket.new socket
-      sslSocket.connect
-      return sslSocket
-    else
-      @logger.info("Starting plaintext connection", :host => @host, :port => @port)
-      return TCPSocket.new @host, @port
+  class DatadogTCPClient < DatadogClient
+    require "socket"
+
+    def initialize(logger, use_ssl, no_ssl_validation, host, port)
+      @logger = logger
+      @use_ssl = use_ssl
+      @no_ssl_validation = no_ssl_validation
+      @host = host
+      @port = port
+    end
+
+    def connect
+      if @use_ssl
+        @logger.info("Starting SSL connection #{@host} #{@port}")
+        socket = TCPSocket.new @host, @port
+        ssl_context = OpenSSL::SSL::SSLContext.new
+        if @no_ssl_validation
+          ssl_context.set_params({:verify_mode => OpenSSL::SSL::VERIFY_NONE})
+        end
+        ssl_context = OpenSSL::SSL::SSLSocket.new socket, ssl_context
+        ssl_context.connect
+        ssl_context
+      else
+        @logger.info("Starting plaintext connection #{@host} #{@port}")
+        TCPSocket.new @host, @port
+      end
+    end
+
+    def send(payload)
+      begin
+        @socket ||= connect
+        @socket.puts(payload)
+      rescue => e
+        @socket.close rescue nil
+        @socket = nil
+        raise RetryableError.new "Unable to send payload: #{e.message}."
+      end
+    end
+
+    def close
+      @socket.close rescue nil
     end
   end
 

data/logstash-output-datadog_logs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-datadog_logs'
-  s.version       = '0.3.1'
+  s.version       = '0.4.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
   s.homepage      = 'https://www.datadoghq.com/'
@@ -9,14 +9,17 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+  s.metadata = {"logstash_plugin" => "true", "logstash_group" => "output"}
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-codec-json'
+
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/outputs/datadog_logs_spec.rb

@@ -4,3 +4,101 @@
 # Copyright 2017 Datadog, Inc.
 
 require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/datadog_logs"
+
+describe LogStash::Outputs::DatadogLogs do
+  context "should register" do
+    it "with an api key" do
+      plugin = LogStash::Plugin.lookup("output", "datadog_logs").new({"api_key" => "xxx"})
+      expect { plugin.register }.to_not raise_error
+    end
+
+    it "without an api key" do
+      expect { LogStash::Plugin.lookup("output", "datadog_logs").new() }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  subject do
+    plugin = LogStash::Plugin.lookup("output", "datadog_logs").new({"api_key" => "xxx"})
+    plugin.register
+    plugin
+  end
+
+  context "when truncating" do
+    it "should truncate messages of the given length" do
+      input = "foobarfoobarfoobarfoobar"
+      expect(subject.truncate(input, 15).length).to eq(15)
+    end
+
+    it "should replace the end of the message with a marker when truncated" do
+      input = "foobarfoobarfoobarfoobar"
+      expect(subject.truncate(input, 15)).to end_with("...TRUNCATED...")
+    end
+
+    it "should return the marker if the message length is smaller than the marker length" do
+      input = "foobar"
+      expect(subject.truncate(input, 1)).to eq("...TRUNCATED...")
+    end
+
+    it "should do nothing if the input length is smaller than the given length" do
+      input = "foobar"
+      expect(subject.truncate(input, 15)).to eq("foobar")
+    end
+  end
+
+  context "when using HTTP" do
+    it "should respect the batch length and create one batch of one event" do
+      input_events = [[LogStash::Event.new({"message" => "dd"}), "dd"]]
+      expect(subject.batch_http_events(input_events, 1, 1000).length).to eq(1)
+    end
+
+    it "should respect the batch length and create two batches of one event" do
+      input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
+      actual_events = subject.batch_http_events(input_events, 1, 1000)
+      expect(actual_events.length).to eq(2)
+      expect(actual_events[0][0]).to eq("dd1")
+      expect(actual_events[1][0]).to eq("dd2")
+    end
+
+    it "should respect the request size and create two batches of one event" do
+      input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
+      actual_events = subject.batch_http_events(input_events, 10, 3)
+      expect(actual_events.length).to eq(2)
+      expect(actual_events[0][0]).to eq("dd1")
+      expect(actual_events[1][0]).to eq("dd2")
+    end
+
+    it "should respect the request size and create two batches of two events" do
+      input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "dd2"}), "dd2"], [LogStash::Event.new({"message" => "dd3"}), "dd3"], [LogStash::Event.new({"message" => "dd4"}), "dd4"]]
+      actual_events = subject.batch_http_events(input_events, 6, 6)
+      expect(actual_events.length).to eq(2)
+      expect(actual_events[0][0]).to eq("dd1")
+      expect(actual_events[0][1]).to eq("dd2")
+      expect(actual_events[1][0]).to eq("dd3")
+      expect(actual_events[1][1]).to eq("dd4")
+    end
+
+    it "should truncate events whose length is bigger than the max request size" do
+      input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}),"foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
+      actual_events = subject.batch_http_events(input_events, 10, 3)
+      expect(actual_events.length).to eq(3)
+      expect(actual_events[0][0]).to eq("dd1")
+      expect(actual_events[1][0]).to eq("...TRUNCATED...")
+      expect(actual_events[2][0]).to eq("dd2")
+    end
+  end
+
+  context "when using TCP" do
+    it "should re-encode events" do
+      input_event = "{message=dd}"
+      encoded_event = subject.format_tcp_event(input_event, "xxx", 1000)
+      expect(encoded_event).to eq("xxx " + input_event)
+    end
+
+    it "should truncate too long messages" do
+      input_event = "{message=foobarfoobarfoobar}"
+      encoded_event = subject.format_tcp_event(input_event, "xxx", 20)
+      expect(encoded_event).to eq("xxx {...TRUNCATED...")
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-datadog_logs
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Datadog
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-15 00:00:00.000000000 Z
+date: 2020-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -25,6 +25,40 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -76,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: DatadogLogs lets you send logs to Datadog based on LogStash events.