logstash-output-httpcodec 5.2.5

data/docs/index.asciidoc

@@ -0,0 +1,391 @@
+:plugin: http
+:type: output
+:default_codec: plain
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== Http output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This output lets you send events to a generic HTTP(S) endpoint.
+
+This output will execute up to 'pool_max' requests in parallel for performance.
+Consider this when tuning this plugin for performance.
+
+Additionally, note that when parallel execution is used strict ordering of events is not
+guaranteed!
+
+Beware, to use codecs, please use the 'format' option for now.
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Http Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-automatic_retries>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-client_key>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-connect_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-content_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-cookies>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-follow_redirects>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-format>> |<<string,string>>, one of `["json", "json_batch", "form", "message"]`|No
+| <<plugins-{type}s-{plugin}-headers>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-http_method>> |<<string,string>>, one of `["put", "post", "patch", "delete", "get", "head"]`|Yes
+| <<plugins-{type}s-{plugin}-ignorable_codes>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-keepalive>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-keystore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-mapping>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-message>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-proxy>> |<<,>>|No
+| <<plugins-{type}s-{plugin}-request_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_failed>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-retryable_codes>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-url>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-automatic_retries"]
+===== `automatic_retries` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+How many times should the client retry a failing URL. We highly recommend NOT setting this value
+to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
+Only IO related failures will be retried, such as connection timeouts and unreachable hosts.
+Valid but non 2xx HTTP responses will always be retried, regardless of the value of this setting,
+unless `retry_failed` is set.
+Note: if `retry_non_idempotent` is NOT set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
+
+[id="plugins-{type}s-{plugin}-cacert"]
+===== `cacert` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If you need to use a custom X.509 CA (.pem certs) specify the path to that here
+
+[id="plugins-{type}s-{plugin}-client_cert"]
+===== `client_cert` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here
+
+[id="plugins-{type}s-{plugin}-client_key"]
+===== `client_key` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If you're using a client certificate specify the path to the encryption key here
+
+[id="plugins-{type}s-{plugin}-connect_timeout"]
+===== `connect_timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+Timeout (in seconds) to wait for a connection to be established. Default is `10s`
+
+[id="plugins-{type}s-{plugin}-content_type"]
+===== `content_type` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Content type
+
+If not specified, this defaults to the following:
+
+* if format is "json", "application/json"
+* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request.
+* if format is "form", "application/x-www-form-urlencoded"
+
+[id="plugins-{type}s-{plugin}-cookies"]
+===== `cookies` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Enable cookie support. With this enabled the client will persist cookies
+across requests as a normal web browser would. Enabled by default
+
+[id="plugins-{type}s-{plugin}-follow_redirects"]
+===== `follow_redirects` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Should redirects be followed? Defaults to `true`
+
+[id="plugins-{type}s-{plugin}-format"]
+===== `format` 
+
+  * Value can be any of: `json`, `json_batch`, `form`, `message`, `codec`
+  * Default value is `"json"`
+
+Set the format of the http body.
+
+If json_batch, each batch of events received by this output will be placed
+into a single JSON array and sent in one request. This is particularly useful
+for high throughput scenarios such as sending data between Logstash instaces.
+
+If form, then the body will be the mapping (or whole event) converted
+into a query parameter string, e.g. `foo=bar&baz=fizz...`
+
+If message, then the body will be the result of formatting the event according to message
+
+Otherwise, the event is sent as json.
+
+[id="plugins-{type}s-{plugin}-headers"]
+===== `headers` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Custom headers to use
+format is `headers => ["X-My-Header", "%{host}"]`
+
+[id="plugins-{type}s-{plugin}-http_compression"]
+===== `http_compression`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Enable request compression support. With this enabled the plugin will compress
+http requests using gzip.
+
+[id="plugins-{type}s-{plugin}-http_method"]
+===== `http_method` 
+
+  * This is a required setting.
+  * Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head`
+  * There is no default value for this setting.
+
+The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
+
+[id="plugins-{type}s-{plugin}-ignorable_codes"]
+===== `ignorable_codes` 
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+If you would like to consider some non-2xx codes to be successes 
+enumerate them here. Responses returning these codes will be considered successes
+
+[id="plugins-{type}s-{plugin}-keepalive"]
+===== `keepalive` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
+one with this to fix interactions with broken keepalive implementations.
+
+[id="plugins-{type}s-{plugin}-keystore"]
+===== `keystore` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys!
+
+[id="plugins-{type}s-{plugin}-keystore_password"]
+===== `keystore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Specify the keystore password here.
+Note, most .jks files created with keytool require a password!
+
+[id="plugins-{type}s-{plugin}-keystore_type"]
+===== `keystore_type` 
+
+  * Value type is <<string,string>>
+  * Default value is `"JKS"`
+
+Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS`
+
+[id="plugins-{type}s-{plugin}-mapping"]
+===== `mapping` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+This lets you choose the structure and parts of the event that are sent.
+
+
+For example:
+[source,ruby]
+   mapping => {"foo" => "%{host}"
+              "bar" => "%{type}"}
+
+[id="plugins-{type}s-{plugin}-message"]
+===== `message` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+
+
+[id="plugins-{type}s-{plugin}-pool_max"]
+===== `pool_max` 
+
+  * Value type is <<number,number>>
+  * Default value is `50`
+
+Max number of concurrent connections. Defaults to `50`
+
+[id="plugins-{type}s-{plugin}-pool_max_per_route"]
+===== `pool_max_per_route` 
+
+  * Value type is <<number,number>>
+  * Default value is `25`
+
+Max number of concurrent connections to a single host. Defaults to `25`
+
+[id="plugins-{type}s-{plugin}-proxy"]
+===== `proxy` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
+
+1. Proxy host in form: `http://proxy.org:1234`
+2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
+3. Proxy host in form: `{url =>  'http://proxy.org:1234', user => 'username@host', password => 'password'}`
+
+[id="plugins-{type}s-{plugin}-request_timeout"]
+===== `request_timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+This module makes it easy to add a very fully configured HTTP client to logstash
+based on [Manticore](https://github.com/cheald/manticore).
+For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller
+Timeout (in seconds) for the entire request
+
+[id="plugins-{type}s-{plugin}-retry_failed"]
+===== `retry_failed` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Set this to false if you don't want this output to retry failed requests
+
+[id="plugins-{type}s-{plugin}-retry_non_idempotent"]
+===== `retry_non_idempotent` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
+This only affects connectivity related errors (see related `automatic_retries` setting).
+
+[id="plugins-{type}s-{plugin}-retryable_codes"]
+===== `retryable_codes` 
+
+  * Value type is <<number,number>>
+  * Default value is `[429, 500, 502, 503, 504]`
+
+If encountered as response codes this plugin will retry these requests
+
+[id="plugins-{type}s-{plugin}-socket_timeout"]
+===== `socket_timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+Timeout (in seconds) to wait for data on the socket. Default is `10s`
+
+[id="plugins-{type}s-{plugin}-truststore"]
+===== `truststore` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs!
+
+[id="plugins-{type}s-{plugin}-truststore_password"]
+===== `truststore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Specify the truststore password here.
+Note, most .jks files created with keytool require a password!
+
+[id="plugins-{type}s-{plugin}-truststore_type"]
+===== `truststore_type` 
+
+  * Value type is <<string,string>>
+  * Default value is `"JKS"`
+
+Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
+
+[id="plugins-{type}s-{plugin}-url"]
+===== `url` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URL to use
+
+[id="plugins-{type}s-{plugin}-validate_after_inactivity"]
+===== `validate_after_inactivity` 
+
+  * Value type is <<number,number>>
+  * Default value is `200`
+
+How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
+You may want to set this lower, possibly to 0 if you get connection errors regularly
+Quoting the Apache commons docs (this client is based Apache Commmons):
+'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.'
+See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/outputs/httpcodec.rb

@@ -0,0 +1,390 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require "uri"
+require "logstash/plugin_mixins/http_client"
+require "zlib"
+
+class LogStash::Outputs::HttpCodec < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+
+  concurrency :shared
+
+  attr_accessor :is_batch
+
+  VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
+
+  RETRYABLE_MANTICORE_EXCEPTIONS = [
+    ::Manticore::Timeout,
+    ::Manticore::SocketException,
+    ::Manticore::ClientProtocolException,
+    ::Manticore::ResolutionFailure,
+    ::Manticore::SocketTimeout
+  ]
+
+  # This output lets you send events to a
+  # generic HTTP(S) endpoint
+  #
+  # This output will execute up to 'pool_max' requests in parallel for performance.
+  # Consider this when tuning this plugin for performance.
+  #
+  # Additionally, note that when parallel execution is used strict ordering of events is not
+  # guaranteed!
+  #
+  # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
+
+  config_name "httpcodec"
+
+  # URL to use
+  config :url, :validate => :string, :required => :true
+
+  # The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
+  config :http_method, :validate => VALID_METHODS, :required => :true
+
+  # Custom headers to use
+  # format is `headers => ["X-My-Header", "%{host}"]`
+  config :headers, :validate => :hash, :default => {}
+
+  # Content type
+  #
+  # If not specified, this defaults to the following:
+  #
+  # * if format is "json", "application/json"
+  # * if format is "form", "application/x-www-form-urlencoded"
+  config :content_type, :validate => :string
+
+  # Set this to false if you don't want this output to retry failed requests
+  config :retry_failed, :validate => :boolean, :default => true
+
+  # If encountered as response codes this plugin will retry these requests
+  config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
+
+  # If you would like to consider some non-2xx codes to be successes
+  # enumerate them here. Responses returning these codes will be considered successes
+  config :ignorable_codes, :validate => :number, :list => true
+
+  # This lets you choose the structure and parts of the event that are sent.
+  #
+  #
+  # For example:
+  # [source,ruby]
+  #    mapping => {"foo" => "%{host}"
+  #               "bar" => "%{type}"}
+  config :mapping, :validate => :hash
+
+  # Set the format of the http body.
+  #
+  # If form, then the body will be the mapping (or whole event) converted
+  # into a query parameter string, e.g. `foo=bar&baz=fizz...`
+  #
+  # If message, then the body will be the result of formatting the event according to message
+  #
+  # Otherwise, the event is sent as json.
+  config :format, :validate => ["json", "json_batch", "form", "message", "codec"], :default => "json"
+
+  # Set this to true if you want to enable gzip compression for your http requests
+  config :http_compression, :validate => :boolean, :default => false
+
+  config :message, :validate => :string
+  default :codec, "json"
+
+  def register
+    @codec.on_event do |event, payload|
+      payload
+    end
+
+    @http_method = @http_method.to_sym
+
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
+
+    @requests = Array.new
+
+    if @content_type.nil?
+      case @format
+        when "form" ; @content_type = "application/x-www-form-urlencoded"
+        when "json" ; @content_type = "application/json"
+        when "json_batch" ; @content_type = "application/json"
+        when "message" ; @content_type = "text/plain"
+        when "codec" ; @content_type = "text/plain"
+      end
+    end
+
+    @is_batch = @format == "json_batch"
+
+    @headers["Content-Type"] = @content_type
+
+    validate_format!
+
+    # Run named Timer as daemon thread
+    @timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
+  end # def register
+
+  def multi_receive(events)
+    return if events.empty?
+    send_events(events)
+  end
+
+  class RetryTimerTask < java.util.TimerTask
+    def initialize(pending, event, attempt)
+      @pending = pending
+      @event = event
+      @attempt = attempt
+      super()
+    end
+
+    def run
+      @pending << [@event, @attempt]
+    end
+  end
+
+  def log_retryable_response(response)
+    if (response.code == 429)
+      @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
+    else
+      @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
+    end
+  end
+
+  def log_error_response(response, url, event)
+    log_failure(
+              "Encountered non-2xx HTTP code #{response.code}",
+              :response_code => response.code,
+              :url => url,
+              :event => event
+            )
+  end
+
+  def send_events(events)
+    successes = java.util.concurrent.atomic.AtomicInteger.new(0)
+    failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
+    retries = java.util.concurrent.atomic.AtomicInteger.new(0)
+    event_count = @is_batch ? 1 : events.size
+
+    pending = Queue.new
+    if @is_batch
+      pending << [events, 0]
+    else
+      events.each {|e| pending << [e, 0]}
+    end
+
+    while popped = pending.pop
+      break if popped == :done
+
+      event, attempt = popped
+
+      action, event, attempt = send_event(event, attempt)
+      begin
+        action = :failure if action == :retry && !@retry_failed
+
+        case action
+        when :success
+          successes.incrementAndGet
+        when :retry
+          retries.incrementAndGet
+
+          next_attempt = attempt+1
+          sleep_for = sleep_for_attempt(next_attempt)
+          @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
+          timer_task = RetryTimerTask.new(pending, event, next_attempt)
+          @timer.schedule(timer_task, sleep_for*1000)
+        when :failure
+          failures.incrementAndGet
+        else
+          raise "Unknown action #{action}"
+        end
+
+        if action == :success || action == :failure
+          if successes.get+failures.get == event_count
+            pending << :done
+          end
+        end
+      rescue => e
+        # This should never happen unless there's a flat out bug in the code
+        @logger.error("Error sending HTTP Request",
+          :class => e.class.name,
+          :message => e.message,
+          :backtrace => e.backtrace)
+        failures.incrementAndGet
+        raise e
+      end
+    end
+  rescue => e
+    @logger.error("Error in http output loop",
+            :class => e.class.name,
+            :message => e.message,
+            :backtrace => e.backtrace)
+    raise e
+  end
+
+  def sleep_for_attempt(attempt)
+    sleep_for = attempt**2
+    sleep_for = sleep_for <= 60 ? sleep_for : 60
+    (sleep_for/2) + (rand(0..sleep_for)/2)
+  end
+
+  def send_event(event, attempt)
+    body = event_body(event)
+
+    # Send the request
+    url = @is_batch ? @url : event.sprintf(@url)
+    headers = @is_batch ? @headers : event_headers(event)
+
+    # Compress the body and add appropriate header
+    if @http_compression == true
+      headers["Content-Encoding"] = "gzip"
+      body = gzip(body)
+    end
+
+    # Create an async request
+    response = client.send(@http_method, url, :body => body, :headers => headers).call
+
+    if !response_success?(response)
+      if retryable_response?(response)
+        log_retryable_response(response)
+        return :retry, event, attempt
+      else
+        log_error_response(response, url, event)
+        return :failure, event, attempt
+      end
+    else
+      return :success, event, attempt
+    end
+
+  rescue => exception
+    will_retry = retryable_exception?(exception)
+    log_failure("Could not fetch URL",
+                :url => url,
+                :method => @http_method,
+                :body => body,
+                :headers => headers,
+                :message => exception.message,
+                :class => exception.class.name,
+                :backtrace => exception.backtrace,
+                :will_retry => will_retry
+    )
+
+    if will_retry
+      return :retry, event, attempt
+    else
+      return :failure, event, attempt
+    end
+  end
+
+  def close
+    @timer.cancel
+    client.close
+  end
+
+  private
+
+  def response_success?(response)
+    code = response.code
+    return true if @ignorable_codes && @ignorable_codes.include?(code)
+    return code >= 200 && code <= 299
+  end
+
+  def retryable_response?(response)
+    @retryable_codes && @retryable_codes.include?(response.code)
+  end
+
+  def retryable_exception?(exception)
+    RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
+  end
+
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
+
+  # Format the HTTP body
+  def event_body(event)
+    # TODO: Create an HTTP post data codec, use that here
+    if @format == "json"
+      LogStash::Json.dump(map_event(event))
+    elsif @format == "message"
+      event.sprintf(@message)
+    elsif @format == "codec"
+      @codec.encode(event)
+    elsif @format == "json_batch"
+      LogStash::Json.dump(event.map {|e| map_event(e) })
+    else
+      encode(map_event(event))
+    end
+  end
+
+  # gzip data
+  def gzip(data)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz)
+    z.write(data)
+    z.close
+    gz.string
+  end
+
+  def convert_mapping(mapping, event)
+    if mapping.is_a?(Hash)
+      mapping.reduce({}) do |acc, kv|
+        k, v = kv
+        acc[k] = convert_mapping(v, event)
+        acc
+      end
+    elsif mapping.is_a?(Array)
+      mapping.map { |elem| convert_mapping(elem, event) }
+    else
+      event.sprintf(mapping)
+    end
+  end
+
+  def map_event(event)
+    if @mapping
+      convert_mapping(@mapping, event)
+    else
+      event.to_hash
+    end
+  end
+
+  def event_headers(event)
+    custom_headers(event) || {}
+  end
+
+  def custom_headers(event)
+    return nil unless @headers
+
+    @headers.reduce({}) do |acc,kv|
+      k,v = kv
+      acc[k] = event.sprintf(v)
+      acc
+    end
+  end
+
+  #TODO Extract this to a codec
+  def encode(hash)
+    return hash.collect do |key, value|
+      CGI.escape(key) + "=" + CGI.escape(value.to_s)
+    end.join("&")
+  end
+
+
+  def validate_format!
+    if @format == "message"
+      if @message.nil?
+        raise "message must be set if message format is used"
+      end
+
+      if @content_type.nil?
+        raise "content_type must be set if message format is used"
+      end
+
+      unless @mapping.nil?
+        @logger.warn "mapping is not supported and will be ignored if message format is used"
+      end
+    end
+  end
+end