logstash-integration-aws 0.1.0.pre

data/docs/output-sqs.asciidoc

@@ -0,0 +1,242 @@
+// :integration: aws
+:plugin: sqs
+:type: output
+:default_codec: json
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== Sqs output plugin
+
+// include::{include_path}/plugin_header-integration.asciidoc[]
+
+==== Description
+
+Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue.
+
+SQS is a simple, scalable queue system that is part of the Amazon Web
+Services suite of tools. Although SQS is similar to other queuing systems
+such as Advanced Message Queuing Protocol (AMQP), it uses a custom API and
+requires that you have an AWS account. See http://aws.amazon.com/sqs/ for
+more details on how SQS works, what the pricing schedule looks like and how
+to setup a queue.
+
+The "consumer" identity must have the following permissions on the queue:
+
+  * `sqs:GetQueueUrl`
+  * `sqs:SendMessage`
+  * `sqs:SendMessageBatch`
+
+Typically, you should setup an IAM policy, create a user and apply the IAM
+policy to the user. See http://aws.amazon.com/iam/ for more details on
+setting up AWS identities. A sample policy is as follows:
+
+[source,json]
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:GetQueueUrl",
+        "sqs:SendMessage",
+        "sqs:SendMessageBatch"
+      ],
+      "Resource": "arn:aws:sqs:us-east-1:123456789012:my-sqs-queue"
+    }
+  ]
+}
+
+==== Batch Publishing
+This output publishes messages to SQS in batches in order to optimize event
+throughput and increase performance. This is done using the
+[`SendMessageBatch`](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html)
+API. When publishing messages to SQS in batches, the following service limits
+must be respected (see
+[Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)):
+
+  * The maximum allowed individual message size is 256KiB.
+  * The maximum total payload size (i.e. the sum of the sizes of all
+    individual messages within a batch) is also 256KiB.
+
+This plugin will dynamically adjust the size of the batch published to SQS in
+order to ensure that the total payload size does not exceed 256KiB.
+
+WARNING: This output cannot currently handle messages larger than 256KiB. Any
+single message exceeding this size will be dropped.
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Sqs Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-batch_events>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-message_max_size>> |<<bytes,bytes>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-queue>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-queue_owner_aws_account_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-batch_events"]
+===== `batch_events` 
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+The number of events to be sent in each batch. Set this to `1` to disable
+the batch sending of messages.
+
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+
+[id="plugins-{type}s-{plugin}-message_max_size"]
+===== `message_max_size` 
+
+  * Value type is <<bytes,bytes>>
+  * Default value is `"256KiB"`
+
+The maximum number of bytes for any message sent to SQS. Messages exceeding
+this size will be dropped. See
+http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html.
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-queue"]
+===== `queue` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the target SQS queue. Note that this is just the name of the
+queue, not the URL or ARN.
+
+[id="plugins-{type}s-{plugin}-queue_owner_aws_account_id"]
+===== `queue_owner_aws_account_id`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The owning account id of the target SQS queue. IAM permissions need to be
+configured on both accounts to function.
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value type is <<string,string>>
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+Session name to use when assuming an IAM role.
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/codecs/cloudfront.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/plain"
+require "logstash/json"
+
+# This codec will read cloudfront encoded content
+class LogStash::Codecs::Cloudfront < LogStash::Codecs::Base
+  config_name "cloudfront"
+
+
+  # The character encoding used in this codec. Examples include "UTF-8" and
+  # "CP1252"
+  #
+  # JSON requires valid UTF-8 strings, but in some cases, software that
+  # emits JSON does so in another encoding (nxlog, for example). In
+  # weird cases like this, you can set the charset setting to the
+  # actual encoding of the text and logstash will convert it for you.
+  #
+  # For nxlog users, you'll want to set this to "CP1252"
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def initialize(params={})
+    super(params)
+    @converter = LogStash::Util::Charset.new(@charset)
+    @converter.logger = @logger
+  end
+
+  public
+  def decode(data)
+    begin
+      @gzip = Zlib::GzipReader.new(data)
+
+      metadata = extract_metadata(@gzip)
+
+      @logger.debug("Cloudfront: Extracting metadata", :metadata => metadata)
+
+      @gzip.each_line do |line|
+        yield create_event(line, metadata)
+      end
+
+    rescue Zlib::Error, Zlib::GzipFile::Error=> e
+      file = data.is_a?(String) ? data : data.class
+
+      @logger.error("Cloudfront codec: We cannot uncompress the gzip file", :filename => file)
+      raise e
+    end
+  end # def decode
+
+  public
+  def create_event(line, metadata)
+    event = LogStash::Event.new("message" => @converter.convert(line))
+    event.set("cloudfront_version", metadata["cloudfront_version"])
+    event.set("cloudfront_fields", metadata["cloudfront_fields"])
+    event
+  end
+
+
+  def extract_metadata(io)
+    version = extract_version(io.gets)
+    fields = extract_fields(io.gets)
+
+    return {
+      "cloudfront_version" => version,
+      "cloudfront_fields" => fields,
+    }
+  end
+
+
+  def extract_version(line)
+    if /^#Version: .+/.match(line)
+      junk, version = line.strip().split(/#Version: (.+)/)
+      version unless version.nil?
+    end
+  end
+
+
+  def extract_fields(line)
+    if /^#Fields: .+/.match(line)
+      junk, format = line.strip().split(/#Fields: (.+)/)
+      format unless format.nil?
+    end
+  end
+end # class LogStash::Codecs::Cloudfront

data/lib/logstash/codecs/cloudtrail.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/json"
+require "logstash/util/charset"
+
+# This is the base class for logstash codecs.
+class LogStash::Codecs::CloudTrail < LogStash::Codecs::Base
+  config_name "cloudtrail"
+
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def register
+    @converter = LogStash::Util::Charset.new(@charset)
+    @converter.logger = @logger
+  end
+
+  public
+  def decode(data)
+    decoded = LogStash::Json.load(@converter.convert(data))
+    decoded['Records'].to_a.each do |event|
+      event['@timestamp'] = event.delete('eventTime')
+
+      if event["requestParameters"] && event['requestParameters'].has_key?("disableApiTermination")
+        if event['requestParameters']['disableApiTermination'].class != Hash
+          disableApiTermination = event['requestParameters'].delete('disableApiTermination')
+          event['requestParameters']['disableApiTermination']= {"value" => disableApiTermination}
+        end
+      end
+
+      substitute_invalid_ip_address(event)
+
+      yield LogStash::Event.new(event)
+    end
+  end # def decode
+
+  # Workaround for https://github.com/logstash-plugins/logstash-codec-cloudtrail/issues/20
+  # API calls from support will fill the sourceIpAddress with a hostname string instead of an ip
+  # address.
+  def substitute_invalid_ip_address(event)
+    source_ip_address = event["sourceIpAddress"]
+    if source_ip_address && source_ip_address !~ Resolv::IPv4::Regex && source_ip_address !~ Resolv::IPv6::Regex
+      event["sourceHost"] = event.delete("sourceIpAddress")
+    end
+  end
+
+end # class LogStash::Codecs::CloudTrail