logstash-integration-aws 0.1.0.pre

data/docs/output-s3.asciidoc

@@ -0,0 +1,442 @@
+// :integration: aws
+:plugin: s3
+:type: output
+:default_codec: line
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== S3 output plugin
+
+// include::{include_path}/plugin_header-integration.asciidoc[]
+
+==== Description
+
+This plugin batches and uploads logstash events into Amazon Simple Storage
+Service (Amazon S3).
+
+IMPORTANT: The S3 output plugin only supports AWS S3. 
+Other S3 compatible storage solutions are not supported.
+
+S3 outputs create temporary files into the OS' temporary directory. 
+You can specify where to save them using the `temporary_directory` option.
+
+IMPORTANT: For configurations containing multiple s3 outputs with the restore
+option enabled, each output should define its own 'temporary_directory'.
+
+===== Requirements
+
+* Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key)
+* S3 PutObject permission
+
+===== S3 output file
+ 
+[source,txt]
+-----
+`ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt`
+-----
+
+|=======
+| ls.s3 | indicates logstash plugin s3 |
+| 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. |
+| 2013-04-18T10.00 | represents the time whenever you specify time_file. |
+| tag_hello | indicates the event's tag. |
+| part0 | If you indicate size_file, it will generate more parts if your file.size > size_file. 
+When a file is full, it gets pushed to the bucket and then deleted from the temporary directory. 
+If a file is empty, it is simply deleted.  Empty files will not be pushed. |
+|=======
+
+===== Crash Recovery
+
+This plugin will recover and upload temporary log files after crash/abnormal termination when using `restore` set to true
+
+===== Usage
+This is an example of logstash config:
+[source,ruby]
+output {
+   s3{
+     access_key_id => "crazy_key"             (optional)
+     secret_access_key => "monkey_access_key" (optional)
+     region => "eu-west-1"                    (optional, default = "us-east-1")
+     bucket => "your_bucket"                  (required)
+     size_file => 2048                        (optional) - Bytes
+     time_file => 5                           (optional) - Minutes
+     codec => "plain"                         (optional)
+     canned_acl => "private"                  (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" )
+   }
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== S3 Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-additional_settings>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-canned_acl>> |<<string,string>>, one of `["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"]`|No
+| <<plugins-{type}s-{plugin}-encoding>> |<<string,string>>, one of `["none", "gzip"]`|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-restore>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-retry_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_delay>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-rotation_strategy>> |<<string,string>>, one of `["size_and_time", "size", "time"]`|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-server_side_encryption>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-server_side_encryption_algorithm>> |<<string,string>>, one of `["AES256", "aws:kms"]`|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-signature_version>> |<<string,string>>, one of `["v2", "v4"]`|No
+| <<plugins-{type}s-{plugin}-size_file>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssekms_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-storage_class>> |<<string,string>>, one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"]`|No
+| <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-time_file>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_multipart_threshold>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_queue_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_workers_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+. External credentials file specified by `aws_credentials_file`
+. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-additional_settings"]
+===== `additional_settings`
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Key-value pairs of settings and corresponding values used to parametrize
+the connection to S3. See full list in https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html[the AWS SDK documentation]. Example:
+
+[source,ruby]
+    output {
+      s3 {
+        access_key_id => "1234",
+        secret_access_key => "secret",
+        region => "eu-west-1",
+        bucket => "logstash-test",
+        additional_settings => {
+          "force_path_style" => true,
+          "follow_redirects" => false
+        }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-bucket"]
+===== `bucket` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+S3 bucket
+
+[id="plugins-{type}s-{plugin}-canned_acl"]
+===== `canned_acl` 
+
+  * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`, `aws-exec-read`, `bucket-owner-read`, `bucket-owner-full-control`, `log-delivery-write`
+  * Default value is `"private"`
+
+The S3 canned ACL to use when putting the file. Defaults to "private".
+
+[id="plugins-{type}s-{plugin}-encoding"]
+===== `encoding` 
+
+  * Value can be any of: `none`, `gzip`
+  * Default value is `"none"`
+
+Specify the content encoding. Supports ("gzip"). Defaults to "none"
+
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+The endpoint should be an HTTP or HTTPS URL, e.g. https://example.com
+
+[id="plugins-{type}s-{plugin}-prefix"]
+===== `prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Specify a prefix to the uploaded filename to simulate directories on S3.
+Prefix does not require leading slash.
+This option supports
+{logstash-ref}/event-dependent-configuration.html#sprintf[Logstash
+interpolation]. For example, files can be prefixed with the event date using
+`prefix = "%{+YYYY}/%{+MM}/%{+dd}"`.
+
+IMPORTANT: Take care when you are using interpolated strings in prefixes. This
+has the potential to create large numbers of unique prefixes, causing large
+numbers of in-progress uploads. This scenario may result in performance and
+stability issues, which can be further exacerbated when you use a
+rotation_strategy that delays uploads.
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value type is <<string,string>>
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-restore"]
+===== `restore` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Used to enable recovery after crash/abnormal termination.
+Temporary log files will be recovered and uploaded.
+
+[id="plugins-{type}s-{plugin}-retry_count"]
+===== `retry_count`
+
+* Value type is <<number,number>>
+* Default value is `Infinity`
+
+Allows to limit number of retries when S3 uploading fails.
+
+[id="plugins-{type}s-{plugin}-retry_delay"]
+===== `retry_delay`
+
+* Value type is <<number,number>>
+* Default value is `1`
+
+Delay (in seconds) to wait between consecutive retries on upload failures.
+
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+Session name to use when assuming an IAM role.
+
+[id="plugins-{type}s-{plugin}-rotation_strategy"]
+===== `rotation_strategy` 
+
+  * Value can be any of: `size_and_time`, `size`, `time`
+  * Default value is `"size_and_time"`
+
+Controls when to close the file and push it to S3. 
+
+If you set this value to `size`, it uses the value set in
+<<plugins-{type}s-{plugin}-size_file,`size_file`>>.
+If you set this value to `time`, it uses the value set in
+<<plugins-{type}s-{plugin}-time_file,`time_file`>>.
+If you set this value to `size_and_time`, it uses the values from
+<<plugins-{type}s-{plugin}-size_file,`size_file`>> and
+<<plugins-{type}s-{plugin}-time_file,`time_file`>>, and splits the file when
+either one matches. 
+
+The default strategy checks both size and time. The first value to
+match triggers file rotation.
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-server_side_encryption"]
+===== `server_side_encryption` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Specifies whether or not to use S3's server side encryption. Defaults to no encryption.
+
+[id="plugins-{type}s-{plugin}-server_side_encryption_algorithm"]
+===== `server_side_encryption_algorithm` 
+
+  * Value can be any of: `AES256`, `aws:kms`
+  * Default value is `"AES256"`
+
+Specifies what type of encryption to use when SSE is enabled.
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-signature_version"]
+===== `signature_version` 
+
+  * Value can be any of: `v2`, `v4`
+  * There is no default value for this setting.
+
+The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly
+specified here
+
+[id="plugins-{type}s-{plugin}-size_file"]
+===== `size_file` 
+
+  * Value type is <<number,number>>
+  * Default value is `5242880`
+
+Set the file size in bytes. When the number of bytes exceeds the `size_file`
+value, a new file is created. If you use tags, Logstash generates a specific size
+file for every tag.
+
+[id="plugins-{type}s-{plugin}-ssekms_key_id"]
+===== `ssekms_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The key to use when specified along with server_side_encryption => aws:kms.
+If server_side_encryption => aws:kms is set but this is not default KMS key is used.
+http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+
+[id="plugins-{type}s-{plugin}-storage_class"]
+===== `storage_class` 
+
+  * Value can be any of: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`
+  * Default value is `"STANDARD"`
+
+Specifies what S3 storage class to use when uploading the file.
+More information about the different storage classes can be found:
+http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html
+Defaults to STANDARD.
+
+[id="plugins-{type}s-{plugin}-temporary_directory"]
+===== `temporary_directory` 
+
+  * Value type is <<string,string>>
+  * Default value is `"/tmp/logstash"`
+
+Set the directory where logstash will store the tmp files before sending it to S3
+default to the current OS temporary directory in linux /tmp/logstash
+
+[id="plugins-{type}s-{plugin}-time_file"]
+===== `time_file` 
+
+  * Value type is <<number,number>>
+  * Default value is `15`
+
+Set the time, in MINUTES, to close the current sub_time_section of bucket.
+If <<plugins-{type}s-{plugin}-rotation_strategy,`rotation_strategy`>> is set to `time` or `size_and_time`, then `time_file` cannot be set to 0. 
+Otherwise, the plugin raises a configuration error.
+
+[id="plugins-{type}s-{plugin}-upload_multipart_threshold"]
+===== `upload_multipart_threshold`
+
+  * Value type is <<number,number>>
+  * Default value is `15728640`
+
+Files larger than this number are uploaded using the S3 multipart APIs
+
+[id="plugins-{type}s-{plugin}-upload_queue_size"]
+===== `upload_queue_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `4`
+
+Number of items we can keep in the local queue before uploading them
+
+[id="plugins-{type}s-{plugin}-upload_workers_count"]
+===== `upload_workers_count` 
+
+  * Value type is <<number,number>>
+  * Default value is `4`
+
+Specify how many workers to use to upload the files to S3
+
+[id="plugins-{type}s-{plugin}-validate_credentials_on_root_bucket"]
+===== `validate_credentials_on_root_bucket` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+The common use case is to define permissions on the root bucket and give Logstash
+full access to write logs. 
+In some circumstances, you need more granular permissions on the subfolder. This
+allows you to disable the check at startup.
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/docs/output-sns.asciidoc

@@ -0,0 +1,166 @@
+// :integration: aws
+:plugin: sns
+:type: output
+:default_codec: plain
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== Sns output plugin
+
+// include::{include_path}/plugin_header-integration.asciidoc[]
+
+==== Description
+
+SNS output.
+
+Send events to Amazon's Simple Notification Service, a hosted pub/sub
+framework.  It supports various subscription types, including email, HTTP/S, SMS, and SQS.
+
+For further documentation about the service see:
+
+  http://docs.amazonwebservices.com/sns/latest/api/
+
+This plugin looks for the following fields on events it receives:
+
+ * `sns` - If no ARN is found in the configuration file, this will be used as
+ the ARN to publish.
+ * `sns_subject` - The subject line that should be used.
+ Optional. The "%{host}" will be used if `sns_subject` is not present. The subject
+ will be truncated to 100 characters. If `sns_subject` is set to a non-string value a JSON version of that value will be saved.
+ * `sns_message` - Optional string of message to be sent. If this is set to a non-string value it will be encoded with the specified `codec`. If this is not set the entire event will be encoded with the codec.
+ with the @message truncated so that the length of the JSON fits in
+ `32768` bytes.
+
+==== Upgrading to 2.0.0
+
+This plugin used to have a `format` option for controlling the encoding of messages prior to being sent to SNS.
+This plugin now uses the logstash standard {logstash-ref}/configuration-file-structure.html#codec[codec] option for encoding instead.
+If you want the same 'plain' format as the v0/1 codec (`format => "plain"`) use `codec => "s3_plain"`.
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Sns Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-publish_boot_message_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-arn"]
+===== `arn` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Optional ARN to send messages to. If you do not set this you must
+include the `sns` field in your events to set the ARN on a per-message basis!
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-publish_boot_message_arn"]
+===== `publish_boot_message_arn` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+When an ARN for an SNS topic is specified here, the message
+"Logstash successfully booted" will be sent to it when this plugin
+is registered.
+
+Example: arn:aws:sns:us-east-1:770975001275:logstash-testing
+
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!: