logstash-integration-aws 0.1.0.pre

data/docs/input-cloudwatch.asciidoc

@@ -0,0 +1,320 @@
+// :integration: aws
+:plugin: cloudwatch
+:type: input
+:default_codec: plain
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== Cloudwatch input plugin
+
+// include::{include_path}/plugin_header-integration.asciidoc[]
+
+==== Description
+
+Pull events from the Amazon Web Services CloudWatch API.
+
+To use this plugin, you *must* have an AWS account, and the following policy
+
+Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
+A sample policy for EC2 metrics is as follows:
+[source,json]
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "Stmt1444715676000",
+                "Effect": "Allow",
+                "Action": [
+                    "cloudwatch:GetMetricStatistics",
+                    "cloudwatch:ListMetrics"
+                ],
+                "Resource": "*"
+            },
+            {
+                "Sid": "Stmt1444716576170",
+                "Effect": "Allow",
+                "Action": [
+                    "ec2:DescribeInstances"
+                ],
+                "Resource": "*"
+            }
+        ]
+    }
+
+See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
+
+===== Configuration examples
+[source, ruby]
+    input {
+      cloudwatch {
+        namespace => "AWS/EC2"
+        metrics => [ "CPUUtilization" ]
+        filters => { "tag:Group" => "API-Production" }
+        region => "us-east-1"
+      }
+    }
+
+[source, ruby]
+    input {
+      cloudwatch {
+        namespace => "AWS/EBS"
+        metrics => ["VolumeQueueLength"]
+        filters => { "tag:Monitoring" => "Yes" }
+        region => "us-east-1"
+      }
+    }
+
+[source, ruby]
+    input {
+      cloudwatch {
+        namespace => "AWS/RDS"
+        metrics => ["CPUUtilization", "CPUCreditUsage"]
+        filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier
+        region => "us-east-1"
+      }
+    }
+
+[source, ruby]
+    input {
+      cloudwatch {
+        namespace => "sqlserver_test2"
+        metrics => [ "Memory Available Bytes"]
+        filters =>  {
+            InstanceId => "i-xxxxxxxxxxx"
+            objectname => "Memory"
+        }
+        combined => true
+        interval => 600
+        period => 300
+      }
+    }
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Cloudwatch Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-combined>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-filters>> |<<array,array>>|See <<plugins-{type}s-{plugin}-filters,note>>
+| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-metrics>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-namespace>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-period>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-statistics>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-use_ssl>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-combined"]
+===== `combined` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Use this for namespaces that need to combine the dimensions like S3 and SNS.
+
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+
+[id="plugins-{type}s-{plugin}-filters"]
+===== `filters` 
+
+  * This setting can be required or optional. See note below.
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+NOTE: This setting is optional when the namespace is `AWS/EC2`. Otherwise this is a required field.
+
+Specify the filters to apply when fetching resources. Follow the AWS convention:
+
+* Instances: { 'instance-id' => 'i-12344321' }
+* Tags: { "tag:Environment" => "Production" }
+* Volumes: { 'attachment.status' => 'attached' }
+
+Each namespace uniquely support certain dimensions. Please consult the documentation
+to ensure you're using valid filters.
+
+[id="plugins-{type}s-{plugin}-interval"]
+===== `interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `900`
+
+Set how frequently CloudWatch should be queried
+
+The default, `900`, means check every 15 minutes. Setting this value too low
+(generally less than 300) results in no metrics being returned from CloudWatch.
+
+[id="plugins-{type}s-{plugin}-metrics"]
+===== `metrics` 
+
+  * Value type is <<array,array>>
+  * Default value is `["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"]`
+
+Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
+for the available metrics for other namespaces.
+
+[id="plugins-{type}s-{plugin}-namespace"]
+===== `namespace` 
+
+  * Value type is <<string,string>>
+  * Default value is `"AWS/EC2"`
+
+If undefined, LogStash will complain, even if codec is unused.
+The service namespace of the metrics to fetch.
+
+The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
+for valid values.
+
+[id="plugins-{type}s-{plugin}-period"]
+===== `period` 
+
+  * Value type is <<number,number>>
+  * Default value is `300`
+
+Set the granularity of the returned datapoints.
+
+Must be at least 60 seconds and in multiples of 60.
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value type is <<string,string>>
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+Session name to use when assuming an IAM role.
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-statistics"]
+===== `statistics` 
+
+  * Value type is <<array,array>>
+  * Default value is `["SampleCount", "Average", "Minimum", "Maximum", "Sum"]`
+
+Specify the statistics to fetch for each namespace
+
+[id="plugins-{type}s-{plugin}-use_ssl"]
+===== `use_ssl` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Make sure we require the V1 classes when including this module.
+require 'aws-sdk' will load v2 classes.
+Should we require (true) or disable (false) using SSL for communicating with the AWS API
+The AWS SDK for Ruby defaults to SSL so we preserve that
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/docs/input-s3.asciidoc

@@ -0,0 +1,346 @@
+// :integration: aws
+:plugin: s3
+:type: input
+:default_codec: plain
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== S3 input plugin
+
+// include::{include_path}/plugin_header-integration.asciidoc[]
+
+==== Description
+
+Stream events from files from a S3 bucket.
+
+IMPORTANT: The S3 input plugin only supports AWS S3. 
+Other S3 compatible storage solutions are not supported.
+
+Each line from each file generates an event.
+Files ending in `.gz` are handled as gzip'ed files.
+
+Files that are archived to AWS Glacier will be skipped.
+
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+This plugin adds cloudfront metadata to event.
+When ECS compatibility is disabled, the value is stored in the root level.
+When ECS is enabled, the value is stored in the `@metadata` where it can be used by other plugins in your pipeline.
+
+Here’s how ECS compatibility mode affects output.
+[cols="<l,<l,e,<e"]
+|=======================================================================
+| ECS disabled | ECS v1 | Availability | Description
+
+| cloudfront_fields | [@metadata][s3][cloudfront][fields] | available when the file is a CloudFront log | column names of log
+| cloudfront_version | [@metadata][s3][cloudfront][version] | available when the file is a CloudFront log | version of log
+|=======================================================================
+
+[id="plugins-{type}s-{plugin}-options"]
+==== S3 Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-additional_settings>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_add_prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_to_bucket>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_to_dir>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-gzip_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-include_object_properties>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-watch_for_new_files>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+
+[id="plugins-{type}s-{plugin}-additional_settings"]
+===== `additional_settings`
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Key-value pairs of settings and corresponding values used to parametrize
+the connection to s3. See full list in https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html[the AWS SDK documentation]. Example:
+
+[source,ruby]
+    input {
+      s3 {
+        access_key_id => "1234"
+        secret_access_key => "secret"
+        bucket => "logstash-test"
+        additional_settings => {
+          force_path_style => true
+          follow_redirects => false
+        }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-backup_add_prefix"]
+===== `backup_add_prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Append a prefix to the key (full path including file name in s3) after processing.
+If backing up to another (or the same) bucket, this effectively lets you
+choose a new 'folder' to place the files in
+
+[id="plugins-{type}s-{plugin}-backup_to_bucket"]
+===== `backup_to_bucket` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Name of a S3 bucket to backup processed files to.
+
+[id="plugins-{type}s-{plugin}-backup_to_dir"]
+===== `backup_to_dir` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Path of a local directory to backup processed files to.
+
+[id="plugins-{type}s-{plugin}-bucket"]
+===== `bucket` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the S3 bucket.
+
+[id="plugins-{type}s-{plugin}-delete"]
+===== `delete` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Whether to delete processed files from the original bucket.
+
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+  ** `disabled`: does not use ECS-compatible field names
+  ** `v1`,`v8`: uses metadata fields that are compatible with Elastic Common Schema
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
+
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+
+[id="plugins-{type}s-{plugin}-exclude_pattern"]
+===== `exclude_pattern` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Ruby style regexp of keys to exclude from the bucket.
+
+Note that files matching the pattern are skipped _after_ they have been listed. 
+Consider using <<plugins-{type}s-{plugin}-prefix>> instead where possible.
+
+Example:
+
+[source,ruby]
+-----
+"exclude_pattern" => "\/2020\/04\/"
+-----
+
+This pattern excludes all logs containing "/2020/04/" in the path.
+
+
+[id="plugins-{type}s-{plugin}-gzip_pattern"]
+===== `gzip_pattern`
+
+  * Value type is <<string,string>>
+  * Default value is `"\.gz(ip)?$"`
+
+Regular expression used to determine whether an input file is in gzip format.
+
+[id="plugins-{type}s-{plugin}-include_object_properties"]
+===== `include_object_properties`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Whether or not to include the S3 object's properties (last_modified, content_type, metadata) into each Event at
+`[@metadata][s3]`. Regardless of this setting, `[@metadata][s3][key]` will always be present.
+
+[id="plugins-{type}s-{plugin}-interval"]
+===== `interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+Interval to wait between to check the file list again after a run is finished.
+Value is in seconds.
+
+[id="plugins-{type}s-{plugin}-prefix"]
+===== `prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+If specified, the prefix of filenames in the bucket must match (not a regexp)
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value type is <<string,string>>
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+Session name to use when assuming an IAM role.
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-sincedb_path"]
+===== `sincedb_path` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Where to write the since database (keeps track of the date
+the last handled file was added to S3). The default will write
+sincedb files to in the directory '{path.data}/plugins/inputs/s3/'
+
+If specified, this setting must be a filename path and not just a directory.
+
+[id="plugins-{type}s-{plugin}-temporary_directory"]
+===== `temporary_directory` 
+
+  * Value type is <<string,string>>
+  * Default value is `"/tmp/logstash"`
+
+Set the directory where logstash will store the tmp files before processing them.
+
+[id="plugins-{type}s-{plugin}-watch_for_new_files"]
+===== `watch_for_new_files`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Whether or not to watch for new files.
+Disabling this option causes the input to close itself after processing the files from a single listing.
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!: