fluent-plugin-cloudwatch-logs 0.7.4 → 0.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1889a17c18ac0d47b2f8d4fb6e13cebd381743275698119fbdd2cfcd6742a752
-  data.tar.gz: cbde319d241346c6935429887547ffa1ff45aedf3df134156a19bf825bb6523b
+  metadata.gz: ec1239dd4ccfdea1b4989f51bd5949ab73fc6864d68716266e3fd536f06a1958
+  data.tar.gz: cf34cad429a40c2f3d78a659dc23ebdad49b57aeef39a0a166a4b82bfe820ed9
 SHA512:
-  metadata.gz: 4827ac8cdfb0aab178aa9590c356e17a3476310383201ae1bc585d447b81143bfdc78907b2bd08615653af77c41bbc401bcb8a12bab3aa5f2c1415c25794afd8
-  data.tar.gz: 5a7472ba731deea05d57f7e584ee5cac64a0be2368c22d38b61be2a5b1992366182592d2c191d1fdc0c85bad58036bad51c2bb99c8d3b5c05a9362aa582b70af
+  metadata.gz: 88fd26f6f050cb2b776edbe5938392a1e7f8fc07b113836db98cafd40cb490f439a6bc097822d70557bcf40924e5f15056acb107953ceaeb722d79c1dea5d1b3
+  data.tar.gz: a83e07ae68fddcfc81788a56964c3926176ee44db1b4a92a65a169fcf5692807ad39acab639814c7bd419dd42cfaa53da9179299db74bd6fe1dbc37a8699bedd

data/README.md

@@ -15,7 +15,9 @@
 
 ## Installation
 
-    $ gem install fluent-plugin-cloudwatch-logs
+```sh
+gem install fluent-plugin-cloudwatch-logs
+```
 
 ## Preparation
 
@@ -40,39 +42,62 @@ Create IAM user with a policy like the following:
 }
 ```
 
-Set region and credentials:
+## Authentication
+
+There are several methods to provide authentication credentials.  Be aware that there are various tradeoffs for these methods,
+although most of these tradeoffs are highly dependent on the specific environment.
+
+### Environment 
 
+Set region and credentials via the environment:
+
+```sh
+export AWS_REGION=us-east-1
+export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
+export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
 ```
-$ export AWS_REGION=us-east-1
-$ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
-$ export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
+
+Note: For this to work persistently the enviornment will need to be set in the startup scripts or docker variables.
+
+### AWS Configuration
+
+The plugin will look for the `$HOME/.aws/config` and `$HOME/.aws/credentials` for configuration information.  To setup, as the 
+fluentd user, run:
+
+```sh
+aws configure
 ```
 
+### Configuration Parameters
+
+The authentication information can also be set 
+
 ## Example
 
 Start fluentd:
 
-```
-$ fluentd -c example/fluentd.conf
+```sh
+fluentd -c example/fluentd.conf
 ```
 
 Send sample log to CloudWatch Logs:
 
-```
-$ echo '{"hello":"world"}' | fluent-cat test.cloudwatch_logs.out
+```sh
+echo '{"hello":"world"}' | fluent-cat test.cloudwatch_logs.out
 ```
 
 Fetch sample log from CloudWatch Logs:
 
-```
+```sh
 # stdout
 2014-07-17 00:28:02 +0900 test.cloudwatch_logs.in: {"hello":"world"}
 ```
 
 ## Configuration
+
 ### out_cloudwatch_logs
 
-```
+```aconf
 <match tag>
   @type cloudwatch_logs
   log_group_name log-group-name
@@ -98,6 +123,8 @@ Fetch sample log from CloudWatch Logs:
 ```
 
 * `auto_create_stream`: to create log group and stream automatically. (defaults to false)
+* `aws_key_id`: AWS Access Key.  See [Authentication](#authentication) for more information.
+* `aws_sec_key`: AWS Secret Access Key.  See [Authentication](#authentication) for more information.
 * `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1)
 * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
 * `http_proxy`: use to set an optional HTTP proxy
@@ -117,6 +144,7 @@ Fetch sample log from CloudWatch Logs:
 * `put_log_events_disable_retry_limit`: if true, `put_log_events_retry_limit` will be ignored
 * `put_log_events_retry_limit`: maximum count of retry (if exceeding this, the events will be discarded)
 * `put_log_events_retry_wait`: time before retrying PutLogEvents (retry interval increases exponentially like `put_log_events_retry_wait * (2 ^ retry_count)`)
+* `region`: AWS Region.  See [Authentication](#authentication) for more information.
 * `remove_log_group_aws_tags_key`: remove field specified by `log_group_aws_tags_key`
 * `remove_log_group_name_key`: remove field specified by `log_group_name_key`
 * `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
@@ -126,9 +154,12 @@ Fetch sample log from CloudWatch Logs:
 * `use_tag_as_group`: to use tag as a group name
 * `use_tag_as_stream`: to use tag as a stream name
 
+**NOTE:** `retention_in_days` requests additional IAM permission `logs:PutRetentionPolicy` for log_group.
+Please refer to [the PutRetentionPolicy column in documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html) for details.
+
 ### in_cloudwatch_logs
 
-```
+```aconf
 <source>
   @type cloudwatch_logs
   tag cloudwatch.in
@@ -141,6 +172,8 @@ Fetch sample log from CloudWatch Logs:
 </source>
 ```
 
+* `aws_key_id`: AWS Access Key.  See [Authentication](#authentication) for more information.
+* `aws_sec_key`: AWS Secret Access Key.  See [Authentication](#authentication) for more information.
 * `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
 * `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
 * `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
@@ -150,6 +183,7 @@ Fetch sample log from CloudWatch Logs:
 * `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
 * `log_group_name`: name of log group to fetch logs
 * `log_stream_name`: name of log stream to fetch logs
+* `region`: AWS Region.  See [Authentication](#authentication) for more information.
 * `state_file`: file to store current state (e.g. next\_forward\_token)
 * `tag`: fluentd tag
 * `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
@@ -160,7 +194,7 @@ Fetch sample log from CloudWatch Logs:
 
 Set credentials:
 
-```
+```aconf
 $ export AWS_REGION=us-east-1
 $ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
 $ export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_KEY"
@@ -168,29 +202,30 @@ $ export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_KEY"
 
 Run tests:
 
-```
-$ rake test
+```sh
+rake test
 ```
 
 Or, If you do not want to use IAM roll or ENV(this is just like writing to configuration file) :
 
-```
-$ rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 test
+```sh
+rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 test
 ```
 
 If you want to run the test suite against a mock server, set `endpoint` as below:
 
-```
-$ export endpoint='http://localhost:5000/'
-$ rake test
+```sh
+export endpoint='http://localhost:5000/'
+rake test
 ```
 
 
 ## Caution
 
-- If an event message exceeds API limit (256KB), the event will be discarded.
+If an event message exceeds API limit (256KB), the event will be discarded.
 
 ## Cross-Account Operation
+
 In order to have an instance of this plugin running in one AWS account to fetch logs from another account cross-account IAM authentication is required. Whilst this can be accomplished by configuring specific instances of the plugin manually with credentials for the source account in question this is not desirable for a number of reasons.
 
 In this case IAM can be used to allow the fluentd instance in one account ("A") to ingest Cloudwatch logs from another ("B") via the following mechanic:
@@ -204,7 +239,7 @@ In this case IAM can be used to allow the fluentd instance in one account ("A")
 * Create an IAM role `cloudwatch`
 * Attach a policy to allow the role holder to assume another role (where `ACCOUNT-B` is substituted for the appropriate account number):
 
-```
+```json
 {
     "Version": "2012-10-17",
     "Statement": [
@@ -228,7 +263,7 @@ In this case IAM can be used to allow the fluentd instance in one account ("A")
 * Create an IAM role `fluentd`
 * Ensure the `fluentd` role as account "A" as a trusted entity:
 
-```
+```json
 {
   "Version": "2012-10-17",
   "Statement": [
@@ -245,7 +280,7 @@ In this case IAM can be used to allow the fluentd instance in one account ("A")
 
 * Attach a policy:
 
-```
+```json
 {
     "Version": "2012-10-17",
     "Statement": [
@@ -270,7 +305,8 @@ In this case IAM can be used to allow the fluentd instance in one account ("A")
 ```
 
 ### Configuring the plugin for STS authentication
-```
+
+```aconf
 <source>
   @type cloudwatch_logs
   region us-east-1      # You must supply a region

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.7.4"
+        VERSION = "0.7.5"
       end
     end
   end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -84,6 +84,8 @@ module Fluent::Plugin
       super
 
       options = {}
+      options[:logger] = log if log
+      options[:log_level] = ({0 => :trace, 1 => :debug, 2 => :info, 3 => :warn, 4 => :error, 5 => :fatal}[log.level] || :info) if log
       options[:region] = @region if @region
       options[:endpoint] = @endpoint if @endpoint
       options[:instance_profile_credentials_retries] = @aws_instance_profile_credentials_retries if @aws_instance_profile_credentials_retries
@@ -102,6 +104,8 @@ module Fluent::Plugin
       @sequence_tokens = {}
       @store_next_sequence_token_mutex = Mutex.new
 
+      log.debug "Aws::CloudWatchLogs::Client initialized: log.level #{log.level} => #{options[:log_level]}"
+
       @json_handler = case @json_handler
                       when :yajl
                         Yajl
@@ -259,6 +263,9 @@ module Fluent::Plugin
       when Array
         record.each {|v| scrub_record!(v) }
       when String
+        # The AWS API requires UTF-8 encoding
+        # https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatchLogsConcepts.html
+        record.force_encoding('UTF-8')
         record.scrub!
       end
     end
@@ -286,7 +293,7 @@ module Fluent::Plugin
       while event = events.shift
         event_bytesize = event[:message].bytesize + EVENT_HEADER_SIZE
         if MAX_EVENT_SIZE < event_bytesize
-          log.warn "Log event is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
+          log.warn "Log event in #{group_name} is discarded because it is too large: #{event_bytesize} bytes exceeds limit of #{MAX_EVENT_SIZE}"
           break
         end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.5
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-16 00:00:00.000000000 Z
+date: 2019-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -150,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd