fluent-plugin-cloudwatch-logs 0.13.2 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51935a06f74f11a53f106e38e4f08132a7342d6509dc38c0d468d54e19e5fa28
-  data.tar.gz: 738e36cb222adca31a2f79e56ec71cc691f26468eb8025bb9d61773d0dd9d633
+  metadata.gz: 9e5c502cb8ad443406678f2c8ce7f6c8ea031c75f18f502d2b85ee1401e42faf
+  data.tar.gz: 3a06cf5ea24ef24bab2078ed216930508a2bd51940d659ff104c2de1515072db
 SHA512:
-  metadata.gz: 06675b2f4beedcf0d39c79bf93c59586c0d11816b97a22ee3f472fa6fc892e666c96f8600cfc8f98fba3b2b82f40dd452ec9b451caf6170da199fcf71ad42bff
-  data.tar.gz: 294a5c7fe16c450a58478b982cba8d174d603885234619b5631c1da314b5b7b8a0874424d2b5c997a4790333f99739ecac3cd47a4af97836bf15c9da90ed25af
+  metadata.gz: 2e5e8033091ea7f2f461095271b64ede8b5d807c4a7215ba403b25c7f0028bdc0d68ad4a2ed3171f9ea50b9997902e52140bdd6587c8d67af1a925f7dc5be726
+  data.tar.gz: 21d9c62b75f66b6fb8e5b65915e91dd26696612e8439ea3ffd65c2b2fff7fd169556e47b8d451874d0134c97d5523c5d8afe5f1f052bc016b08c6ad147cecd8f

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/README.md

@@ -16,10 +16,18 @@
 
 ## Installation
 
+### For Fluentd
+
 ```sh
 gem install fluent-plugin-cloudwatch-logs
 ```
 
+### For td-agent
+
+```sh
+td-agent-gem install fluent-plugin-cloudwatch-logs
+```
+
 ## Preparation
 
 Create IAM user with a policy like the following:
@@ -98,7 +106,7 @@ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
 export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
 ```
 
-Note: For this to work persistently the enviornment will need to be set in the startup scripts or docker variables.
+Note: For this to work persistently the environment will need to be set in the startup scripts or docker variables.
 
 ### AWS Configuration
 
@@ -176,6 +184,7 @@ Fetch sample log from CloudWatch Logs:
 * `aws_sec_key`: AWS Secret Access Key.  See [Authentication](#authentication) for more information.
 * `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1)
 * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
+* `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
 * `http_proxy`: use to set an optional HTTP proxy
 * `include_time_key`: include time key as part of the log entry (defaults to UTC)
 * `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
@@ -255,6 +264,7 @@ Please refer to [the PutRetentionPolicy column in documentation](https://docs.aw
 * `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
 * `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
 * `endpoint`: use this parameter to connect to the local API endpoint (for testing)
+* `ssl_verify_peer`: when `true` (default), SSL peer certificates are verified when establishing a connection. Setting to `false` can be useful for testing.
 * `fetch_interval`: time period in seconds between checking CloudWatch for new logs. (default: 60)
 * `http_proxy`: use to set an optional HTTP proxy
 * `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
@@ -444,10 +454,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
 
 * out_cloudwatch_logs
   * if the data is too big for API, split into multiple requests
-  * format
   * check data size
 * in_cloudwatch_logs
-  * format
   * fallback to start_time because next_token expires after 24 hours
 
 ## Contributing

data/fluent-plugin-cloudwatch-logs.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "mocha"
+  spec.add_development_dependency "nokogiri"
 end

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.13.2"
+        VERSION = "0.14.1"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -17,9 +17,13 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_external_id, :string, default: nil
+    config_param :aws_sts_policy, :string, default: nil
+    config_param :aws_sts_duration_seconds, :time, default: nil
     config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, default: nil
     config_param :endpoint, :string, default: nil
+    config_param :ssl_verify_peer, :bool, :default => true
     config_param :tag, :string
     config_param :log_group_name, :string
     config_param :add_log_group_name, :bool, default: false
@@ -28,7 +32,7 @@ module Fluent::Plugin
     config_param :log_stream_name, :string, default: nil
     config_param :use_log_stream_name_prefix, :bool, default: false
     config_param :state_file, :string, default: nil,
-                 deprecated: "Use <stroage> instead."
+                 deprecated: "Use <storage> instead."
     config_param :fetch_interval, :time, default: 60
     config_param :http_proxy, :string, default: nil
     config_param :json_handler, :enum, list: [:yajl, :json], default: :yajl
@@ -82,13 +86,17 @@ module Fluent::Plugin
       options = {}
       options[:region] = @region if @region
       options[:endpoint] = @endpoint if @endpoint
+      options[:ssl_verify_peer] = @ssl_verify_peer
       options[:http_proxy] = @http_proxy if @http_proxy
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
         credentials_options = {
           role_arn: @aws_sts_role_arn,
-          role_session_name: @aws_sts_session_name
+          role_session_name: @aws_sts_session_name,
+          external_id: @aws_sts_external_id,
+          policy: @aws_sts_policy,
+          duration_seconds: @aws_sts_duration_seconds
         }
         credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
         if @region and @aws_sts_endpoint_url
@@ -221,8 +229,8 @@ module Fluent::Plugin
               end
             end
           end
-        sleep 1
         end
+        sleep 1
       end
     end
 
@@ -266,8 +274,13 @@ module Fluent::Plugin
         }
         request.merge!(start_time: @start_time) if @start_time
         request.merge!(end_time: @end_time) if @end_time
-        log_next_token = next_token(log_group_name, log_stream_name)
+        if @use_log_group_name_prefix
+          log_next_token = next_token(log_stream_name, log_group_name)
+        else
+          log_next_token = next_token(log_stream_name)
+        end
         request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
+        request[:start_from_head] = true if read_from_head?(log_next_token)
         response = @logs.get_log_events(request)
         if valid_next_token(log_next_token, response.next_forward_token)
           if @use_log_group_name_prefix
@@ -281,6 +294,10 @@ module Fluent::Plugin
       end
     end
 
+    def read_from_head?(next_token)
+      (!next_token.nil? && !next_token.empty?) || @start_time || @end_time
+    end
+
     def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
       throttling_handler('describe_log_streams') do
         request = {

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -19,9 +19,13 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_external_id, :string, default: nil
+    config_param :aws_sts_policy, :string, default: nil
+    config_param :aws_sts_duration_seconds, :time, default: nil
     config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, :default => nil
     config_param :endpoint, :string, :default => nil
+    config_param :ssl_verify_peer, :bool, :default => true
     config_param :log_group_name, :string, :default => nil
     config_param :log_stream_name, :string, :default => nil
     config_param :auto_create_stream, :bool, default: false
@@ -117,13 +121,17 @@ module Fluent::Plugin
       options[:log_level] = :debug if log
       options[:region] = @region if @region
       options[:endpoint] = @endpoint if @endpoint
+      options[:ssl_verify_peer] = @ssl_verify_peer
       options[:instance_profile_credentials_retries] = @aws_instance_profile_credentials_retries if @aws_instance_profile_credentials_retries
 
       if @aws_use_sts
         Aws.config[:region] = options[:region]
         credentials_options = {
           role_arn: @aws_sts_role_arn,
-          role_session_name: @aws_sts_session_name
+          role_session_name: @aws_sts_session_name,
+          external_id: @aws_sts_external_id,
+          policy: @aws_sts_policy,
+          duration_seconds: @aws_sts_duration_seconds
         }
         credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
         if @region and @aws_sts_endpoint_url

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.13.2
+  version: 0.14.1
 platform: ruby
 authors:
 - Ryota Arai
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-18 00:00:00.000000000 Z
+date: 2021-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,7 +108,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
 email:
 - ryota.arai@gmail.com
 executables: []
@@ -119,7 +133,6 @@ files:
 - ".github/workflows/linux.yml"
 - ".github/workflows/windows.yml"
 - ".gitignore"
-- ".travis.yml"
 - Gemfile
 - ISSUE_TEMPLATE.md
 - LICENSE.txt
@@ -138,7 +151,7 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-log
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -153,8 +166,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.2.5
+signing_key: 
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd
 test_files:

data/.travis.yml

@@ -1,8 +0,0 @@
-sudo: false
-language: ruby
-
-rvm:
-  - 2.6
-  - 2.5
-  - 2.4
-  - 2.3