fluent-plugin-cloudwatch-logs 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17e367fb1fc1c2b58cd9a22cd9caefaf6396b3542f589b44616f160c40c3219d
-  data.tar.gz: 37eee2b6a48d17d86451a8eee88435fc1ddd572248b3ed181bf3d833d97a3d96
+  metadata.gz: ecdab23fc3b9c43ff71a49331694a6f9f88b0a7a27c70259b1a5b4c55894744d
+  data.tar.gz: 4197f274338c7134601dccfbb3e56b8672923b602f59e8fccc2d3ed43c9e4c6b
 SHA512:
-  metadata.gz: 30807c60113fe1eed915092e005e83863a4cfc5911a859eb00925299a1a4e45dc0dd581006d0486d3ab176ed2babf7b1fb5e90f0b41e04838c855a578db31faf
-  data.tar.gz: a44051e6aa3f1010dd5ee17f1046dfccabd59a929ed5b8976eeda135410f681400f1f024c61a84992cc28500285148e598aad7aa2daab10a965aca0204398a28
+  metadata.gz: 228c5d942bba9b410f926136c59a5ad65482580d67076f8703e5dbf7b1cd778beb447b2c175d07c20d20616a2dad8d74607db2665393387edda0e534a7da8362
+  data.tar.gz: 6e3cd9935e8c64b390877240a18175d8b1dd0ea3df34300658ebe66df22d728e0a45beba10ac8cbc63f9159d5a725abedcd4f62cdd7b244bd5c25e17a812153a

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/windows.yml

@@ -0,0 +1,26 @@
+name: Testing on Windows
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        os:
+          - windows-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/README.md

@@ -16,10 +16,18 @@
 
 ## Installation
 
+### For Fluentd
+
 ```sh
 gem install fluent-plugin-cloudwatch-logs
 ```
 
+### For td-agent
+
+```sh
+td-agent-gem install fluent-plugin-cloudwatch-logs
+```
+
 ## Preparation
 
 Create IAM user with a policy like the following:
@@ -444,10 +452,8 @@ In more detail, please refer to [the officilal document for built-in placeholder
 
 * out_cloudwatch_logs
   * if the data is too big for API, split into multiple requests
-  * format
   * check data size
 * in_cloudwatch_logs
-  * format
   * fallback to start_time because next_token expires after 24 hours
 
 ## Contributing

data/fluent-plugin-cloudwatch-logs.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "mocha"
+  spec.add_development_dependency "nokogiri"
 end

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.13.0"
+        VERSION = "0.14.0"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -17,6 +17,9 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_external_id, :string, default: nil
+    config_param :aws_sts_policy, :string, default: nil
+    config_param :aws_sts_duration_seconds, :time, default: nil
     config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, default: nil
     config_param :endpoint, :string, default: nil
@@ -88,7 +91,10 @@ module Fluent::Plugin
         Aws.config[:region] = options[:region]
         credentials_options = {
           role_arn: @aws_sts_role_arn,
-          role_session_name: @aws_sts_session_name
+          role_session_name: @aws_sts_session_name,
+          external_id: @aws_sts_external_id,
+          policy: @aws_sts_policy,
+          duration_seconds: @aws_sts_duration_seconds
         }
         credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
         if @region and @aws_sts_endpoint_url
@@ -185,7 +191,7 @@ module Fluent::Plugin
             if @use_log_stream_name_prefix || @use_todays_log_stream
               log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
               begin
-                log_streams = describe_log_streams(log_stream_name_prefix)
+                log_streams = describe_log_streams(log_stream_name_prefix, nil, nil, log_group_name)
                 log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
                 log_streams.each do |log_stream|
                   log_stream_name = log_stream.log_stream_name
@@ -193,7 +199,7 @@ module Fluent::Plugin
                   metadata = if @include_metadata
                                {
                                  "log_stream_name" => log_stream_name,
-                                 "log_group_name" => @log_group_name
+                                 "log_group_name" => log_group_name
                                }
                              else
                                {}
@@ -221,8 +227,8 @@ module Fluent::Plugin
               end
             end
           end
-        sleep 1
         end
+        sleep 1
       end
     end
 
@@ -266,8 +272,13 @@ module Fluent::Plugin
         }
         request.merge!(start_time: @start_time) if @start_time
         request.merge!(end_time: @end_time) if @end_time
-        log_next_token = next_token(log_group_name, log_stream_name)
+        if @use_log_group_name_prefix
+          log_next_token = next_token(log_stream_name, log_group_name)
+        else
+          log_next_token = next_token(log_stream_name)
+        end
         request[:next_token] = log_next_token if !log_next_token.nil? && !log_next_token.empty?
+        request[:start_from_head] = true if read_from_head?(log_next_token)
         response = @logs.get_log_events(request)
         if valid_next_token(log_next_token, response.next_forward_token)
           if @use_log_group_name_prefix
@@ -281,10 +292,14 @@ module Fluent::Plugin
       end
     end
 
-    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil)
+    def read_from_head?(next_token)
+      (!next_token.nil? && !next_token.empty?) || @start_time || @end_time
+    end
+
+    def describe_log_streams(log_stream_name_prefix, log_streams = nil, next_token = nil, log_group_name=nil)
       throttling_handler('describe_log_streams') do
         request = {
-          log_group_name: @log_group_name
+          log_group_name: log_group_name != nil ? log_group_name : @log_group_name
         }
         request[:next_token] = next_token if next_token
         request[:log_stream_name_prefix] = log_stream_name_prefix if log_stream_name_prefix
@@ -295,7 +310,7 @@ module Fluent::Plugin
           log_streams = response.log_streams
         end
         if response.next_token
-          log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token)
+          log_streams = describe_log_streams(log_stream_name_prefix, log_streams, response.next_token, log_group_name)
         end
         log_streams
       end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -19,6 +19,9 @@ module Fluent::Plugin
     config_param :aws_use_sts, :bool, default: false
     config_param :aws_sts_role_arn, :string, default: nil
     config_param :aws_sts_session_name, :string, default: 'fluentd'
+    config_param :aws_sts_external_id, :string, default: nil
+    config_param :aws_sts_policy, :string, default: nil
+    config_param :aws_sts_duration_seconds, :time, default: nil
     config_param :aws_sts_endpoint_url, :string, default: nil
     config_param :region, :string, :default => nil
     config_param :endpoint, :string, :default => nil
@@ -123,7 +126,10 @@ module Fluent::Plugin
         Aws.config[:region] = options[:region]
         credentials_options = {
           role_arn: @aws_sts_role_arn,
-          role_session_name: @aws_sts_session_name
+          role_session_name: @aws_sts_session_name,
+          external_id: @aws_sts_external_id,
+          policy: @aws_sts_policy,
+          duration_seconds: @aws_sts_duration_seconds
         }
         credentials_options[:sts_endpoint_url] = @aws_sts_endpoint_url if @aws_sts_endpoint_url
         if @region and @aws_sts_endpoint_url

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Ryota Arai
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-08 00:00:00.000000000 Z
+date: 2021-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - ryota.arai@gmail.com
@@ -116,8 +130,9 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/issue-auto-closer.yml"
+- ".github/workflows/linux.yml"
+- ".github/workflows/windows.yml"
 - ".gitignore"
-- ".travis.yml"
 - Gemfile
 - ISSUE_TEMPLATE.md
 - LICENSE.txt
@@ -151,7 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.15
 signing_key:
 specification_version: 4
 summary: CloudWatch Logs Plugin for Fluentd

data/.travis.yml

@@ -1,8 +0,0 @@
-sudo: false
-language: ruby
-
-rvm:
-  - 2.6
-  - 2.5
-  - 2.4
-  - 2.3