fluent-plugin-cloudwatch-logs 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d9202fe11ec14b83d2eaa50fa1797564123dd3dd20363381143955dacf89729
-  data.tar.gz: c054f3d52d90d7497f818160ab8421f8b69582884b32b6547f6073d7cff6ea16
+  metadata.gz: d6ac14d761ae8cfffe1fd6cc11abdc282861f8639924776117c6773a4599057a
+  data.tar.gz: 933cc8a97a346ade8287f5fbfc339aa2340a8982c3252cf822a9b95d70c0c78f
 SHA512:
-  metadata.gz: 3693fb0907a876f2f0fbaac363e67bfcce42cc2d5b356d0eeb5b2a41fff8e859a2ba3ee3f08977a7f39318a67a220e7489d9d5dce82e7e82bf234fad8e1b9b9f
-  data.tar.gz: 3bf4f873cd6104e26356790e5867dfa21d17b4d67e06dc7f02b07864b83c3f302c176995c3884fce587e1a201d9c373b256b874edd99414b08b597532eef2cf0
+  metadata.gz: 64c6c225b57fe907a6ac9e1941d3cb3a1c0356096aeda2e9213bee288e479c18ed33d73eea754a58474d60e818f6c6ce7f33f5efa0b339c663d3a84f743d8ccf
+  data.tar.gz: 5300d8d29f83242479ac2f38c8bdf26bacd3d8dd353f8b286eb593912671ec6143ad26ff3640affbd33293ae07cea1b9bf3d22dcbe0eb81acb2ffc5564cc0aec

data/ISSUE_TEMPLATE.md

@@ -0,0 +1,21 @@
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Provide example config and message
+
+#### Expected Behavior or What you need to ask
+
+...
+
+#### Using Fluentd and CloudWatchLogs plugin versions
+
+* OS version
+* Bare Metal or within Docker or Kubernetes or others?
+* Fluentd v0.12 or v0.14/v1.0
+  * paste result of ``fluentd --version`` or ``td-agent --version``
+* Dependent gem versions
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock

data/README.md

@@ -262,6 +262,30 @@ In this case IAM can be used to allow the fluentd instance in one account ("A")
 </source>
 ```
 
+### Using build-in placeholders, but they don't replace placeholders with actual values, why?
+
+Built-in placeholders use buffer metadata when replacing placeholders with actual values.
+So, you should specify buffer attributes what you want to replace placeholders with.
+
+Using `${tag}` placeholders, you should specify `tag` attributes in buffer:
+
+```aconf
+<buffer tag>
+  @type memory
+</buffer>
+```
+
+Using `%Y%m%d` placeholders, you should specify `time` attributes in buffer:
+
+```aconf
+<buffer time>
+  @type memory
+  timekey 3600
+</buffer>
+```
+
+In more detail, please refer to [the officilal document for built-in placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders).
+
 ## TODO
 
 * out_cloudwatch_logs

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -2,7 +2,7 @@ module Fluent
   module Plugin
     module Cloudwatch
       module Logs
-        VERSION = "0.7.0"
+        VERSION = "0.7.1"
       end
     end
   end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -109,14 +109,19 @@ module Fluent::Plugin
 
           if @use_log_stream_name_prefix || @use_todays_log_stream
             log_stream_name_prefix = @use_todays_log_stream ? get_todays_date : @log_stream_name
-            log_streams = describe_log_streams(log_stream_name_prefix)
-            log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
-            log_streams.each do |log_stream|
-              log_stream_name = log_stream.log_stream_name
-              events = get_events(log_stream_name)
-              events.each do |event|
-                emit(log_stream_name, event)
+            begin
+              log_streams = describe_log_streams(log_stream_name_prefix)
+              log_streams.concat(describe_log_streams(get_yesterdays_date)) if @use_todays_log_stream
+              log_streams.each do |log_stream|
+                log_stream_name = log_stream.log_stream_name
+                events = get_events(log_stream_name)
+                events.each do |event|
+                  emit(log_stream_name, event)
+                end
               end
+            rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+              log.warn "'#{@log_stream_name}' prefixed log stream(s) are not found"
+              next
             end
           else
             events = get_events(@log_stream_name)

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -257,6 +257,82 @@ class CloudwatchLogsOutputTest < Test::Unit::TestCase
     assert_equal('message2 logs2', events[1].message)
   end
 
+  def test_write_use_placeholders_parts
+    new_log_stream
+
+    config = {'@type' => 'cloudwatch_logs',
+              'auto_create_stream' => true,
+              'message_keys' => ["message","cloudwatch"],
+              'log_stream_name' => "${tag[0]}-${tag[1]}-${tag[2]}-${tag[3]}",
+              'log_group_name' => log_group_name}
+    config.merge!(config_elementify(aws_key_id)) if aws_key_id
+    config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+    config.merge!(config_elementify(region)) if region
+    config.merge!(config_elementify(endpoint)) if endpoint
+
+    d = create_driver(
+      Fluent::Config::Element.new('ROOT', '', config,[
+                                    Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                  '@type' => 'memory',
+                                                                  'timekey' => 3600
+                                                                }, [])
+                                  ])
+    )
+
+    time = event_time
+    d.run(default_tag: fluentd_tag) do
+      d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
+      d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
+    end
+
+    sleep 10
+
+    events = get_log_events(log_group_name, 'fluent-plugin-cloudwatch-test')
+    assert_equal(2, events.size)
+    assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+    assert_equal('message1 logs1', events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal('message2 logs2', events[1].message)
+  end
+
+  def test_write_use_time_placeholders
+    new_log_stream
+
+    config = {'@type' => 'cloudwatch_logs',
+              'auto_create_stream' => true,
+              'message_keys' => ["message","cloudwatch"],
+              'log_stream_name' => "fluent-plugin-cloudwatch-test-%Y%m%d",
+              'log_group_name' => log_group_name}
+    config.merge!(config_elementify(aws_key_id)) if aws_key_id
+    config.merge!(config_elementify(aws_sec_key)) if aws_sec_key
+    config.merge!(config_elementify(region)) if region
+    config.merge!(config_elementify(endpoint)) if endpoint
+
+    d = create_driver(
+      Fluent::Config::Element.new('ROOT', '', config,[
+                                    Fluent::Config::Element.new('buffer', 'tag, time', {
+                                                                  '@type' => 'memory',
+                                                                  'timekey' => 3600
+                                                                }, [])
+                                  ])
+    )
+
+    time = event_time
+    d.run(default_tag: fluentd_tag) do
+      d.feed(time, {'cloudwatch' => 'logs1', 'message' => 'message1'})
+      d.feed(time + 1, {'cloudwatch' => 'logs2', 'message' => 'message2'})
+    end
+
+    sleep 10
+
+    events = get_log_events(log_group_name, "fluent-plugin-cloudwatch-test-#{Time.at(time).strftime("%Y%m%d")}")
+    assert_equal(2, events.size)
+    assert_equal((time.to_f * 1000).floor, events[0].timestamp)
+    assert_equal('message1 logs1', events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal('message2 logs2', events[1].message)
+  end
+
   def test_include_time_key
     new_log_stream
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-logs
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Ryota Arai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-27 00:00:00.000000000 Z
+date: 2018-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -103,6 +103,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - Gemfile
+- ISSUE_TEMPLATE.md
 - LICENSE.txt
 - README.md
 - Rakefile