rb-fluent-plugin-cloudwatch-logs 0.7.1.pre.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cc1f0f04c8474e79922dc8a8ed0536d0d36ef9c039a281e6eb98cb35770ce736
+  data.tar.gz: 9f67a5fe54c8c0b788c742ffd6867920585a99643a335fb19b78305df825fd48
+SHA512:
+  metadata.gz: 20c89a6c3e89ce26dccffd4bfc7096af60c4ec601ad975b5153784c51739bd6ef450b61e6af27747347b86a9290074551c4eae671c99d6365a32c72fe86f87f1
+  data.tar.gz: 6f31d33d83fe96d66dbf8d63cb563a6cdfdc6a93896ab2db865672a925d7ebfb2f12949c2044ee26de8d011427be944595a5a3645edc4bd2c22ff860951470c8

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-cloudwatch-logs.gemspec
+gemspec

data/ISSUE_TEMPLATE.md

@@ -0,0 +1,21 @@
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Provide example config and message
+
+#### Expected Behavior or What you need to ask
+
+...
+
+#### Using Fluentd and CloudWatchLogs plugin versions
+
+* OS version
+* Bare Metal or within Docker or Kubernetes or others?
+* Fluentd v0.12 or v0.14/v1.0
+  * paste result of ``fluentd --version`` or ``td-agent --version``
+* Dependent gem versions
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Ryota Arai
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,314 @@
+# fluent-plugin-cloudwatch-logs
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-cloudwatch-logs.svg)](http://badge.fury.io/rb/fluent-plugin-cloudwatch-logs)
+
+[CloudWatch Logs](http://aws.amazon.com/blogs/aws/cloudwatch-log-service/) Plugin for Fluentd
+
+## Requirements
+
+|fluent-plugin-cloudwatch-logs|     fluentd      |  ruby  |
+|-----------------------------|------------------|--------|
+|     >= 0.5.0                | >= 0.14.15       | >= 2.1 |
+|     <= 0.4.5                | ~> 0.12.0 *      | >= 1.9 |
+
+* May not support all future fluentd features
+
+## Installation
+
+    $ gem install fluent-plugin-cloudwatch-logs
+
+## Preparation
+
+Create IAM user with a policy like the following:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:*",
+        "s3:GetObject"
+      ],
+      "Resource": [
+        "arn:aws:logs:us-east-1:*:*",
+        "arn:aws:s3:::*"
+      ]
+    }
+  ]
+}
+```
+
+Set region and credentials:
+
+```
+$ export AWS_REGION=us-east-1
+$ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
+$ export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY"
+```
+
+## Example
+
+Start fluentd:
+
+```
+$ fluentd -c example/fluentd.conf
+```
+
+Send sample log to CloudWatch Logs:
+
+```
+$ echo '{"hello":"world"}' | fluent-cat test.cloudwatch_logs.out
+```
+
+Fetch sample log from CloudWatch Logs:
+
+```
+# stdout
+2014-07-17 00:28:02 +0900 test.cloudwatch_logs.in: {"hello":"world"}
+```
+
+## Configuration
+### out_cloudwatch_logs
+
+```
+<match tag>
+  @type cloudwatch_logs
+  log_group_name log-group-name
+  log_stream_name log-stream-name
+  auto_create_stream true
+  #message_keys key1,key2,key3,...
+  #max_message_length 32768
+  #use_tag_as_group false
+  #use_tag_as_stream false
+  #include_time_key true
+  #localtime true
+  #log_group_name_key group_name_key
+  #log_stream_name_key stream_name_key
+  #remove_log_group_name_key true
+  #remove_log_stream_name_key true
+  #put_log_events_retry_wait 1s
+  #put_log_events_retry_limit 17
+  #put_log_events_disable_retry_limit false
+  #endpoint http://localhost:5000/
+  #json_handler json
+</match>
+```
+
+* `log_group_name`: name of log group to store logs
+* `log_stream_name`: name of log stream to store logs
+* `auto_create_stream`: to create log group and stream automatically
+* `message_keys`: keys to send messages as events
+* `max_message_length`: maximum length of the message
+* `max_events_per_batch`: maximum number of events to send at once (default 10000)
+* `use_tag_as_group`: to use tag as a group name
+* `use_tag_as_stream`: to use tag as a stream name
+* `include_time_key`: include time key as part of the log entry (defaults to UTC)
+* `localtime`: use localtime timezone for `include_time_key` output (overrides UTC default)
+* `log_group_name_key`: use specified field of records as log group name
+* `log_stream_name_key`: use specified field of records as log stream name
+* `remove_log_group_name_key`: remove field specified by `log_group_name_key`
+* `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
+* `put_log_events_retry_wait`: time before retrying PutLogEvents (retry interval increases exponentially like `put_log_events_retry_wait * (2 ^ retry_count)`)
+* `put_log_events_retry_limit`: maximum count of retry (if exceeding this, the events will be discarded)
+* `put_log_events_disable_retry_limit`: if true, `put_log_events_retry_limit` will be ignored
+* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
+* `json_handler`: name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
+
+### in_cloudwatch_logs
+
+```
+<source>
+  @type cloudwatch_logs
+  tag cloudwatch.in
+  log_group_name group
+  log_stream_name stream
+  #use_log_stream_name_prefix true
+  state_file /var/lib/fluent/group_stream.in.state
+  #endpoint http://localhost:5000/
+  #json_handler json
+</source>
+```
+
+* `tag`: fluentd tag
+* `log_group_name`: name of log group to fetch logs
+* `log_stream_name`: name of log stream to fetch logs
+* `use_log_stream_name_prefix`: to use `log_stream_name` as log stream name prefix (default false)
+* `state_file`: file to store current state (e.g. next\_forward\_token)
+* `endpoint`: use this parameter to connect to the local API endpoint (for testing)
+* `aws_use_sts`: use [AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html) to authenticate, rather than the [default credential hierarchy](http://docs.aws.amazon.com/sdkforruby/api/Aws/CloudWatchLogs/Client.html#initialize-instance_method). See 'Cross-Account Operation' below for more detail.
+* `aws_sts_role_arn`: the role ARN to assume when using cross-account sts authentication
+* `aws_sts_session_name`: the session name to use with sts authentication (default: `fluentd`)
+* `json_handler`:  name of the library to be used to handle JSON data. For now, supported libraries are `json` (default) and `yajl`.
+* `use_todays_log_stream`: use todays and yesterdays date as log stream name prefix (formatted YYYY/MM/DD). (default: `false`)
+
+## Test
+
+Set credentials:
+
+```
+$ export AWS_REGION=us-east-1
+$ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
+$ export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_KEY"
+```
+
+Run tests:
+
+```
+$ rake test
+```
+
+Or, If you do not want to use IAM roll or ENV(this is just like writing to configuration file) :
+
+```
+$ rake aws_key_id=YOUR_ACCESS_KEY aws_sec_key=YOUR_SECRET_KEY region=us-east-1 test
+```
+
+If you want to run the test suite against a mock server, set `endpoint` as below:
+
+```
+$ export endpoint='http://localhost:5000/'
+$ rake test
+```
+
+
+## Caution
+
+- If an event message exceeds API limit (256KB), the event will be discarded.
+
+## Cross-Account Operation
+In order to have an instance of this plugin running in one AWS account to fetch logs from another account cross-account IAM authentication is required. Whilst this can be accomplished by configuring specific instances of the plugin manually with credentials for the source account in question this is not desirable for a number of reasons.
+
+In this case IAM can be used to allow the fluentd instance in one account ("A") to ingest Cloudwatch logs from another ("B") via the following mechanic:
+
+* plugin instance running in account "A" has an IAM instance role assigned to the underlying EC2 instance
+* The IAM instance role and associated policies permit the EC2 instance to assume a role in another account
+* An IAM role in account "B" and associated policies allow read access to the Cloudwatch Logs service, as appropriate.
+
+### IAM Detail: Consuming Account "A"
+
+* Create an IAM role `cloudwatch`
+* Attach a policy to allow the role holder to assume another role (where `ACCOUNT-B` is substituted for the appropriate account number):
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "sts:*"
+            ],
+            "Resource": [
+                "arn:aws:iam::ACCOUNT-B:role/fluentd"
+            ]
+        }
+    ]
+}
+```
+
+* Ensure the EC2 instance on which this plugin is executing as role `cloudwatch` as its assigned IAM instance role.
+
+### IAM Detail: Log Source Account "B"
+
+* Create an IAM role `fluentd`
+* Ensure the `fluentd` role as account "A" as a trusted entity:
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "arn:aws:iam::ACCOUNT-A:root"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+```
+
+* Attach a policy:
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "logs:DescribeDestinations",
+                "logs:DescribeExportTasks",
+                "logs:DescribeLogGroups",
+                "logs:DescribeLogStreams",
+                "logs:DescribeMetricFilters",
+                "logs:DescribeSubscriptionFilters",
+                "logs:FilterLogEvents",
+                "logs:GetLogEvents"
+            ],
+            "Resource": [
+                "arn:aws:logs:eu-west-1:ACCOUNT-B:log-group:LOG_GROUP_NAME_FOR_CONSUMPTION:*"
+            ]
+        }
+    ]
+}
+```
+
+### Configuring the plugin for STS authentication
+```
+<source>
+  @type cloudwatch_logs
+  region us-east-1      # You must supply a region
+  aws_use_sts true
+  aws_sts_role_arn arn:aws:iam::ACCOUNT-B:role/fluentd
+  log_group_name LOG_GROUP_NAME_FOR_CONSUMPTION
+  log_stream_name SOME_PREFIX
+  use_log_stream_name_prefix true
+  state_file /path/to/state_file
+  format /(?<message>.+)/
+</source>
+```
+
+### Using build-in placeholders, but they don't replace placeholders with actual values, why?
+
+Built-in placeholders use buffer metadata when replacing placeholders with actual values.
+So, you should specify buffer attributes what you want to replace placeholders with.
+
+Using `${tag}` placeholders, you should specify `tag` attributes in buffer:
+
+```aconf
+<buffer tag>
+  @type memory
+</buffer>
+```
+
+Using `%Y%m%d` placeholders, you should specify `time` attributes in buffer:
+
+```aconf
+<buffer time>
+  @type memory
+  timekey 3600
+</buffer>
+```
+
+In more detail, please refer to [the officilal document for built-in placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders).
+
+## TODO
+
+* out_cloudwatch_logs
+  * if the data is too big for API, split into multiple requests
+  * format
+  * check data size
+* in_cloudwatch_logs
+  * format
+  * fallback to start_time because next_token expires after 24 hours
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/fluent-plugin-cloudwatch-logs/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+end
+

data/example/fluentd.conf

@@ -0,0 +1,23 @@
+<source>
+  @type forward
+</source>
+
+<source>
+  @type cloudwatch_logs
+  tag test.cloudwatch_logs.in
+  log_group_name fluent-plugin-cloudwatch-example
+  log_stream_name fluent-plugin-cloudwatch-example
+  state_file /tmp/fluent-plugin-cloudwatch-example.state
+</source>
+
+<match test.cloudwatch_logs.out>
+  @type cloudwatch_logs
+  log_group_name fluent-plugin-cloudwatch-example
+  log_stream_name fluent-plugin-cloudwatch-example
+  auto_create_stream true
+</match>
+
+<match test.cloudwatch_logs.in>
+  @type stdout
+</match>
+

data/fluent-plugin-cloudwatch-logs.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/cloudwatch/logs/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rb-fluent-plugin-cloudwatch-logs"
+  spec.version       = Fluent::Plugin::Cloudwatch::Logs::VERSION
+  spec.authors       = ["Ryota Arai", "Other People"]
+  spec.email         = ["ryota.arai@gmail.com"]
+  spec.summary       = %q{CloudWatch Logs Plugin for Fluentd with memory profiling}
+  spec.homepage      = "https://github.com/redbubble/fluent-plugin-cloudwatch-logs"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'fluentd', '>= 0.14.15'
+  spec.add_dependency 'aws-sdk-cloudwatchlogs', '~> 1.0'
+  spec.add_dependency 'memory_profiler'
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "mocha"
+end