fluent-plugin-ecs-metadata-filter-v2 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.codeclimate.yml +3 -0
- data/.gitignore +10 -0
- data/.rubocop.yml +9 -0
- data/.travis.yml +26 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +11 -0
- data/LICENSE.txt +21 -0
- data/README.md +66 -0
- data/Rakefile +10 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/fluent_plugin_ecs_metadata_filter.gemspec +33 -0
- data/lib/fluent/plugin/filter_ecs_metadata.rb +95 -0
- data/lib/fluent_ecs/configurable.rb +17 -0
- data/lib/fluent_ecs/container.rb +60 -0
- data/lib/fluent_ecs/error.rb +5 -0
- data/lib/fluent_ecs/metadata.rb +33 -0
- data/lib/fluent_ecs/resource.rb +28 -0
- data/lib/fluent_ecs/task.rb +24 -0
- data/lib/fluent_ecs/version.rb +3 -0
- data/lib/fluent_ecs.rb +11 -0
- metadata +208 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 30a0dac540e0662f7050ad365a8287b64c22a28ec26ef5ecef466de266228c4b
|
4
|
+
data.tar.gz: 554785e3a2011fd0bbe35184b664fb265058bab36eab17d17d451c10bd4274fa
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3486dd8698e1741760453b9541187cab1479bcd2c88cec8f3e4c7d11beb0b2ed18ce83bd903148b76e58a42675ebf0490789fcef43f7d1b43f82bc0b19dc8c1f
|
7
|
+
data.tar.gz: 32a1435a57ac04cb4a7f756bc8202a303fd8646798ba20dd698f692d77bf6078e535b85bbcb7f9eb9031c6395db416f40168b653ca423138e5c2c09dfa442f73
|
data/.codeclimate.yml
ADDED
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
data/.travis.yml
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
language: ruby
|
2
|
+
sudo: false
|
3
|
+
|
4
|
+
cache:
|
5
|
+
bundler: true
|
6
|
+
|
7
|
+
addons:
|
8
|
+
code_climate:
|
9
|
+
repo_token:
|
10
|
+
secure: DfwrADIFzZWfRSA5Kebg7ij0WNMhC22v5GHBpkTmMgk68iXzQZnUayAwNsoe8cVK3xVpxKK8dTa+pxA2GvSNGV3JBqdTkC+h7WThuoGSqVMqpiJKv1AnA7jH6MfAMNo+p8v/WyICvCTnxuEQ06E8jPEyR7XZWnDNACMGmQgQ+0XZg+BIP4KmgQ2VdOyEqqyF3F/xwmYmQaLL0/AAGGueoQYtRrJOA/5APu7Btu0Blf2QAC/rk9Ae1FjvSXZjMkyAdhHT0KaNHRRVN6rqkXHtF5W9zCToo9TinsPADijhSg0yeMQZkB7FbbTbUyBDRHNbQI1pUjIhuv0Ha43TS5D4CE4MhbNsshPvHiMbXjW4QM2h9LcQ7SN2j5M2Vs8ykr/nuHBjb8Zr/fTcCi6IBC2rJLUBfHdz1OrnX+sWsB19vUO+cTC9djc+SJEKPWKY5T70UjP7AAo55X5PTPMsL6Wc0DjA31ELxRw9lS1Kkkp9Kgs0pVRd4miAszgDknkHBooHqC/sqbHjhc1y2oeaVBoFoX4LvZZTox6tFIWwYITUpLetKPiS+6Rs5vz14zO/IXDrpFN5l/iSd94kWtGB7Fdm/a86fYbkpWIlK0NRRFaXBADNR5FdlTTlWjtluw45U8p4XwisKyeXlhZdI7YQUAdAkR9QENqDehQ5es/Rjx3c6W0=
|
11
|
+
|
12
|
+
rvm:
|
13
|
+
- 2.2.6
|
14
|
+
- 2.3.3
|
15
|
+
- 2.4.0
|
16
|
+
- ruby-head
|
17
|
+
|
18
|
+
matrix:
|
19
|
+
allow_failures:
|
20
|
+
- rvm: ruby-head
|
21
|
+
|
22
|
+
before_install:
|
23
|
+
- gem update bundler
|
24
|
+
|
25
|
+
after_success:
|
26
|
+
- bundle exec codeclimate-test-reporter
|
data/CODE_OF_CONDUCT.md
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
# Contributor Covenant Code of Conduct
|
2
|
+
|
3
|
+
## Our Pledge
|
4
|
+
|
5
|
+
In the interest of fostering an open and welcoming environment, we as
|
6
|
+
contributors and maintainers pledge to making participation in our project and
|
7
|
+
our community a harassment-free experience for everyone, regardless of age, body
|
8
|
+
size, disability, ethnicity, gender identity and expression, level of experience,
|
9
|
+
nationality, personal appearance, race, religion, or sexual identity and
|
10
|
+
orientation.
|
11
|
+
|
12
|
+
## Our Standards
|
13
|
+
|
14
|
+
Examples of behavior that contributes to creating a positive environment
|
15
|
+
include:
|
16
|
+
|
17
|
+
* Using welcoming and inclusive language
|
18
|
+
* Being respectful of differing viewpoints and experiences
|
19
|
+
* Gracefully accepting constructive criticism
|
20
|
+
* Focusing on what is best for the community
|
21
|
+
* Showing empathy towards other community members
|
22
|
+
|
23
|
+
Examples of unacceptable behavior by participants include:
|
24
|
+
|
25
|
+
* The use of sexualized language or imagery and unwelcome sexual attention or
|
26
|
+
advances
|
27
|
+
* Trolling, insulting/derogatory comments, and personal or political attacks
|
28
|
+
* Public or private harassment
|
29
|
+
* Publishing others' private information, such as a physical or electronic
|
30
|
+
address, without explicit permission
|
31
|
+
* Other conduct which could reasonably be considered inappropriate in a
|
32
|
+
professional setting
|
33
|
+
|
34
|
+
## Our Responsibilities
|
35
|
+
|
36
|
+
Project maintainers are responsible for clarifying the standards of acceptable
|
37
|
+
behavior and are expected to take appropriate and fair corrective action in
|
38
|
+
response to any instances of unacceptable behavior.
|
39
|
+
|
40
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
41
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
42
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
43
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
44
|
+
threatening, offensive, or harmful.
|
45
|
+
|
46
|
+
## Scope
|
47
|
+
|
48
|
+
This Code of Conduct applies both within project spaces and in public spaces
|
49
|
+
when an individual is representing the project or its community. Examples of
|
50
|
+
representing a project or community include using an official project e-mail
|
51
|
+
address, posting via an official social media account, or acting as an appointed
|
52
|
+
representative at an online or offline event. Representation of a project may be
|
53
|
+
further defined and clarified by project maintainers.
|
54
|
+
|
55
|
+
## Enforcement
|
56
|
+
|
57
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
58
|
+
reported by contacting the project team at mail@michaelgruber.me. All
|
59
|
+
complaints will be reviewed and investigated and will result in a response that
|
60
|
+
is deemed necessary and appropriate to the circumstances. The project team is
|
61
|
+
obligated to maintain confidentiality with regard to the reporter of an incident.
|
62
|
+
Further details of specific enforcement policies may be posted separately.
|
63
|
+
|
64
|
+
Project maintainers who do not follow or enforce the Code of Conduct in good
|
65
|
+
faith may face temporary or permanent repercussions as determined by other
|
66
|
+
members of the project's leadership.
|
67
|
+
|
68
|
+
## Attribution
|
69
|
+
|
70
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
71
|
+
available at [http://contributor-covenant.org/version/1/4][version]
|
72
|
+
|
73
|
+
[homepage]: http://contributor-covenant.org
|
74
|
+
[version]: http://contributor-covenant.org/version/1/4/
|
data/Gemfile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gem 'rubocop', '~> 0.46.0', require: false
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem 'codeclimate-test-reporter', '~> 1.0.0'
|
7
|
+
gem 'simplecov'
|
8
|
+
end
|
9
|
+
|
10
|
+
# Specify your gem's dependencies in fluent-plugin-ecs-metadata-filter.gemspec
|
11
|
+
gemspec
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Michael Gruber
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
# fluent-plugin-ecs-metadata-filter-v2
|
2
|
+
|
3
|
+
[![Build Status](https://travis-ci.org/michaelgruber/fluent-plugin-ecs-metadata-filter.svg?branch=master)](https://travis-ci.org/michaelgruber/fluent-plugin-ecs-metadata-filter)
|
4
|
+
[![Code Climate](https://codeclimate.com/github/michaelgruber/fluent-plugin-ecs-metadata-filter/badges/gpa.svg)](https://codeclimate.com/github/michaelgruber/fluent-plugin-ecs-metadata-filter)
|
5
|
+
[![Test Coverage](https://codeclimate.com/github/michaelgruber/fluent-plugin-ecs-metadata-filter/badges/coverage.svg)](https://codeclimate.com/github/michaelgruber/fluent-plugin-ecs-metadata-filter/coverage)
|
6
|
+
|
7
|
+
Filter plugin to add AWS ECS metadata to fluentd events. Based on [fabric8io/fluent-plugin-kubernetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter).
|
8
|
+
|
9
|
+
## Requirements
|
10
|
+
|
11
|
+
| fluent-plugin-ecs-metadata-filter | fluentd | ruby |
|
12
|
+
|------------------------|---------|------|
|
13
|
+
| >= 0.2.0 | >= v0.14.0 | >= 2.1 |
|
14
|
+
| < 0.2.0 | >= v0.12.0 | >= 1.9 |
|
15
|
+
|
16
|
+
## Installation
|
17
|
+
|
18
|
+
```ruby
|
19
|
+
gem install fluent-plugin-ecs-metadata-filter-v2
|
20
|
+
```
|
21
|
+
|
22
|
+
## Configuration
|
23
|
+
|
24
|
+
Configuration options for `fluent.conf` are:
|
25
|
+
|
26
|
+
* `cache_size` - Size of the cache of ECS container metadata which reduces requests to the API server - default: `1000`
|
27
|
+
* `cache_ttl` - TTL in seconds for each cached element. Set to negative value to disable TTL eviction - default: `3600` (1 hour)
|
28
|
+
* `ecs_agent_uri` - Base address used to contact the ecs-agent
|
29
|
+
- default: `http://localhost:51678/v1`
|
30
|
+
* `fields_key` - Key in the final record holding the metadata fields. Set to "" to set fields in the record itself - default: `ecs`
|
31
|
+
* `fields` - Array of metadata fields that should be added to a log record - default: `docker_name`, `family`, `cluster`, `name` - **Available options:**
|
32
|
+
+ `cluster`
|
33
|
+
+ `container_instance_arn`
|
34
|
+
+ `container_instance_version`
|
35
|
+
+ `desired_status`
|
36
|
+
+ `docker_id`
|
37
|
+
+ `docker_name` - Name of the docker container
|
38
|
+
+ `family`
|
39
|
+
+ `known_status`
|
40
|
+
+ `name` - Name as specified in the task definition
|
41
|
+
+ `task_arn`
|
42
|
+
+ `version`
|
43
|
+
* `merge_json_log` - Merge in JSON format as top level keys in the log, if the log is JSON - default: `true`
|
44
|
+
* `tag_regexp` - Regular expression used to extract the `docker_id` from the fluentd tag - default: `var\.lib\.docker\.containers\.(?<docker_id>[a-z0-9]{64})\.[a-z0-9]{64}-json.log$` (note the full 64-character id is needed - a short id will not work)
|
45
|
+
|
46
|
+
Reading from the docker container
|
47
|
+
|
48
|
+
```
|
49
|
+
<source>
|
50
|
+
type tail
|
51
|
+
path /var/lib/docker/containers/*/*-json.log
|
52
|
+
pos_file fluentd-docker.pos
|
53
|
+
time_format %Y-%m-%dT%H:%M:%S
|
54
|
+
tag ecs.*
|
55
|
+
format json
|
56
|
+
read_from_head true
|
57
|
+
</source>
|
58
|
+
|
59
|
+
<filter ecs.var.lib.docker.containers.*.*-json.log>
|
60
|
+
type ecs_metadata
|
61
|
+
</filter>
|
62
|
+
|
63
|
+
<match **>
|
64
|
+
type stdout
|
65
|
+
</match>
|
66
|
+
```
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'fluent_ecs'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require 'irb'
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'fluent_ecs/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'fluent-plugin-ecs-metadata-filter-v2'
|
8
|
+
spec.version = FluentECS::VERSION
|
9
|
+
spec.authors = ['Matusalem Andrdade']
|
10
|
+
spec.email = ['matusalem879@gmail.com']
|
11
|
+
|
12
|
+
spec.summary = 'Filter plugin to add AWS ECS metadata to fluentd events'
|
13
|
+
spec.homepage = 'https://github.com/matusalem-santos/fluent-plugin-ecs-metadata-filter-v2'
|
14
|
+
spec.license = 'MIT'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
17
|
+
f.match(%r{^(test|spec|features)/})
|
18
|
+
end
|
19
|
+
spec.bindir = 'exe'
|
20
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
|
+
spec.require_paths = ['lib']
|
22
|
+
|
23
|
+
spec.add_dependency 'fluentd', '>= 0.14.0'
|
24
|
+
spec.add_dependency 'lru_redux', '~> 1.1'
|
25
|
+
spec.add_dependency 'httparty', '~> 0.14.0'
|
26
|
+
|
27
|
+
spec.add_development_dependency 'bundler', '~> 1.13'
|
28
|
+
spec.add_development_dependency 'minitest', '~> 5.0'
|
29
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
30
|
+
spec.add_development_dependency 'test-unit', '~> 3.2', '>= 3.2.3'
|
31
|
+
spec.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.3'
|
32
|
+
spec.add_development_dependency 'webmock', '~> 2.3', '>= 2.3.1'
|
33
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path('../../../lib', __FILE__)
|
2
|
+
|
3
|
+
require 'fluent/plugin/filter'
|
4
|
+
|
5
|
+
module Fluent::Plugin
|
6
|
+
class ECSMetadataFilter < Filter
|
7
|
+
Fluent::Plugin.register_filter('ecs_metadata', self)
|
8
|
+
|
9
|
+
config_param :cache_size, :integer, default: 1000
|
10
|
+
config_param :cache_ttl, :integer, default: 60 * 60
|
11
|
+
config_param :ecs_agent_uri, :string, default: 'http://localhost:51678/v1'
|
12
|
+
config_param :merge_json_log, :bool, default: true
|
13
|
+
config_param :fields_key, :string, default: 'ecs'
|
14
|
+
config_param :fields, :array,
|
15
|
+
default: %w(docker_name family cluster name),
|
16
|
+
value_type: :string
|
17
|
+
config_param :tag_regexp, :string,
|
18
|
+
default: 'var\.lib\.docker\.containers\.' \
|
19
|
+
'(?<docker_id>[a-z0-9]{64})\.[a-z0-9]{64}-json.log$'
|
20
|
+
|
21
|
+
def configure(conf)
|
22
|
+
super
|
23
|
+
|
24
|
+
require 'fluent_ecs'
|
25
|
+
|
26
|
+
validate_params
|
27
|
+
|
28
|
+
FluentECS.configure do |c|
|
29
|
+
c.cache_size = @cache_size
|
30
|
+
c.cache_ttl = @cache_ttl < 0 ? :none : @cache_ttl
|
31
|
+
c.fields = @fields
|
32
|
+
end
|
33
|
+
|
34
|
+
FluentECS::Metadata.base_uri @ecs_agent_uri
|
35
|
+
FluentECS::Task.base_uri @ecs_agent_uri
|
36
|
+
|
37
|
+
@tag_regexp_compiled = Regexp.compile(@tag_regexp)
|
38
|
+
end
|
39
|
+
|
40
|
+
def filter_stream(tag, es)
|
41
|
+
new_es = Fluent::MultiEventStream.new
|
42
|
+
metadata = metadata_for_tag(tag)
|
43
|
+
|
44
|
+
es.each do |time, record|
|
45
|
+
if metadata
|
46
|
+
record = merge_log_json(record) if merge_json_logs?
|
47
|
+
if @fields_key.empty?
|
48
|
+
record.merge! metadata.to_h
|
49
|
+
else
|
50
|
+
record[@fields_key] = metadata.to_h
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
new_es.add(time, record)
|
55
|
+
end
|
56
|
+
|
57
|
+
new_es
|
58
|
+
end
|
59
|
+
|
60
|
+
def validate_params
|
61
|
+
bad_field = @fields.find { |f| !FluentECS::Container.method_defined?(f) }
|
62
|
+
raise Fluent::ConfigError, "Invalid field: '#{bad_field}'" if bad_field
|
63
|
+
end
|
64
|
+
|
65
|
+
def metadata_for_tag(tag)
|
66
|
+
match = tag.match(@tag_regexp_compiled)
|
67
|
+
FluentECS::Container.find(match['docker_id']) unless match.nil?
|
68
|
+
rescue FluentECS::IntrospectError => e
|
69
|
+
log.error(e)
|
70
|
+
nil
|
71
|
+
end
|
72
|
+
|
73
|
+
def looks_like_json?(str)
|
74
|
+
str.is_a?(String) && str[0] == '{' && str[-1] == '}'
|
75
|
+
end
|
76
|
+
|
77
|
+
def merge_json_logs?
|
78
|
+
@merge_json_log
|
79
|
+
end
|
80
|
+
|
81
|
+
def merge_log_json(record)
|
82
|
+
log = record['log']
|
83
|
+
if looks_like_json?(log)
|
84
|
+
begin
|
85
|
+
record = JSON.parse(log).merge!(record)
|
86
|
+
record.delete('log')
|
87
|
+
rescue JSON::ParserError => e
|
88
|
+
self.log.error(e)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
record
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module FluentECS
|
2
|
+
class Container
|
3
|
+
extend Forwardable
|
4
|
+
attr_accessor :docker_id, :docker_name, :name, :task
|
5
|
+
|
6
|
+
def_delegators :@task, :container_instance, :desired_status,
|
7
|
+
:family, :known_status, :version
|
8
|
+
|
9
|
+
def_delegators :container_instance, :cluster
|
10
|
+
|
11
|
+
def_delegator :container_instance, :arn, :container_instance_arn
|
12
|
+
def_delegator :container_instance, :version, :container_instance_version
|
13
|
+
def_delegator :task, :arn, :task_arn
|
14
|
+
|
15
|
+
def initialize(attrs = {})
|
16
|
+
@docker_id = attrs['DockerId']
|
17
|
+
@docker_name = attrs['DockerName']
|
18
|
+
@name = attrs['Name']
|
19
|
+
end
|
20
|
+
|
21
|
+
def method_missing(method_id, *args, &_block)
|
22
|
+
if method_id == :to_h
|
23
|
+
self.class.class_eval hash_definition
|
24
|
+
to_h
|
25
|
+
else
|
26
|
+
super
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def respond_to_missing?(method_id, include_private = false)
|
31
|
+
method_id == :to_h || super
|
32
|
+
end
|
33
|
+
|
34
|
+
def hash_definition
|
35
|
+
fields = FluentECS.config.fields
|
36
|
+
%(
|
37
|
+
def to_h
|
38
|
+
{ #{fields.map { |f| "'#{f}' => #{f}" }.join(',')} }
|
39
|
+
end
|
40
|
+
)
|
41
|
+
end
|
42
|
+
private :hash_definition
|
43
|
+
|
44
|
+
class << self
|
45
|
+
def cache
|
46
|
+
@cache ||= LruRedux::TTL::ThreadSafeCache.new(
|
47
|
+
FluentECS.config.cache_size,
|
48
|
+
FluentECS.config.cache_ttl
|
49
|
+
)
|
50
|
+
end
|
51
|
+
|
52
|
+
def find(docker_id)
|
53
|
+
cache.getset(docker_id) do
|
54
|
+
Metadata.take.containers.each { |c| cache[c.docker_id] = c }
|
55
|
+
cache[docker_id] # cache value nil if container is not in response
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module FluentECS
|
2
|
+
class Metadata
|
3
|
+
include Resource
|
4
|
+
|
5
|
+
self.resource_endpoint = '/metadata'
|
6
|
+
|
7
|
+
attr_accessor :arn, :cluster, :version
|
8
|
+
attr_writer :tasks, :containers
|
9
|
+
|
10
|
+
def initialize(attrs = {})
|
11
|
+
@arn = attrs['ContainerInstanceArn']
|
12
|
+
@cluster = attrs['Cluster']
|
13
|
+
@version = attrs['Version']
|
14
|
+
@task_data = attrs['Tasks'] || Array(Task.get['Tasks'])
|
15
|
+
end
|
16
|
+
|
17
|
+
def tasks
|
18
|
+
@tasks ||= @task_data.map do |d|
|
19
|
+
Task.new(d).tap { |t| t.container_instance = self }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def containers
|
24
|
+
@containers ||= tasks.map(&:containers).flatten
|
25
|
+
end
|
26
|
+
|
27
|
+
class << self
|
28
|
+
def take
|
29
|
+
Metadata.new(get)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module FluentECS
|
2
|
+
module Resource
|
3
|
+
def self.included(base)
|
4
|
+
base.include HTTParty
|
5
|
+
base.base_uri 'http://localhost:51678/v1'
|
6
|
+
base.format :json
|
7
|
+
|
8
|
+
base.extend ClassMethods
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
attr_accessor :resource_endpoint
|
13
|
+
|
14
|
+
def get
|
15
|
+
response = super(resource_endpoint)
|
16
|
+
if response.success?
|
17
|
+
response.parsed_response
|
18
|
+
else
|
19
|
+
err = "GET #{base_uri}#{resource_endpoint}" \
|
20
|
+
" failed with code: #{response.code}"
|
21
|
+
raise RequestError, err
|
22
|
+
end
|
23
|
+
rescue Errno::ECONNREFUSED, HTTParty::Error, Timeout::Error => e
|
24
|
+
raise IntrospectError, e.message
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module FluentECS
|
2
|
+
class Task
|
3
|
+
include Resource
|
4
|
+
self.resource_endpoint = '/tasks'
|
5
|
+
|
6
|
+
attr_accessor :arn, :container_instance, :desired_status,
|
7
|
+
:family, :known_status, :version
|
8
|
+
|
9
|
+
def initialize(attrs = {})
|
10
|
+
@arn = attrs['Arn']
|
11
|
+
@desired_status = attrs['DesiredStatus']
|
12
|
+
@family = attrs['Family']
|
13
|
+
@known_status = attrs['KnownStatus']
|
14
|
+
@version = attrs['Version']
|
15
|
+
@container_data = attrs['Containers']
|
16
|
+
end
|
17
|
+
|
18
|
+
def containers
|
19
|
+
@containers ||= @container_data.map do |d|
|
20
|
+
Container.new(d).tap { |c| c.task = self }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/fluent_ecs.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'lru_redux'
|
3
|
+
|
4
|
+
require 'fluent_ecs/configurable'
|
5
|
+
require 'fluent_ecs/error'
|
6
|
+
require 'fluent_ecs/version'
|
7
|
+
|
8
|
+
require 'fluent_ecs/resource'
|
9
|
+
require 'fluent_ecs/container'
|
10
|
+
require 'fluent_ecs/metadata'
|
11
|
+
require 'fluent_ecs/task'
|
metadata
ADDED
@@ -0,0 +1,208 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-ecs-metadata-filter-v2
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Matusalem Andrdade
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-06-21 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.14.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.14.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: lru_redux
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.1'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.1'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: httparty
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.14.0
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.14.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.13'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.13'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: minitest
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '5.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '5.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '10.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '10.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: test-unit
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '3.2'
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: 3.2.3
|
107
|
+
type: :development
|
108
|
+
prerelease: false
|
109
|
+
version_requirements: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - "~>"
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '3.2'
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 3.2.3
|
117
|
+
- !ruby/object:Gem::Dependency
|
118
|
+
name: vcr
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - "~>"
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '3.0'
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: 3.0.3
|
127
|
+
type: :development
|
128
|
+
prerelease: false
|
129
|
+
version_requirements: !ruby/object:Gem::Requirement
|
130
|
+
requirements:
|
131
|
+
- - "~>"
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '3.0'
|
134
|
+
- - ">="
|
135
|
+
- !ruby/object:Gem::Version
|
136
|
+
version: 3.0.3
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: webmock
|
139
|
+
requirement: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - "~>"
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: '2.3'
|
144
|
+
- - ">="
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: 2.3.1
|
147
|
+
type: :development
|
148
|
+
prerelease: false
|
149
|
+
version_requirements: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - "~>"
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: '2.3'
|
154
|
+
- - ">="
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: 2.3.1
|
157
|
+
description:
|
158
|
+
email:
|
159
|
+
- matusalem879@gmail.com
|
160
|
+
executables: []
|
161
|
+
extensions: []
|
162
|
+
extra_rdoc_files: []
|
163
|
+
files:
|
164
|
+
- ".codeclimate.yml"
|
165
|
+
- ".gitignore"
|
166
|
+
- ".rubocop.yml"
|
167
|
+
- ".travis.yml"
|
168
|
+
- CODE_OF_CONDUCT.md
|
169
|
+
- Gemfile
|
170
|
+
- LICENSE.txt
|
171
|
+
- README.md
|
172
|
+
- Rakefile
|
173
|
+
- bin/console
|
174
|
+
- bin/setup
|
175
|
+
- fluent_plugin_ecs_metadata_filter.gemspec
|
176
|
+
- lib/fluent/plugin/filter_ecs_metadata.rb
|
177
|
+
- lib/fluent_ecs.rb
|
178
|
+
- lib/fluent_ecs/configurable.rb
|
179
|
+
- lib/fluent_ecs/container.rb
|
180
|
+
- lib/fluent_ecs/error.rb
|
181
|
+
- lib/fluent_ecs/metadata.rb
|
182
|
+
- lib/fluent_ecs/resource.rb
|
183
|
+
- lib/fluent_ecs/task.rb
|
184
|
+
- lib/fluent_ecs/version.rb
|
185
|
+
homepage: https://github.com/matusalem-santos/fluent-plugin-ecs-metadata-filter-v2
|
186
|
+
licenses:
|
187
|
+
- MIT
|
188
|
+
metadata: {}
|
189
|
+
post_install_message:
|
190
|
+
rdoc_options: []
|
191
|
+
require_paths:
|
192
|
+
- lib
|
193
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
194
|
+
requirements:
|
195
|
+
- - ">="
|
196
|
+
- !ruby/object:Gem::Version
|
197
|
+
version: '0'
|
198
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
199
|
+
requirements:
|
200
|
+
- - ">="
|
201
|
+
- !ruby/object:Gem::Version
|
202
|
+
version: '0'
|
203
|
+
requirements: []
|
204
|
+
rubygems_version: 3.4.10
|
205
|
+
signing_key:
|
206
|
+
specification_version: 4
|
207
|
+
summary: Filter plugin to add AWS ECS metadata to fluentd events
|
208
|
+
test_files: []
|