fluent-plugin-gelf 0.0.0.dev1 → 0.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.circleci/config.yml +76 -0
- data/.gitignore +2 -0
- data/.rubocop.yml +12 -0
- data/.ruby-version +1 -0
- data/Gemfile +5 -0
- data/LICENSE +29 -0
- data/README.md +60 -0
- data/Rakefile +18 -0
- data/fluent-plugin-gelf.gemspec +31 -0
- data/lib/fluent/plugin/formatter_gelf.rb +18 -0
- data/lib/fluent/plugin/gelf_plugin_util.rb +114 -0
- data/lib/fluent/plugin/out_gelf.rb +60 -0
- data/test/fixtures/docker_futureproof.json +32 -0
- data/test/fixtures/docker_gelf_formatted.json +45 -0
- data/test/fixtures/docker_gelf_formatted_already_extracted.json +61 -0
- data/test/fixtures/docker_inner_json.json +32 -0
- data/test/fixtures/docker_inner_json_with_time.json +32 -0
- data/test/fixtures/docker_unformatted.json +31 -0
- data/test/fixtures/level_from_tag.json +23 -0
- data/test/fixtures/message_json_parsable_to_nil.json +50 -0
- data/test/fixtures/systemd.json +50 -0
- data/test/fixtures/systemd_inner_json.json +52 -0
- data/test/fixtures/systemd_with_real_timestamp.json +53 -0
- data/test/helper.rb +21 -0
- data/test/plugin/test_formatter_gelf.rb +26 -0
- data/test/plugin/test_out_gelf.rb +56 -0
- metadata +70 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 70a3623ed2adde0ca73a4f9b406657a50ba9cbb33ffbe5512fad9c9fb76591db
|
4
|
+
data.tar.gz: 6a7acdaa55e1088efbce002224c88c8795a84b5d1ac634f48dd2c5e063c3f4ad
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76b60605cf69b29427372dedf3792132a761410e23f7bf7f918290694e1eb06225f90f5263a6a54c08ca7bc52ab44a8cdbb0baa4cf34d31d685bb4ffc75f2e7f
|
7
|
+
data.tar.gz: 73742730f414d4643e12914c15cbdd61a4900ee50219b74538afe312ca74243a3a249cd77c26c63b2ec20aa1260e3297ca0f60b29e6cb873bddc0cede6ef8492
|
@@ -0,0 +1,76 @@
|
|
1
|
+
version: 2.1
|
2
|
+
commands:
|
3
|
+
install_gems:
|
4
|
+
steps:
|
5
|
+
- restore_cache:
|
6
|
+
key: fluent-plugin-gelf2-{{ checksum "fluent-plugin-gelf.gemspec" }}-{{ checksum "Gemfile" }}
|
7
|
+
- run: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3 --full-index
|
8
|
+
- save_cache:
|
9
|
+
key: fluent-plugin-gelf2-{{ checksum "fluent-plugin-gelf.gemspec" }}-{{ checksum "Gemfile" }}
|
10
|
+
paths:
|
11
|
+
- vendor/bundle
|
12
|
+
executors:
|
13
|
+
ruby:
|
14
|
+
parameters:
|
15
|
+
version:
|
16
|
+
type: string
|
17
|
+
default: latest
|
18
|
+
docker:
|
19
|
+
- image: circleci/ruby:<< parameters.version >>
|
20
|
+
working_directory: ~/fluent-plugin-gelf
|
21
|
+
jobs:
|
22
|
+
publish:
|
23
|
+
executor: ruby
|
24
|
+
steps:
|
25
|
+
- checkout
|
26
|
+
- install_gems
|
27
|
+
- run:
|
28
|
+
name: Configure RubyGems API key
|
29
|
+
command: |
|
30
|
+
mkdir -p ~/.gem
|
31
|
+
echo ":rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials
|
32
|
+
chmod 0600 ~/.gem/credentials
|
33
|
+
- run:
|
34
|
+
name: Build gem
|
35
|
+
command: bundle exec rake build --trace
|
36
|
+
- run:
|
37
|
+
name: Release gem
|
38
|
+
command: bundle exec rake release:rubygem_push --trace
|
39
|
+
rubocop:
|
40
|
+
executor: ruby
|
41
|
+
steps:
|
42
|
+
- checkout
|
43
|
+
- install_gems
|
44
|
+
- run:
|
45
|
+
name: Run RuboCop
|
46
|
+
command: bundle exec rake rubocop --trace
|
47
|
+
test:
|
48
|
+
executor: ruby
|
49
|
+
steps:
|
50
|
+
- checkout
|
51
|
+
- install_gems
|
52
|
+
- run:
|
53
|
+
name: Run tests
|
54
|
+
command: bundle exec rake test --trace
|
55
|
+
|
56
|
+
workflows:
|
57
|
+
test:
|
58
|
+
jobs:
|
59
|
+
- rubocop:
|
60
|
+
filters:
|
61
|
+
tags:
|
62
|
+
only: /.*/
|
63
|
+
- test:
|
64
|
+
filters:
|
65
|
+
tags:
|
66
|
+
only: /.*/
|
67
|
+
- publish:
|
68
|
+
context: org-rubygems
|
69
|
+
filters:
|
70
|
+
branches:
|
71
|
+
ignore: /.*/
|
72
|
+
tags:
|
73
|
+
only: /^v\d{1,2}\.\d{1,2}\.\d{1,2}.*/
|
74
|
+
requires:
|
75
|
+
- rubocop
|
76
|
+
- test
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 2.4
|
3
|
+
Style/TrailingCommaInArguments:
|
4
|
+
EnforcedStyleForMultiline: comma
|
5
|
+
Style/TrailingCommaInLiteral:
|
6
|
+
EnforcedStyleForMultiline: comma
|
7
|
+
Metrics/LineLength:
|
8
|
+
Max: 120
|
9
|
+
Style/StringLiterals:
|
10
|
+
EnforcedStyle: double_quotes
|
11
|
+
Style/Documentation:
|
12
|
+
Enabled: false
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.4.2
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
BSD 3-Clause License
|
2
|
+
|
3
|
+
Copyright (c) 2017, Funding Circle
|
4
|
+
All rights reserved.
|
5
|
+
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
8
|
+
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
10
|
+
list of conditions and the following disclaimer.
|
11
|
+
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
14
|
+
and/or other materials provided with the distribution.
|
15
|
+
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
17
|
+
contributors may be used to endorse or promote products derived from
|
18
|
+
this software without specific prior written permission.
|
19
|
+
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
# fluent-plugin-gelf
|
2
|
+
|
3
|
+
[![CircleCI](https://circleci.com/gh/FundingCircle/fluent-plugin-gelf/tree/master.svg?style=svg&circle-token=d781ef54862db51be146a2a2ad7aa9d783bb177a)](https://circleci.com/gh/FundingCircle/fluent-plugin-gelf/tree/master)
|
4
|
+
|
5
|
+
A [fluentd](https://www.fluentd.org/) output plugin for sending log events to
|
6
|
+
[Graylog](https://docs.graylog.org/).
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
### RubyGems
|
11
|
+
|
12
|
+
```
|
13
|
+
$ gem install fluent-plugin-gelf
|
14
|
+
```
|
15
|
+
|
16
|
+
### Bundler
|
17
|
+
|
18
|
+
Add this line to your application's Gemfile:
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
gem "fluent-plugin-gelf"
|
22
|
+
```
|
23
|
+
|
24
|
+
And then execute:
|
25
|
+
|
26
|
+
```
|
27
|
+
$ bundle
|
28
|
+
```
|
29
|
+
|
30
|
+
## Usage
|
31
|
+
|
32
|
+
This `gelf` plugin is for fluentd v1.0 or later.
|
33
|
+
|
34
|
+
```
|
35
|
+
<match app.**>
|
36
|
+
output_data_type gelf
|
37
|
+
</match>
|
38
|
+
```
|
39
|
+
|
40
|
+
## Configuration
|
41
|
+
|
42
|
+
#### host (string) (required)
|
43
|
+
|
44
|
+
The hostname of your Graylog cluster.
|
45
|
+
|
46
|
+
####o port (integer) (optional)
|
47
|
+
|
48
|
+
The TCP port of your Graylog cluster. Default value: `12201`.
|
49
|
+
|
50
|
+
## Releases
|
51
|
+
|
52
|
+
The CircleCI build for this project manages gem releases to RubyGems. To release
|
53
|
+
a new version of this gem, create a tag for the version and push it. It will
|
54
|
+
then be built and deployed automatically.
|
55
|
+
|
56
|
+
## Copyright and License
|
57
|
+
|
58
|
+
Copyright © 2017 Funding Circle Ltd.
|
59
|
+
|
60
|
+
Licensed under the [BSD 3-Clause License](LICENSE).
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "bundler"
|
4
|
+
Bundler::GemHelper.install_tasks
|
5
|
+
|
6
|
+
require "rake/testtask"
|
7
|
+
require "rubocop/rake_task"
|
8
|
+
|
9
|
+
Rake::TestTask.new(:test) do |t|
|
10
|
+
t.libs.push("lib", "test")
|
11
|
+
t.test_files = FileList["test/**/test_*.rb"]
|
12
|
+
t.verbose = true
|
13
|
+
t.warning = true
|
14
|
+
end
|
15
|
+
|
16
|
+
RuboCop::RakeTask.new(:rubocop)
|
17
|
+
|
18
|
+
task default: %i[rubocop test]
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path("../lib", __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "fluent-plugin-gelf"
|
8
|
+
spec.version = "0.2.4"
|
9
|
+
spec.authors = ["Funding Circle"]
|
10
|
+
spec.email = ["engineering+fluent-plugin-gelf@fundingcircle.com"]
|
11
|
+
|
12
|
+
spec.summary = "Graylog output plugin for fluentd"
|
13
|
+
spec.description = "Converts fluentd log events into GELF format and sends them to Graylog"
|
14
|
+
spec.license = "BSD-3-Clause"
|
15
|
+
|
16
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
17
|
+
f.match(%r{^(test|spec|features)/})
|
18
|
+
end
|
19
|
+
spec.files = files
|
20
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
|
+
spec.test_files = test_files
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_development_dependency "oj", "~> 3.3.10"
|
25
|
+
spec.add_development_dependency "bundler", "~> 2.1.4"
|
26
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
27
|
+
spec.add_development_dependency "test-unit", "~> 3.0"
|
28
|
+
spec.add_development_dependency "test-unit-rr", "~> 1.0.5"
|
29
|
+
spec.add_development_dependency "rubocop", "~> 0.50.0"
|
30
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
31
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "fluent/plugin/formatter"
|
4
|
+
require "fluent/plugin/gelf_plugin_util"
|
5
|
+
require "yajl"
|
6
|
+
|
7
|
+
module Fluent
|
8
|
+
module Plugin
|
9
|
+
class GelfFormatter < Fluent::Plugin::Formatter
|
10
|
+
Fluent::Plugin.register_formatter("gelf", self)
|
11
|
+
include Fluent::GelfPluginUtil
|
12
|
+
|
13
|
+
def format(tag, time, record)
|
14
|
+
Yajl::Encoder.encode(make_gelfentry(tag, time, record))
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "oj"
|
4
|
+
require "date"
|
5
|
+
|
6
|
+
module Fluent
|
7
|
+
module GelfPluginUtil
|
8
|
+
SYSLOG_FACILITY = {
|
9
|
+
"0" => "kern",
|
10
|
+
"1" => "user",
|
11
|
+
"2" => "mail",
|
12
|
+
"3" => "daemon",
|
13
|
+
"4" => "auth",
|
14
|
+
"5" => "syslog",
|
15
|
+
"6" => "lpr",
|
16
|
+
"7" => "news",
|
17
|
+
"8" => "uucp",
|
18
|
+
"9" => "cron",
|
19
|
+
"10" => "authpriv",
|
20
|
+
"16" => "local0",
|
21
|
+
"17" => "local1",
|
22
|
+
"18" => "local2",
|
23
|
+
"19" => "local3",
|
24
|
+
"20" => "local4",
|
25
|
+
"21" => "local5",
|
26
|
+
"22" => "local6",
|
27
|
+
"23" => "local7",
|
28
|
+
}.freeze
|
29
|
+
|
30
|
+
LEVEL_MAPPING = {
|
31
|
+
"error" => 3,
|
32
|
+
"warn" => 4,
|
33
|
+
"info" => 6,
|
34
|
+
"debug" => 7,
|
35
|
+
}.freeze
|
36
|
+
|
37
|
+
def merge_inner_json(record, key)
|
38
|
+
return record unless record[key]
|
39
|
+
json = Oj.load(record[key].strip)
|
40
|
+
return record unless json
|
41
|
+
json["host"] = record["host"] # preserve host
|
42
|
+
record.delete(key)
|
43
|
+
record.merge(json)
|
44
|
+
rescue Oj::ParseError
|
45
|
+
record
|
46
|
+
end
|
47
|
+
|
48
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
|
49
|
+
def make_gelfentry(tag, time, record)
|
50
|
+
record = merge_inner_json(record, "message")
|
51
|
+
record = merge_inner_json(record, "log")
|
52
|
+
gelfentry = {}
|
53
|
+
gelfentry["timestamp"] = if defined?(Fluent::EventTime) && time.is_a?(Fluent::EventTime)
|
54
|
+
time.sec + (time.nsec.to_f / 1_000_000_000).round(3)
|
55
|
+
else
|
56
|
+
time
|
57
|
+
end
|
58
|
+
|
59
|
+
gelfentry["_fluentd_tag"] = tag
|
60
|
+
|
61
|
+
record.each_pair do |k, v| # rubocop:disable Metrics/BlockLength
|
62
|
+
case k
|
63
|
+
when "timestamp", "time"
|
64
|
+
gelfentry["timestamp"] = if v.is_a?(Integer) || v.is_a?(Float)
|
65
|
+
v
|
66
|
+
else
|
67
|
+
begin
|
68
|
+
(DateTime.parse(v).strftime("%Q").to_f / 1_000).round(3)
|
69
|
+
rescue ArgumentError
|
70
|
+
v
|
71
|
+
end
|
72
|
+
end
|
73
|
+
when "msec" then
|
74
|
+
if time.is_a?(Integer) && record["timestamp"].nil? && record["time"].nil?
|
75
|
+
gelfentry["timestamp"] = "#{time}.#{v}".to_f
|
76
|
+
else
|
77
|
+
gelfentry["_msec"] = v
|
78
|
+
end
|
79
|
+
when "level"
|
80
|
+
gelfentry["level"] = if v.is_a?(Integer)
|
81
|
+
v
|
82
|
+
else
|
83
|
+
level = LEVEL_MAPPING.keys.find { |l| v.downcase =~ /#{l}/ }
|
84
|
+
level.nil? ? v : LEVEL_MAPPING[level]
|
85
|
+
end
|
86
|
+
when "source_realtime_timestamp" then gelfentry["timestamp"] = (v.to_f / 1_000_000).round(3)
|
87
|
+
when "host", "hostname" then gelfentry["host"] = v.to_s
|
88
|
+
when "priority" then gelfentry["level"] = v.to_i
|
89
|
+
when "syslog_facility" then gelfentry["facility"] = SYSLOG_FACILITY[v]
|
90
|
+
when "short_message", "version", "full_message", "facility", "file", "line" then gelfentry[k] = v
|
91
|
+
else
|
92
|
+
k.to_s.start_with?("_") ? gelfentry[k] = v : gelfentry["_#{k}"] = v
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
if gelfentry["short_message"].nil? || gelfentry["short_message"].to_s.empty?
|
97
|
+
gelfentry["short_message"] = if gelfentry.key?("_message") && !gelfentry["_message"].to_s.empty?
|
98
|
+
gelfentry.delete("_message")
|
99
|
+
elsif gelfentry.key?("_log") && !gelfentry["_log"].to_s.empty?
|
100
|
+
gelfentry.delete("_log")
|
101
|
+
else
|
102
|
+
"(no message)"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
if gelfentry["level"].nil?
|
107
|
+
level = LEVEL_MAPPING.keys.find { |k| tag.downcase =~ /\.#{k}/ }
|
108
|
+
gelfentry["level"] = level.nil? ? 6 : LEVEL_MAPPING[level]
|
109
|
+
end
|
110
|
+
|
111
|
+
gelfentry
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "fluent/plugin/output"
|
4
|
+
require "fluent/plugin/gelf_plugin_util"
|
5
|
+
|
6
|
+
module Fluent
|
7
|
+
module Plugin
|
8
|
+
class GelfOutput < Fluent::Plugin::Output
|
9
|
+
Fluent::Plugin.register_output("gelf", self)
|
10
|
+
|
11
|
+
include Fluent::GelfPluginUtil
|
12
|
+
|
13
|
+
config_param :host, :string, default: nil
|
14
|
+
config_param :port, :integer, default: 12_201
|
15
|
+
|
16
|
+
def configure(conf)
|
17
|
+
super
|
18
|
+
raise ConfigError, "'host' parameter is required" unless conf.key?("host")
|
19
|
+
end
|
20
|
+
|
21
|
+
def start
|
22
|
+
super
|
23
|
+
end
|
24
|
+
|
25
|
+
def shutdown
|
26
|
+
super
|
27
|
+
end
|
28
|
+
|
29
|
+
def format(tag, time, record)
|
30
|
+
make_gelfentry(tag, time, record).to_msgpack
|
31
|
+
end
|
32
|
+
|
33
|
+
def write(chunk) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
34
|
+
records = []
|
35
|
+
chunk.msgpack_each do |record|
|
36
|
+
records.push JSON.dump(record) + "\0" # Message delimited by null char
|
37
|
+
end
|
38
|
+
|
39
|
+
log.debug "establishing connection with GrayLog"
|
40
|
+
socket = TCPSocket.new(@host, @port)
|
41
|
+
|
42
|
+
begin
|
43
|
+
log.debug "sending #{records.count} records in batch"
|
44
|
+
socket.write(records.join)
|
45
|
+
ensure
|
46
|
+
log.debug "closing connection with GrayLog"
|
47
|
+
socket.close
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def formatted_to_msgpack_binary
|
52
|
+
true
|
53
|
+
end
|
54
|
+
|
55
|
+
def multi_workers_ready?
|
56
|
+
true
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.312fe8f61472",
|
4
|
+
"comment": "once msec set https://github.com/moby/moby/issues/17181",
|
5
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
6
|
+
"record": {
|
7
|
+
"container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
8
|
+
"container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
9
|
+
"source": "stderr",
|
10
|
+
"log": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "vault-gatekeeper-mesos",
|
13
|
+
"mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
14
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
15
|
+
"msec": "229"
|
16
|
+
}
|
17
|
+
},
|
18
|
+
"expected": {
|
19
|
+
"timestamp": 1512335408.558,
|
20
|
+
"_fluentd_tag": "docker.312fe8f61472",
|
21
|
+
"_container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
22
|
+
"_container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
23
|
+
"_source": "stderr",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "vault-gatekeeper-mesos",
|
26
|
+
"_mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
27
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
28
|
+
"_msec": "229",
|
29
|
+
"short_message": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
30
|
+
"level": 6
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,45 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"comment": "docker log already parsed by https://github.com/joshughes/fluent-plugin-mesosphere-filter (merge_json_log)",
|
6
|
+
"record": {
|
7
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
8
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
9
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
10
|
+
"source": "stdout",
|
11
|
+
"log": "{\"_request.status\":200,\"_request.method\":\"GET\",\"_request.path\":\"/\",\"_request.params\":{},\"_request.request_id\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"_request.request_ip\":\"10.42.8.20\",\"_request.user_agent\":\"spray-can/1.3.4\",\"_request.duration\":1,\"_type\":\"request\",\"_service.name\":\"my_app\",\"_service.version\":\"unknown.sha\",\"_tags\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"short_message\":\"GET / 200 in 1ms\",\"timestamp\":1512385332.824,\"host\":\"d14490734a2f\",\"level\":6,\"version\":\"1.1\"}",
|
12
|
+
"mesos_framework": "marathon",
|
13
|
+
"app": "loga",
|
14
|
+
"mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
15
|
+
"msec": "825"
|
16
|
+
}
|
17
|
+
},
|
18
|
+
"expected": {
|
19
|
+
"timestamp": 1512385332.824,
|
20
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
21
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
22
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
23
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
24
|
+
"_source": "stdout",
|
25
|
+
"_mesos_framework": "marathon",
|
26
|
+
"_app": "loga",
|
27
|
+
"_mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
28
|
+
"_msec": "825",
|
29
|
+
"_request.status": 200,
|
30
|
+
"_request.method": "GET",
|
31
|
+
"_request.path": "/",
|
32
|
+
"_request.params": {},
|
33
|
+
"_request.request_id": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
34
|
+
"_request.request_ip": "10.42.8.20",
|
35
|
+
"_request.user_agent": "spray-can/1.3.4",
|
36
|
+
"_request.duration": 1,
|
37
|
+
"_type": "request",
|
38
|
+
"_service.name": "my_app",
|
39
|
+
"_service.version": "unknown.sha",
|
40
|
+
"_tags": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
41
|
+
"short_message": "GET / 200 in 1ms",
|
42
|
+
"level": 6,
|
43
|
+
"version": "1.1"
|
44
|
+
}
|
45
|
+
}
|
@@ -0,0 +1,61 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"comment": "docker log already parsed by https://github.com/joshughes/fluent-plugin-mesosphere-filter (merge_json_log)",
|
6
|
+
"record": {
|
7
|
+
"_request.status": 200,
|
8
|
+
"_request.method": "GET",
|
9
|
+
"_request.path": "/",
|
10
|
+
"_request.params": {},
|
11
|
+
"_request.request_id": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
12
|
+
"_request.request_ip": "10.42.8.20",
|
13
|
+
"_request.user_agent": "spray-can/1.3.4",
|
14
|
+
"_request.duration": 1,
|
15
|
+
"_type": "request",
|
16
|
+
"_service.name": "my_app",
|
17
|
+
"_service.version": "unknown.sha",
|
18
|
+
"_tags": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
19
|
+
"short_message": "GET / 200 in 1ms",
|
20
|
+
"timestamp": 1512385332.824,
|
21
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
22
|
+
"level": 6,
|
23
|
+
"version": "1.1",
|
24
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
25
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
26
|
+
"source": "stdout",
|
27
|
+
"log": "{\"_request.status\":200,\"_request.method\":\"GET\",\"_request.path\":\"/\",\"_request.params\":{},\"_request.request_id\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"_request.request_ip\":\"10.42.8.20\",\"_request.user_agent\":\"spray-can/1.3.4\",\"_request.duration\":1,\"_type\":\"request\",\"_service.name\":\"my_app\",\"_service.version\":\"unknown.sha\",\"_tags\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"short_message\":\"GET / 200 in 1ms\",\"timestamp\":1512385332.824,\"host\":\"d14490734a2f\",\"level\":6,\"version\":\"1.1\"}",
|
28
|
+
"mesos_framework": "marathon",
|
29
|
+
"app": "loga",
|
30
|
+
"mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
31
|
+
"msec": "825"
|
32
|
+
}
|
33
|
+
},
|
34
|
+
"expected": {
|
35
|
+
"timestamp": 1512385332.824,
|
36
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
37
|
+
"_request.status": 200,
|
38
|
+
"_request.method": "GET",
|
39
|
+
"_request.path": "/",
|
40
|
+
"_request.params": {},
|
41
|
+
"_request.request_id": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
42
|
+
"_request.request_ip": "10.42.8.20",
|
43
|
+
"_request.user_agent": "spray-can/1.3.4",
|
44
|
+
"_request.duration": 1,
|
45
|
+
"_type": "request",
|
46
|
+
"_service.name": "my_app",
|
47
|
+
"_service.version": "unknown.sha",
|
48
|
+
"_tags": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
49
|
+
"short_message": "GET / 200 in 1ms",
|
50
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
51
|
+
"level": 6,
|
52
|
+
"version": "1.1",
|
53
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
54
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
55
|
+
"_source": "stdout",
|
56
|
+
"_mesos_framework": "marathon",
|
57
|
+
"_app": "loga",
|
58
|
+
"_mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
59
|
+
"_msec": "825"
|
60
|
+
}
|
61
|
+
}
|
@@ -0,0 +1,32 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
8
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
9
|
+
"source": "stdout",
|
10
|
+
"log": "{\"foo\":\"bar\",\"bar\":\"baz\"}",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "other",
|
13
|
+
"mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
14
|
+
"msec": "825"
|
15
|
+
}
|
16
|
+
},
|
17
|
+
"expected": {
|
18
|
+
"timestamp": 15123853321.825,
|
19
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
20
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
21
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
22
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
23
|
+
"_source": "stdout",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "other",
|
26
|
+
"_mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
27
|
+
"_foo": "bar",
|
28
|
+
"_bar": "baz",
|
29
|
+
"short_message": "(no message)",
|
30
|
+
"level": 6
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,32 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
8
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
9
|
+
"source": "stdout",
|
10
|
+
"log": "{\"message\":\"foo\",\"bar\":\"baz\",\"time\":\"2020-01-29T16:17:43Z\"}",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "other",
|
13
|
+
"mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
14
|
+
"msec": "825"
|
15
|
+
}
|
16
|
+
},
|
17
|
+
"expected": {
|
18
|
+
"timestamp": 1580314663.0,
|
19
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
20
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
21
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
22
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
23
|
+
"_source": "stdout",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "other",
|
26
|
+
"_mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
27
|
+
"_msec": "825",
|
28
|
+
"_bar": "baz",
|
29
|
+
"short_message": "foo",
|
30
|
+
"level": 6
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,31 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.312fe8f61472",
|
4
|
+
"comment": "no msec from docker: https://github.com/moby/moby/issues/17181#issuecomment-345594658",
|
5
|
+
"time": 1512424624,
|
6
|
+
"record": {
|
7
|
+
"container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
8
|
+
"container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
9
|
+
"source": "stderr",
|
10
|
+
"log": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "vault-gatekeeper-mesos",
|
13
|
+
"mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
14
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
15
|
+
"msec": "229"
|
16
|
+
}
|
17
|
+
},
|
18
|
+
"expected": {
|
19
|
+
"timestamp": 1512424624.229,
|
20
|
+
"_fluentd_tag": "docker.312fe8f61472",
|
21
|
+
"_container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
22
|
+
"_container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
23
|
+
"_source": "stderr",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "vault-gatekeeper-mesos",
|
26
|
+
"_mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
27
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
28
|
+
"short_message": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
29
|
+
"level": 6
|
30
|
+
}
|
31
|
+
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "fluent.warn",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"retry_time": 0,
|
7
|
+
"next_retry_seconds": "2017-12-04 22:56:27 +0000",
|
8
|
+
"chunk": "55f8ba0c48815f9a61ede8f6c2a68208",
|
9
|
+
"error": "#<Errno::ECONNREFUSED: Connection refused...",
|
10
|
+
"message": "failed to flush the buffer..."
|
11
|
+
}
|
12
|
+
},
|
13
|
+
"expected": {
|
14
|
+
"timestamp": 1512335408.558,
|
15
|
+
"_fluentd_tag": "fluent.warn",
|
16
|
+
"_retry_time": 0,
|
17
|
+
"_next_retry_seconds": "2017-12-04 22:56:27 +0000",
|
18
|
+
"_chunk": "55f8ba0c48815f9a61ede8f6c2a68208",
|
19
|
+
"_error": "#<Errno::ECONNREFUSED: Connection refused...",
|
20
|
+
"short_message": "failed to flush the buffer...",
|
21
|
+
"level": 4
|
22
|
+
}
|
23
|
+
}
|
@@ -0,0 +1,50 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"transport": "stdout",
|
8
|
+
"priority": "6",
|
9
|
+
"syslog_facility": "3",
|
10
|
+
"systemd_slice": "system.slice",
|
11
|
+
"selinux_context": "system_u:system_r:init_t:s0",
|
12
|
+
"boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
13
|
+
"machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
14
|
+
"cap_effective": "0",
|
15
|
+
"syslog_identifier": "consul-start.sh",
|
16
|
+
"pid": "3388",
|
17
|
+
"uid": "997",
|
18
|
+
"gid": "994",
|
19
|
+
"comm": "consul",
|
20
|
+
"exe": "/usr/local/bin/consul",
|
21
|
+
"cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
22
|
+
"systemd_cgroup": "/system.slice/consul.service",
|
23
|
+
"systemd_unit": "consul.service",
|
24
|
+
"message": " // Clearing the cache for the prod environment with debug false"
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"expected": {
|
28
|
+
"timestamp": 1512335408.558,
|
29
|
+
"_fluentd_tag": "journald.log",
|
30
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
31
|
+
"_transport": "stdout",
|
32
|
+
"level": 6,
|
33
|
+
"facility": "daemon",
|
34
|
+
"_systemd_slice": "system.slice",
|
35
|
+
"_selinux_context": "system_u:system_r:init_t:s0",
|
36
|
+
"_boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
37
|
+
"_machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
38
|
+
"_cap_effective": "0",
|
39
|
+
"_syslog_identifier": "consul-start.sh",
|
40
|
+
"_pid": "3388",
|
41
|
+
"_uid": "997",
|
42
|
+
"_gid": "994",
|
43
|
+
"_comm": "consul",
|
44
|
+
"_exe": "/usr/local/bin/consul",
|
45
|
+
"_cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
46
|
+
"_systemd_cgroup": "/system.slice/consul.service",
|
47
|
+
"_systemd_unit": "consul.service",
|
48
|
+
"short_message": " // Clearing the cache for the prod environment with debug false"
|
49
|
+
}
|
50
|
+
}
|
@@ -0,0 +1,50 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"transport": "stdout",
|
8
|
+
"priority": "6",
|
9
|
+
"syslog_facility": "3",
|
10
|
+
"systemd_slice": "system.slice",
|
11
|
+
"selinux_context": "system_u:system_r:init_t:s0",
|
12
|
+
"boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
13
|
+
"machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
14
|
+
"cap_effective": "0",
|
15
|
+
"syslog_identifier": "consul-start.sh",
|
16
|
+
"pid": "3388",
|
17
|
+
"uid": "997",
|
18
|
+
"gid": "994",
|
19
|
+
"comm": "consul",
|
20
|
+
"exe": "/usr/local/bin/consul",
|
21
|
+
"cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
22
|
+
"systemd_cgroup": "/system.slice/consul.service",
|
23
|
+
"systemd_unit": "consul.service",
|
24
|
+
"message": "2017/12/03 21:10:08 [INFO] agent: Synced check 'service:loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2_loga_7403'"
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"expected": {
|
28
|
+
"timestamp": 1512335408.558,
|
29
|
+
"_fluentd_tag": "journald.log",
|
30
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
31
|
+
"_transport": "stdout",
|
32
|
+
"level": 6,
|
33
|
+
"facility": "daemon",
|
34
|
+
"_systemd_slice": "system.slice",
|
35
|
+
"_selinux_context": "system_u:system_r:init_t:s0",
|
36
|
+
"_boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
37
|
+
"_machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
38
|
+
"_cap_effective": "0",
|
39
|
+
"_syslog_identifier": "consul-start.sh",
|
40
|
+
"_pid": "3388",
|
41
|
+
"_uid": "997",
|
42
|
+
"_gid": "994",
|
43
|
+
"_comm": "consul",
|
44
|
+
"_exe": "/usr/local/bin/consul",
|
45
|
+
"_cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
46
|
+
"_systemd_cgroup": "/system.slice/consul.service",
|
47
|
+
"_systemd_unit": "consul.service",
|
48
|
+
"short_message": "2017/12/03 21:10:08 [INFO] agent: Synced check 'service:loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2_loga_7403'"
|
49
|
+
}
|
50
|
+
}
|
@@ -0,0 +1,52 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2018-01-09 15:16:47.693Z UTC",
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-0690bffd1e39d7462",
|
7
|
+
"transport": "stdout",
|
8
|
+
"priority": "6",
|
9
|
+
"syslog_facility": "3",
|
10
|
+
"systemd_slice": "system.slice",
|
11
|
+
"selinux_context": "system_u:system_r:container_runtime_t:s0",
|
12
|
+
"boot_id": "a83a6f2a9aa545d18188387fcff0dea9",
|
13
|
+
"machine_id": "f073c429a7456b53ec3e2c53460c5c8f",
|
14
|
+
"cap_effective": "1fffffffff",
|
15
|
+
"syslog_identifier": "kafka-connect-server-kafka-connect-clj",
|
16
|
+
"pid": "1158",
|
17
|
+
"uid": "0",
|
18
|
+
"gid": "0",
|
19
|
+
"comm": "dockerd",
|
20
|
+
"exe": "/usr/bin/dockerd",
|
21
|
+
"cmdline": "/usr/bin/dockerd --exec-opt native.cgroupdriver=cgroupfs --log-opt tag=docker.{{.ID}} --log-opt fluentd-address=unix:///var/run/td-agent/td-agent.sock --log-driver=fluentd --log-opt fluentd-async-connect=true --dns=10.42.8.190",
|
22
|
+
"systemd_cgroup": "/system.slice/docker.service",
|
23
|
+
"systemd_unit": "docker.service",
|
24
|
+
"message": "{\"timestamp\":\"2018-01-09T15:16:47.692+00:00\",\"logger\":\"kafka.connect.SinkTask\",\"level\":\"DEBUG\",\"thread\":\"pool-1-thread-30\",\"message\":\"put 0 records to sink\"}"
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"expected": {
|
28
|
+
"timestamp": 1515511007.692,
|
29
|
+
"_fluentd_tag": "journald.log",
|
30
|
+
"host": "mesosslave-private-0690bffd1e39d7462",
|
31
|
+
"_transport": "stdout",
|
32
|
+
"level": 7,
|
33
|
+
"facility": "daemon",
|
34
|
+
"_systemd_slice": "system.slice",
|
35
|
+
"_selinux_context": "system_u:system_r:container_runtime_t:s0",
|
36
|
+
"_boot_id": "a83a6f2a9aa545d18188387fcff0dea9",
|
37
|
+
"_machine_id": "f073c429a7456b53ec3e2c53460c5c8f",
|
38
|
+
"_cap_effective": "1fffffffff",
|
39
|
+
"_syslog_identifier": "kafka-connect-server-kafka-connect-clj",
|
40
|
+
"_pid": "1158",
|
41
|
+
"_uid": "0",
|
42
|
+
"_gid": "0",
|
43
|
+
"_comm": "dockerd",
|
44
|
+
"_exe": "/usr/bin/dockerd",
|
45
|
+
"_cmdline": "/usr/bin/dockerd --exec-opt native.cgroupdriver=cgroupfs --log-opt tag=docker.{{.ID}} --log-opt fluentd-address=unix:///var/run/td-agent/td-agent.sock --log-driver=fluentd --log-opt fluentd-async-connect=true --dns=10.42.8.190",
|
46
|
+
"_systemd_cgroup": "/system.slice/docker.service",
|
47
|
+
"_systemd_unit": "docker.service",
|
48
|
+
"_logger": "kafka.connect.SinkTask",
|
49
|
+
"_thread": "pool-1-thread-30",
|
50
|
+
"short_message": "put 0 records to sink"
|
51
|
+
}
|
52
|
+
}
|
@@ -0,0 +1,53 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"uid": "0",
|
7
|
+
"gid": "0",
|
8
|
+
"cap_effective": "1fffffffff",
|
9
|
+
"systemd_slice": "system.slice",
|
10
|
+
"boot_id": "652998235194436283ff15d225e268cf",
|
11
|
+
"machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
12
|
+
"hostname": "mesosslave-private-015e1a3477fa81e5e",
|
13
|
+
"transport": "syslog",
|
14
|
+
"priority": "3",
|
15
|
+
"syslog_facility": "1",
|
16
|
+
"syslog_identifier": "mesos-slave",
|
17
|
+
"syslog_pid": "3445",
|
18
|
+
"pid": "3596",
|
19
|
+
"comm": "logger",
|
20
|
+
"exe": "/usr/bin/logger",
|
21
|
+
"cmdline": "logger -p user.err -t mesos-slave[3445]",
|
22
|
+
"systemd_cgroup": "/system.slice/mesos-slave.service",
|
23
|
+
"systemd_unit": "mesos-slave.service",
|
24
|
+
"selinux_context": "system_u:system_r:unconfined_service_t:s0",
|
25
|
+
"message": "I1204 10:25:20.448935 3677 slave.cpp:5920] Current disk usage 5.31%. Max allowed age: 4.234456934750000mins",
|
26
|
+
"source_realtime_timestamp": "1512383120449042"
|
27
|
+
}
|
28
|
+
},
|
29
|
+
"expected": {
|
30
|
+
"timestamp": 1512383120.449,
|
31
|
+
"_fluentd_tag": "journald.log",
|
32
|
+
"_uid": "0",
|
33
|
+
"_gid": "0",
|
34
|
+
"_cap_effective": "1fffffffff",
|
35
|
+
"_systemd_slice": "system.slice",
|
36
|
+
"_boot_id": "652998235194436283ff15d225e268cf",
|
37
|
+
"_machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
38
|
+
"host": "mesosslave-private-015e1a3477fa81e5e",
|
39
|
+
"_transport": "syslog",
|
40
|
+
"level": 3,
|
41
|
+
"facility": "user",
|
42
|
+
"_syslog_identifier": "mesos-slave",
|
43
|
+
"_syslog_pid": "3445",
|
44
|
+
"_pid": "3596",
|
45
|
+
"_comm": "logger",
|
46
|
+
"_exe": "/usr/bin/logger",
|
47
|
+
"_cmdline": "logger -p user.err -t mesos-slave[3445]",
|
48
|
+
"_systemd_cgroup": "/system.slice/mesos-slave.service",
|
49
|
+
"_systemd_unit": "mesos-slave.service",
|
50
|
+
"_selinux_context": "system_u:system_r:unconfined_service_t:s0",
|
51
|
+
"short_message": "I1204 10:25:20.448935 3677 slave.cpp:5920] Current disk usage 5.31%. Max allowed age: 4.234456934750000mins"
|
52
|
+
}
|
53
|
+
}
|
data/test/helper.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
4
|
+
require "test-unit"
|
5
|
+
require "fluent/test"
|
6
|
+
require "fluent/test/driver/output"
|
7
|
+
require "fluent/test/driver/formatter"
|
8
|
+
require "fluent/test/helpers"
|
9
|
+
|
10
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
11
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
12
|
+
|
13
|
+
def fixtures
|
14
|
+
path = File.expand_path("../fixtures", __FILE__)
|
15
|
+
all_fixture_files = Dir["#{path}/*.json"]
|
16
|
+
all_fixtures = {}
|
17
|
+
all_fixture_files.each do |file|
|
18
|
+
all_fixtures[File.basename(file)] = JSON.parse(File.read(file))
|
19
|
+
end
|
20
|
+
all_fixtures
|
21
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
require "fluent/plugin/formatter_gelf.rb"
|
5
|
+
|
6
|
+
class GelfFormatterTest < Test::Unit::TestCase
|
7
|
+
setup do
|
8
|
+
Fluent::Test.setup
|
9
|
+
@d = create_driver("")
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def create_driver(conf)
|
15
|
+
Fluent::Test::Driver::Formatter.new(Fluent::Plugin::GelfFormatter).configure(conf)
|
16
|
+
end
|
17
|
+
|
18
|
+
fixtures.each_pair do |file, v|
|
19
|
+
test "format #{file}" do
|
20
|
+
time = v["event"]["time"]
|
21
|
+
parsed_time = time.is_a?(Integer) ? time : event_time(time)
|
22
|
+
formatted = @d.instance.format(v["event"]["tag"], parsed_time, v["event"]["record"])
|
23
|
+
assert_equal v["expected"].to_json, formatted
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
require "fluent/plugin/out_gelf.rb"
|
5
|
+
require "test/unit/rr"
|
6
|
+
|
7
|
+
class GelfOutputTest < Test::Unit::TestCase
|
8
|
+
CONFIG = %(
|
9
|
+
host localhost
|
10
|
+
)
|
11
|
+
|
12
|
+
MULTI_RECORD_WRITE = "{\"timestamp\":1512335408.558,\"_fluentd_tag\":\"test\",\
|
13
|
+
\"_foo\":\"bar\",\"short_message\":\"(no message)\",\"level\":6}\u0000{\"timestamp\":1512335409.123,\
|
14
|
+
\"_fluentd_tag\":\"test\",\"_bar\":\"baz\",\"short_message\":\"(no message)\",\"level\":6}\u0000"
|
15
|
+
|
16
|
+
setup do
|
17
|
+
Fluent::Test.setup
|
18
|
+
@d = create_driver(CONFIG)
|
19
|
+
@stubbed_tcp = Object.new
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def create_driver(conf)
|
25
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::GelfOutput).configure(conf)
|
26
|
+
end
|
27
|
+
|
28
|
+
test "test config" do
|
29
|
+
assert_equal "localhost", @d.instance.host
|
30
|
+
assert_equal 12_201, @d.instance.port
|
31
|
+
end
|
32
|
+
|
33
|
+
test "write of multiple records chunk" do
|
34
|
+
mock(TCPSocket).new(@d.instance.host, @d.instance.port) { @stubbed_tcp }
|
35
|
+
mock(@stubbed_tcp).write(MULTI_RECORD_WRITE)
|
36
|
+
mock(@stubbed_tcp).close
|
37
|
+
@d.run(default_tag: "test") do
|
38
|
+
@d.feed(event_time("2017-12-03 21:10:08.558Z UTC"), "foo": "bar")
|
39
|
+
@d.feed(event_time("2017-12-03 21:10:09.123Z UTC"), "bar": "baz")
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
fixtures.each_pair do |file, v|
|
44
|
+
test "format #{file}" do
|
45
|
+
stub(TCPSocket).new { @stubbed_tcp }
|
46
|
+
stub(@stubbed_tcp).write
|
47
|
+
stub(@stubbed_tcp).close
|
48
|
+
@d.run(default_tag: v["event"]["tag"]) do
|
49
|
+
time = v["event"]["time"]
|
50
|
+
parsed_time = time.is_a?(Integer) ? time : event_time(time)
|
51
|
+
@d.feed(parsed_time, v["event"]["record"])
|
52
|
+
end
|
53
|
+
assert_equal [v["expected"].to_msgpack], @d.formatted
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
metadata
CHANGED
@@ -1,43 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gelf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
- Funding Circle
|
7
|
+
- Funding Circle
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: oj
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 3.3.10
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 3.3.10
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 2.1.4
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 2.1.4
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 12.3.3
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 12.3.3
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: test-unit
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -100,13 +114,39 @@ dependencies:
|
|
100
114
|
- - "<"
|
101
115
|
- !ruby/object:Gem::Version
|
102
116
|
version: '2'
|
103
|
-
description:
|
117
|
+
description: Converts fluentd log events into GELF format and sends them to Graylog
|
104
118
|
email:
|
105
119
|
- engineering+fluent-plugin-gelf@fundingcircle.com
|
106
120
|
executables: []
|
107
121
|
extensions: []
|
108
122
|
extra_rdoc_files: []
|
109
|
-
files:
|
123
|
+
files:
|
124
|
+
- ".circleci/config.yml"
|
125
|
+
- ".gitignore"
|
126
|
+
- ".rubocop.yml"
|
127
|
+
- ".ruby-version"
|
128
|
+
- Gemfile
|
129
|
+
- LICENSE
|
130
|
+
- README.md
|
131
|
+
- Rakefile
|
132
|
+
- fluent-plugin-gelf.gemspec
|
133
|
+
- lib/fluent/plugin/formatter_gelf.rb
|
134
|
+
- lib/fluent/plugin/gelf_plugin_util.rb
|
135
|
+
- lib/fluent/plugin/out_gelf.rb
|
136
|
+
- test/fixtures/docker_futureproof.json
|
137
|
+
- test/fixtures/docker_gelf_formatted.json
|
138
|
+
- test/fixtures/docker_gelf_formatted_already_extracted.json
|
139
|
+
- test/fixtures/docker_inner_json.json
|
140
|
+
- test/fixtures/docker_inner_json_with_time.json
|
141
|
+
- test/fixtures/docker_unformatted.json
|
142
|
+
- test/fixtures/level_from_tag.json
|
143
|
+
- test/fixtures/message_json_parsable_to_nil.json
|
144
|
+
- test/fixtures/systemd.json
|
145
|
+
- test/fixtures/systemd_inner_json.json
|
146
|
+
- test/fixtures/systemd_with_real_timestamp.json
|
147
|
+
- test/helper.rb
|
148
|
+
- test/plugin/test_formatter_gelf.rb
|
149
|
+
- test/plugin/test_out_gelf.rb
|
110
150
|
homepage:
|
111
151
|
licenses:
|
112
152
|
- BSD-3-Clause
|
@@ -122,13 +162,26 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
122
162
|
version: '0'
|
123
163
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
164
|
requirements:
|
125
|
-
- - "
|
165
|
+
- - ">="
|
126
166
|
- !ruby/object:Gem::Version
|
127
|
-
version:
|
167
|
+
version: '0'
|
128
168
|
requirements: []
|
129
|
-
|
130
|
-
rubygems_version: 2.6.14
|
169
|
+
rubygems_version: 3.0.3
|
131
170
|
signing_key:
|
132
171
|
specification_version: 4
|
133
|
-
summary:
|
134
|
-
test_files:
|
172
|
+
summary: Graylog output plugin for fluentd
|
173
|
+
test_files:
|
174
|
+
- test/fixtures/docker_futureproof.json
|
175
|
+
- test/fixtures/docker_gelf_formatted.json
|
176
|
+
- test/fixtures/docker_gelf_formatted_already_extracted.json
|
177
|
+
- test/fixtures/docker_inner_json.json
|
178
|
+
- test/fixtures/docker_inner_json_with_time.json
|
179
|
+
- test/fixtures/docker_unformatted.json
|
180
|
+
- test/fixtures/level_from_tag.json
|
181
|
+
- test/fixtures/message_json_parsable_to_nil.json
|
182
|
+
- test/fixtures/systemd.json
|
183
|
+
- test/fixtures/systemd_inner_json.json
|
184
|
+
- test/fixtures/systemd_with_real_timestamp.json
|
185
|
+
- test/helper.rb
|
186
|
+
- test/plugin/test_formatter_gelf.rb
|
187
|
+
- test/plugin/test_out_gelf.rb
|