fluent-plugin-gelf 0.0.0.dev1 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.circleci/config.yml +76 -0
- data/.gitignore +2 -0
- data/.rubocop.yml +12 -0
- data/.ruby-version +1 -0
- data/Gemfile +5 -0
- data/LICENSE +29 -0
- data/README.md +60 -0
- data/Rakefile +18 -0
- data/fluent-plugin-gelf.gemspec +31 -0
- data/lib/fluent/plugin/formatter_gelf.rb +18 -0
- data/lib/fluent/plugin/gelf_plugin_util.rb +114 -0
- data/lib/fluent/plugin/out_gelf.rb +60 -0
- data/test/fixtures/docker_futureproof.json +32 -0
- data/test/fixtures/docker_gelf_formatted.json +45 -0
- data/test/fixtures/docker_gelf_formatted_already_extracted.json +61 -0
- data/test/fixtures/docker_inner_json.json +32 -0
- data/test/fixtures/docker_inner_json_with_time.json +32 -0
- data/test/fixtures/docker_unformatted.json +31 -0
- data/test/fixtures/level_from_tag.json +23 -0
- data/test/fixtures/message_json_parsable_to_nil.json +50 -0
- data/test/fixtures/systemd.json +50 -0
- data/test/fixtures/systemd_inner_json.json +52 -0
- data/test/fixtures/systemd_with_real_timestamp.json +53 -0
- data/test/helper.rb +21 -0
- data/test/plugin/test_formatter_gelf.rb +26 -0
- data/test/plugin/test_out_gelf.rb +56 -0
- metadata +70 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 70a3623ed2adde0ca73a4f9b406657a50ba9cbb33ffbe5512fad9c9fb76591db
|
4
|
+
data.tar.gz: 6a7acdaa55e1088efbce002224c88c8795a84b5d1ac634f48dd2c5e063c3f4ad
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76b60605cf69b29427372dedf3792132a761410e23f7bf7f918290694e1eb06225f90f5263a6a54c08ca7bc52ab44a8cdbb0baa4cf34d31d685bb4ffc75f2e7f
|
7
|
+
data.tar.gz: 73742730f414d4643e12914c15cbdd61a4900ee50219b74538afe312ca74243a3a249cd77c26c63b2ec20aa1260e3297ca0f60b29e6cb873bddc0cede6ef8492
|
@@ -0,0 +1,76 @@
|
|
1
|
+
version: 2.1
|
2
|
+
commands:
|
3
|
+
install_gems:
|
4
|
+
steps:
|
5
|
+
- restore_cache:
|
6
|
+
key: fluent-plugin-gelf2-{{ checksum "fluent-plugin-gelf.gemspec" }}-{{ checksum "Gemfile" }}
|
7
|
+
- run: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3 --full-index
|
8
|
+
- save_cache:
|
9
|
+
key: fluent-plugin-gelf2-{{ checksum "fluent-plugin-gelf.gemspec" }}-{{ checksum "Gemfile" }}
|
10
|
+
paths:
|
11
|
+
- vendor/bundle
|
12
|
+
executors:
|
13
|
+
ruby:
|
14
|
+
parameters:
|
15
|
+
version:
|
16
|
+
type: string
|
17
|
+
default: latest
|
18
|
+
docker:
|
19
|
+
- image: circleci/ruby:<< parameters.version >>
|
20
|
+
working_directory: ~/fluent-plugin-gelf
|
21
|
+
jobs:
|
22
|
+
publish:
|
23
|
+
executor: ruby
|
24
|
+
steps:
|
25
|
+
- checkout
|
26
|
+
- install_gems
|
27
|
+
- run:
|
28
|
+
name: Configure RubyGems API key
|
29
|
+
command: |
|
30
|
+
mkdir -p ~/.gem
|
31
|
+
echo ":rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials
|
32
|
+
chmod 0600 ~/.gem/credentials
|
33
|
+
- run:
|
34
|
+
name: Build gem
|
35
|
+
command: bundle exec rake build --trace
|
36
|
+
- run:
|
37
|
+
name: Release gem
|
38
|
+
command: bundle exec rake release:rubygem_push --trace
|
39
|
+
rubocop:
|
40
|
+
executor: ruby
|
41
|
+
steps:
|
42
|
+
- checkout
|
43
|
+
- install_gems
|
44
|
+
- run:
|
45
|
+
name: Run RuboCop
|
46
|
+
command: bundle exec rake rubocop --trace
|
47
|
+
test:
|
48
|
+
executor: ruby
|
49
|
+
steps:
|
50
|
+
- checkout
|
51
|
+
- install_gems
|
52
|
+
- run:
|
53
|
+
name: Run tests
|
54
|
+
command: bundle exec rake test --trace
|
55
|
+
|
56
|
+
workflows:
|
57
|
+
test:
|
58
|
+
jobs:
|
59
|
+
- rubocop:
|
60
|
+
filters:
|
61
|
+
tags:
|
62
|
+
only: /.*/
|
63
|
+
- test:
|
64
|
+
filters:
|
65
|
+
tags:
|
66
|
+
only: /.*/
|
67
|
+
- publish:
|
68
|
+
context: org-rubygems
|
69
|
+
filters:
|
70
|
+
branches:
|
71
|
+
ignore: /.*/
|
72
|
+
tags:
|
73
|
+
only: /^v\d{1,2}\.\d{1,2}\.\d{1,2}.*/
|
74
|
+
requires:
|
75
|
+
- rubocop
|
76
|
+
- test
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 2.4
|
3
|
+
Style/TrailingCommaInArguments:
|
4
|
+
EnforcedStyleForMultiline: comma
|
5
|
+
Style/TrailingCommaInLiteral:
|
6
|
+
EnforcedStyleForMultiline: comma
|
7
|
+
Metrics/LineLength:
|
8
|
+
Max: 120
|
9
|
+
Style/StringLiterals:
|
10
|
+
EnforcedStyle: double_quotes
|
11
|
+
Style/Documentation:
|
12
|
+
Enabled: false
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.4.2
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
BSD 3-Clause License
|
2
|
+
|
3
|
+
Copyright (c) 2017, Funding Circle
|
4
|
+
All rights reserved.
|
5
|
+
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
8
|
+
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
10
|
+
list of conditions and the following disclaimer.
|
11
|
+
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
14
|
+
and/or other materials provided with the distribution.
|
15
|
+
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
17
|
+
contributors may be used to endorse or promote products derived from
|
18
|
+
this software without specific prior written permission.
|
19
|
+
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
# fluent-plugin-gelf
|
2
|
+
|
3
|
+
[](https://circleci.com/gh/FundingCircle/fluent-plugin-gelf/tree/master)
|
4
|
+
|
5
|
+
A [fluentd](https://www.fluentd.org/) output plugin for sending log events to
|
6
|
+
[Graylog](https://docs.graylog.org/).
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
### RubyGems
|
11
|
+
|
12
|
+
```
|
13
|
+
$ gem install fluent-plugin-gelf
|
14
|
+
```
|
15
|
+
|
16
|
+
### Bundler
|
17
|
+
|
18
|
+
Add this line to your application's Gemfile:
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
gem "fluent-plugin-gelf"
|
22
|
+
```
|
23
|
+
|
24
|
+
And then execute:
|
25
|
+
|
26
|
+
```
|
27
|
+
$ bundle
|
28
|
+
```
|
29
|
+
|
30
|
+
## Usage
|
31
|
+
|
32
|
+
This `gelf` plugin is for fluentd v1.0 or later.
|
33
|
+
|
34
|
+
```
|
35
|
+
<match app.**>
|
36
|
+
output_data_type gelf
|
37
|
+
</match>
|
38
|
+
```
|
39
|
+
|
40
|
+
## Configuration
|
41
|
+
|
42
|
+
#### host (string) (required)
|
43
|
+
|
44
|
+
The hostname of your Graylog cluster.
|
45
|
+
|
46
|
+
####o port (integer) (optional)
|
47
|
+
|
48
|
+
The TCP port of your Graylog cluster. Default value: `12201`.
|
49
|
+
|
50
|
+
## Releases
|
51
|
+
|
52
|
+
The CircleCI build for this project manages gem releases to RubyGems. To release
|
53
|
+
a new version of this gem, create a tag for the version and push it. It will
|
54
|
+
then be built and deployed automatically.
|
55
|
+
|
56
|
+
## Copyright and License
|
57
|
+
|
58
|
+
Copyright © 2017 Funding Circle Ltd.
|
59
|
+
|
60
|
+
Licensed under the [BSD 3-Clause License](LICENSE).
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "bundler"
|
4
|
+
Bundler::GemHelper.install_tasks
|
5
|
+
|
6
|
+
require "rake/testtask"
|
7
|
+
require "rubocop/rake_task"
|
8
|
+
|
9
|
+
Rake::TestTask.new(:test) do |t|
|
10
|
+
t.libs.push("lib", "test")
|
11
|
+
t.test_files = FileList["test/**/test_*.rb"]
|
12
|
+
t.verbose = true
|
13
|
+
t.warning = true
|
14
|
+
end
|
15
|
+
|
16
|
+
RuboCop::RakeTask.new(:rubocop)
|
17
|
+
|
18
|
+
task default: %i[rubocop test]
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path("../lib", __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "fluent-plugin-gelf"
|
8
|
+
spec.version = "0.2.4"
|
9
|
+
spec.authors = ["Funding Circle"]
|
10
|
+
spec.email = ["engineering+fluent-plugin-gelf@fundingcircle.com"]
|
11
|
+
|
12
|
+
spec.summary = "Graylog output plugin for fluentd"
|
13
|
+
spec.description = "Converts fluentd log events into GELF format and sends them to Graylog"
|
14
|
+
spec.license = "BSD-3-Clause"
|
15
|
+
|
16
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
17
|
+
f.match(%r{^(test|spec|features)/})
|
18
|
+
end
|
19
|
+
spec.files = files
|
20
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
|
+
spec.test_files = test_files
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_development_dependency "oj", "~> 3.3.10"
|
25
|
+
spec.add_development_dependency "bundler", "~> 2.1.4"
|
26
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
27
|
+
spec.add_development_dependency "test-unit", "~> 3.0"
|
28
|
+
spec.add_development_dependency "test-unit-rr", "~> 1.0.5"
|
29
|
+
spec.add_development_dependency "rubocop", "~> 0.50.0"
|
30
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
31
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "fluent/plugin/formatter"
|
4
|
+
require "fluent/plugin/gelf_plugin_util"
|
5
|
+
require "yajl"
|
6
|
+
|
7
|
+
module Fluent
|
8
|
+
module Plugin
|
9
|
+
class GelfFormatter < Fluent::Plugin::Formatter
|
10
|
+
Fluent::Plugin.register_formatter("gelf", self)
|
11
|
+
include Fluent::GelfPluginUtil
|
12
|
+
|
13
|
+
def format(tag, time, record)
|
14
|
+
Yajl::Encoder.encode(make_gelfentry(tag, time, record))
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "oj"
|
4
|
+
require "date"
|
5
|
+
|
6
|
+
module Fluent
|
7
|
+
module GelfPluginUtil
|
8
|
+
SYSLOG_FACILITY = {
|
9
|
+
"0" => "kern",
|
10
|
+
"1" => "user",
|
11
|
+
"2" => "mail",
|
12
|
+
"3" => "daemon",
|
13
|
+
"4" => "auth",
|
14
|
+
"5" => "syslog",
|
15
|
+
"6" => "lpr",
|
16
|
+
"7" => "news",
|
17
|
+
"8" => "uucp",
|
18
|
+
"9" => "cron",
|
19
|
+
"10" => "authpriv",
|
20
|
+
"16" => "local0",
|
21
|
+
"17" => "local1",
|
22
|
+
"18" => "local2",
|
23
|
+
"19" => "local3",
|
24
|
+
"20" => "local4",
|
25
|
+
"21" => "local5",
|
26
|
+
"22" => "local6",
|
27
|
+
"23" => "local7",
|
28
|
+
}.freeze
|
29
|
+
|
30
|
+
LEVEL_MAPPING = {
|
31
|
+
"error" => 3,
|
32
|
+
"warn" => 4,
|
33
|
+
"info" => 6,
|
34
|
+
"debug" => 7,
|
35
|
+
}.freeze
|
36
|
+
|
37
|
+
def merge_inner_json(record, key)
|
38
|
+
return record unless record[key]
|
39
|
+
json = Oj.load(record[key].strip)
|
40
|
+
return record unless json
|
41
|
+
json["host"] = record["host"] # preserve host
|
42
|
+
record.delete(key)
|
43
|
+
record.merge(json)
|
44
|
+
rescue Oj::ParseError
|
45
|
+
record
|
46
|
+
end
|
47
|
+
|
48
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
|
49
|
+
def make_gelfentry(tag, time, record)
|
50
|
+
record = merge_inner_json(record, "message")
|
51
|
+
record = merge_inner_json(record, "log")
|
52
|
+
gelfentry = {}
|
53
|
+
gelfentry["timestamp"] = if defined?(Fluent::EventTime) && time.is_a?(Fluent::EventTime)
|
54
|
+
time.sec + (time.nsec.to_f / 1_000_000_000).round(3)
|
55
|
+
else
|
56
|
+
time
|
57
|
+
end
|
58
|
+
|
59
|
+
gelfentry["_fluentd_tag"] = tag
|
60
|
+
|
61
|
+
record.each_pair do |k, v| # rubocop:disable Metrics/BlockLength
|
62
|
+
case k
|
63
|
+
when "timestamp", "time"
|
64
|
+
gelfentry["timestamp"] = if v.is_a?(Integer) || v.is_a?(Float)
|
65
|
+
v
|
66
|
+
else
|
67
|
+
begin
|
68
|
+
(DateTime.parse(v).strftime("%Q").to_f / 1_000).round(3)
|
69
|
+
rescue ArgumentError
|
70
|
+
v
|
71
|
+
end
|
72
|
+
end
|
73
|
+
when "msec" then
|
74
|
+
if time.is_a?(Integer) && record["timestamp"].nil? && record["time"].nil?
|
75
|
+
gelfentry["timestamp"] = "#{time}.#{v}".to_f
|
76
|
+
else
|
77
|
+
gelfentry["_msec"] = v
|
78
|
+
end
|
79
|
+
when "level"
|
80
|
+
gelfentry["level"] = if v.is_a?(Integer)
|
81
|
+
v
|
82
|
+
else
|
83
|
+
level = LEVEL_MAPPING.keys.find { |l| v.downcase =~ /#{l}/ }
|
84
|
+
level.nil? ? v : LEVEL_MAPPING[level]
|
85
|
+
end
|
86
|
+
when "source_realtime_timestamp" then gelfentry["timestamp"] = (v.to_f / 1_000_000).round(3)
|
87
|
+
when "host", "hostname" then gelfentry["host"] = v.to_s
|
88
|
+
when "priority" then gelfentry["level"] = v.to_i
|
89
|
+
when "syslog_facility" then gelfentry["facility"] = SYSLOG_FACILITY[v]
|
90
|
+
when "short_message", "version", "full_message", "facility", "file", "line" then gelfentry[k] = v
|
91
|
+
else
|
92
|
+
k.to_s.start_with?("_") ? gelfentry[k] = v : gelfentry["_#{k}"] = v
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
if gelfentry["short_message"].nil? || gelfentry["short_message"].to_s.empty?
|
97
|
+
gelfentry["short_message"] = if gelfentry.key?("_message") && !gelfentry["_message"].to_s.empty?
|
98
|
+
gelfentry.delete("_message")
|
99
|
+
elsif gelfentry.key?("_log") && !gelfentry["_log"].to_s.empty?
|
100
|
+
gelfentry.delete("_log")
|
101
|
+
else
|
102
|
+
"(no message)"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
if gelfentry["level"].nil?
|
107
|
+
level = LEVEL_MAPPING.keys.find { |k| tag.downcase =~ /\.#{k}/ }
|
108
|
+
gelfentry["level"] = level.nil? ? 6 : LEVEL_MAPPING[level]
|
109
|
+
end
|
110
|
+
|
111
|
+
gelfentry
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "fluent/plugin/output"
|
4
|
+
require "fluent/plugin/gelf_plugin_util"
|
5
|
+
|
6
|
+
module Fluent
|
7
|
+
module Plugin
|
8
|
+
class GelfOutput < Fluent::Plugin::Output
|
9
|
+
Fluent::Plugin.register_output("gelf", self)
|
10
|
+
|
11
|
+
include Fluent::GelfPluginUtil
|
12
|
+
|
13
|
+
config_param :host, :string, default: nil
|
14
|
+
config_param :port, :integer, default: 12_201
|
15
|
+
|
16
|
+
def configure(conf)
|
17
|
+
super
|
18
|
+
raise ConfigError, "'host' parameter is required" unless conf.key?("host")
|
19
|
+
end
|
20
|
+
|
21
|
+
def start
|
22
|
+
super
|
23
|
+
end
|
24
|
+
|
25
|
+
def shutdown
|
26
|
+
super
|
27
|
+
end
|
28
|
+
|
29
|
+
def format(tag, time, record)
|
30
|
+
make_gelfentry(tag, time, record).to_msgpack
|
31
|
+
end
|
32
|
+
|
33
|
+
def write(chunk) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
34
|
+
records = []
|
35
|
+
chunk.msgpack_each do |record|
|
36
|
+
records.push JSON.dump(record) + "\0" # Message delimited by null char
|
37
|
+
end
|
38
|
+
|
39
|
+
log.debug "establishing connection with GrayLog"
|
40
|
+
socket = TCPSocket.new(@host, @port)
|
41
|
+
|
42
|
+
begin
|
43
|
+
log.debug "sending #{records.count} records in batch"
|
44
|
+
socket.write(records.join)
|
45
|
+
ensure
|
46
|
+
log.debug "closing connection with GrayLog"
|
47
|
+
socket.close
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def formatted_to_msgpack_binary
|
52
|
+
true
|
53
|
+
end
|
54
|
+
|
55
|
+
def multi_workers_ready?
|
56
|
+
true
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.312fe8f61472",
|
4
|
+
"comment": "once msec set https://github.com/moby/moby/issues/17181",
|
5
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
6
|
+
"record": {
|
7
|
+
"container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
8
|
+
"container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
9
|
+
"source": "stderr",
|
10
|
+
"log": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "vault-gatekeeper-mesos",
|
13
|
+
"mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
14
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
15
|
+
"msec": "229"
|
16
|
+
}
|
17
|
+
},
|
18
|
+
"expected": {
|
19
|
+
"timestamp": 1512335408.558,
|
20
|
+
"_fluentd_tag": "docker.312fe8f61472",
|
21
|
+
"_container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
22
|
+
"_container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
23
|
+
"_source": "stderr",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "vault-gatekeeper-mesos",
|
26
|
+
"_mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
27
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
28
|
+
"_msec": "229",
|
29
|
+
"short_message": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
30
|
+
"level": 6
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,45 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"comment": "docker log already parsed by https://github.com/joshughes/fluent-plugin-mesosphere-filter (merge_json_log)",
|
6
|
+
"record": {
|
7
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
8
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
9
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
10
|
+
"source": "stdout",
|
11
|
+
"log": "{\"_request.status\":200,\"_request.method\":\"GET\",\"_request.path\":\"/\",\"_request.params\":{},\"_request.request_id\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"_request.request_ip\":\"10.42.8.20\",\"_request.user_agent\":\"spray-can/1.3.4\",\"_request.duration\":1,\"_type\":\"request\",\"_service.name\":\"my_app\",\"_service.version\":\"unknown.sha\",\"_tags\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"short_message\":\"GET / 200 in 1ms\",\"timestamp\":1512385332.824,\"host\":\"d14490734a2f\",\"level\":6,\"version\":\"1.1\"}",
|
12
|
+
"mesos_framework": "marathon",
|
13
|
+
"app": "loga",
|
14
|
+
"mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
15
|
+
"msec": "825"
|
16
|
+
}
|
17
|
+
},
|
18
|
+
"expected": {
|
19
|
+
"timestamp": 1512385332.824,
|
20
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
21
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
22
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
23
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
24
|
+
"_source": "stdout",
|
25
|
+
"_mesos_framework": "marathon",
|
26
|
+
"_app": "loga",
|
27
|
+
"_mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
28
|
+
"_msec": "825",
|
29
|
+
"_request.status": 200,
|
30
|
+
"_request.method": "GET",
|
31
|
+
"_request.path": "/",
|
32
|
+
"_request.params": {},
|
33
|
+
"_request.request_id": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
34
|
+
"_request.request_ip": "10.42.8.20",
|
35
|
+
"_request.user_agent": "spray-can/1.3.4",
|
36
|
+
"_request.duration": 1,
|
37
|
+
"_type": "request",
|
38
|
+
"_service.name": "my_app",
|
39
|
+
"_service.version": "unknown.sha",
|
40
|
+
"_tags": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
41
|
+
"short_message": "GET / 200 in 1ms",
|
42
|
+
"level": 6,
|
43
|
+
"version": "1.1"
|
44
|
+
}
|
45
|
+
}
|
@@ -0,0 +1,61 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"comment": "docker log already parsed by https://github.com/joshughes/fluent-plugin-mesosphere-filter (merge_json_log)",
|
6
|
+
"record": {
|
7
|
+
"_request.status": 200,
|
8
|
+
"_request.method": "GET",
|
9
|
+
"_request.path": "/",
|
10
|
+
"_request.params": {},
|
11
|
+
"_request.request_id": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
12
|
+
"_request.request_ip": "10.42.8.20",
|
13
|
+
"_request.user_agent": "spray-can/1.3.4",
|
14
|
+
"_request.duration": 1,
|
15
|
+
"_type": "request",
|
16
|
+
"_service.name": "my_app",
|
17
|
+
"_service.version": "unknown.sha",
|
18
|
+
"_tags": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
19
|
+
"short_message": "GET / 200 in 1ms",
|
20
|
+
"timestamp": 1512385332.824,
|
21
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
22
|
+
"level": 6,
|
23
|
+
"version": "1.1",
|
24
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
25
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
26
|
+
"source": "stdout",
|
27
|
+
"log": "{\"_request.status\":200,\"_request.method\":\"GET\",\"_request.path\":\"/\",\"_request.params\":{},\"_request.request_id\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"_request.request_ip\":\"10.42.8.20\",\"_request.user_agent\":\"spray-can/1.3.4\",\"_request.duration\":1,\"_type\":\"request\",\"_service.name\":\"my_app\",\"_service.version\":\"unknown.sha\",\"_tags\":\"d782a576-12db-4c7e-b8ab-a8bd06240786\",\"short_message\":\"GET / 200 in 1ms\",\"timestamp\":1512385332.824,\"host\":\"d14490734a2f\",\"level\":6,\"version\":\"1.1\"}",
|
28
|
+
"mesos_framework": "marathon",
|
29
|
+
"app": "loga",
|
30
|
+
"mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
31
|
+
"msec": "825"
|
32
|
+
}
|
33
|
+
},
|
34
|
+
"expected": {
|
35
|
+
"timestamp": 1512385332.824,
|
36
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
37
|
+
"_request.status": 200,
|
38
|
+
"_request.method": "GET",
|
39
|
+
"_request.path": "/",
|
40
|
+
"_request.params": {},
|
41
|
+
"_request.request_id": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
42
|
+
"_request.request_ip": "10.42.8.20",
|
43
|
+
"_request.user_agent": "spray-can/1.3.4",
|
44
|
+
"_request.duration": 1,
|
45
|
+
"_type": "request",
|
46
|
+
"_service.name": "my_app",
|
47
|
+
"_service.version": "unknown.sha",
|
48
|
+
"_tags": "d782a576-12db-4c7e-b8ab-a8bd06240786",
|
49
|
+
"short_message": "GET / 200 in 1ms",
|
50
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
51
|
+
"level": 6,
|
52
|
+
"version": "1.1",
|
53
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
54
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
55
|
+
"_source": "stdout",
|
56
|
+
"_mesos_framework": "marathon",
|
57
|
+
"_app": "loga",
|
58
|
+
"_mesos_task_id": "loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
59
|
+
"_msec": "825"
|
60
|
+
}
|
61
|
+
}
|
@@ -0,0 +1,32 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
8
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
9
|
+
"source": "stdout",
|
10
|
+
"log": "{\"foo\":\"bar\",\"bar\":\"baz\"}",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "other",
|
13
|
+
"mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
14
|
+
"msec": "825"
|
15
|
+
}
|
16
|
+
},
|
17
|
+
"expected": {
|
18
|
+
"timestamp": 15123853321.825,
|
19
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
20
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
21
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
22
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
23
|
+
"_source": "stdout",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "other",
|
26
|
+
"_mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
27
|
+
"_foo": "bar",
|
28
|
+
"_bar": "baz",
|
29
|
+
"short_message": "(no message)",
|
30
|
+
"level": 6
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,32 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.d14490734a2f",
|
4
|
+
"time": 15123853321,
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
8
|
+
"container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
9
|
+
"source": "stdout",
|
10
|
+
"log": "{\"message\":\"foo\",\"bar\":\"baz\",\"time\":\"2020-01-29T16:17:43Z\"}",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "other",
|
13
|
+
"mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
14
|
+
"msec": "825"
|
15
|
+
}
|
16
|
+
},
|
17
|
+
"expected": {
|
18
|
+
"timestamp": 1580314663.0,
|
19
|
+
"_fluentd_tag": "docker.d14490734a2f",
|
20
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
21
|
+
"_container_id": "d14490734a2f217609123a06e44df860ac573c76882a7a27e98c1a8e29d32664",
|
22
|
+
"_container_name": "/mesos-10d4a50f-e134-4538-b390-3051c50c431d",
|
23
|
+
"_source": "stdout",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "other",
|
26
|
+
"_mesos_task_id": "other.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2",
|
27
|
+
"_msec": "825",
|
28
|
+
"_bar": "baz",
|
29
|
+
"short_message": "foo",
|
30
|
+
"level": 6
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,31 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "docker.312fe8f61472",
|
4
|
+
"comment": "no msec from docker: https://github.com/moby/moby/issues/17181#issuecomment-345594658",
|
5
|
+
"time": 1512424624,
|
6
|
+
"record": {
|
7
|
+
"container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
8
|
+
"container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
9
|
+
"source": "stderr",
|
10
|
+
"log": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
11
|
+
"mesos_framework": "marathon",
|
12
|
+
"app": "vault-gatekeeper-mesos",
|
13
|
+
"mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
14
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
15
|
+
"msec": "229"
|
16
|
+
}
|
17
|
+
},
|
18
|
+
"expected": {
|
19
|
+
"timestamp": 1512424624.229,
|
20
|
+
"_fluentd_tag": "docker.312fe8f61472",
|
21
|
+
"_container_id": "312fe8f61472d866831525403681ff5d012352ca67b43c685a4d7b4efb60e60d",
|
22
|
+
"_container_name": "/mesos-61cbfd86-74f0-428c-9515-2d36e74b2dc5",
|
23
|
+
"_source": "stderr",
|
24
|
+
"_mesos_framework": "marathon",
|
25
|
+
"_app": "vault-gatekeeper-mesos",
|
26
|
+
"_mesos_task_id": "vault-gatekeeper-mesos.e6848c5f-d5c1-11e7-a79a-02a64f45f7a2",
|
27
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
28
|
+
"short_message": "2017/12/04 11:02:06 http: TLS handshake error from 10.42.9.114:33880: EOF",
|
29
|
+
"level": 6
|
30
|
+
}
|
31
|
+
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "fluent.warn",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"retry_time": 0,
|
7
|
+
"next_retry_seconds": "2017-12-04 22:56:27 +0000",
|
8
|
+
"chunk": "55f8ba0c48815f9a61ede8f6c2a68208",
|
9
|
+
"error": "#<Errno::ECONNREFUSED: Connection refused...",
|
10
|
+
"message": "failed to flush the buffer..."
|
11
|
+
}
|
12
|
+
},
|
13
|
+
"expected": {
|
14
|
+
"timestamp": 1512335408.558,
|
15
|
+
"_fluentd_tag": "fluent.warn",
|
16
|
+
"_retry_time": 0,
|
17
|
+
"_next_retry_seconds": "2017-12-04 22:56:27 +0000",
|
18
|
+
"_chunk": "55f8ba0c48815f9a61ede8f6c2a68208",
|
19
|
+
"_error": "#<Errno::ECONNREFUSED: Connection refused...",
|
20
|
+
"short_message": "failed to flush the buffer...",
|
21
|
+
"level": 4
|
22
|
+
}
|
23
|
+
}
|
@@ -0,0 +1,50 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"transport": "stdout",
|
8
|
+
"priority": "6",
|
9
|
+
"syslog_facility": "3",
|
10
|
+
"systemd_slice": "system.slice",
|
11
|
+
"selinux_context": "system_u:system_r:init_t:s0",
|
12
|
+
"boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
13
|
+
"machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
14
|
+
"cap_effective": "0",
|
15
|
+
"syslog_identifier": "consul-start.sh",
|
16
|
+
"pid": "3388",
|
17
|
+
"uid": "997",
|
18
|
+
"gid": "994",
|
19
|
+
"comm": "consul",
|
20
|
+
"exe": "/usr/local/bin/consul",
|
21
|
+
"cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
22
|
+
"systemd_cgroup": "/system.slice/consul.service",
|
23
|
+
"systemd_unit": "consul.service",
|
24
|
+
"message": " // Clearing the cache for the prod environment with debug false"
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"expected": {
|
28
|
+
"timestamp": 1512335408.558,
|
29
|
+
"_fluentd_tag": "journald.log",
|
30
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
31
|
+
"_transport": "stdout",
|
32
|
+
"level": 6,
|
33
|
+
"facility": "daemon",
|
34
|
+
"_systemd_slice": "system.slice",
|
35
|
+
"_selinux_context": "system_u:system_r:init_t:s0",
|
36
|
+
"_boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
37
|
+
"_machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
38
|
+
"_cap_effective": "0",
|
39
|
+
"_syslog_identifier": "consul-start.sh",
|
40
|
+
"_pid": "3388",
|
41
|
+
"_uid": "997",
|
42
|
+
"_gid": "994",
|
43
|
+
"_comm": "consul",
|
44
|
+
"_exe": "/usr/local/bin/consul",
|
45
|
+
"_cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
46
|
+
"_systemd_cgroup": "/system.slice/consul.service",
|
47
|
+
"_systemd_unit": "consul.service",
|
48
|
+
"short_message": " // Clearing the cache for the prod environment with debug false"
|
49
|
+
}
|
50
|
+
}
|
@@ -0,0 +1,50 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
7
|
+
"transport": "stdout",
|
8
|
+
"priority": "6",
|
9
|
+
"syslog_facility": "3",
|
10
|
+
"systemd_slice": "system.slice",
|
11
|
+
"selinux_context": "system_u:system_r:init_t:s0",
|
12
|
+
"boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
13
|
+
"machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
14
|
+
"cap_effective": "0",
|
15
|
+
"syslog_identifier": "consul-start.sh",
|
16
|
+
"pid": "3388",
|
17
|
+
"uid": "997",
|
18
|
+
"gid": "994",
|
19
|
+
"comm": "consul",
|
20
|
+
"exe": "/usr/local/bin/consul",
|
21
|
+
"cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
22
|
+
"systemd_cgroup": "/system.slice/consul.service",
|
23
|
+
"systemd_unit": "consul.service",
|
24
|
+
"message": "2017/12/03 21:10:08 [INFO] agent: Synced check 'service:loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2_loga_7403'"
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"expected": {
|
28
|
+
"timestamp": 1512335408.558,
|
29
|
+
"_fluentd_tag": "journald.log",
|
30
|
+
"host": "mesosslave-private-06a58a3ce10a783cf",
|
31
|
+
"_transport": "stdout",
|
32
|
+
"level": 6,
|
33
|
+
"facility": "daemon",
|
34
|
+
"_systemd_slice": "system.slice",
|
35
|
+
"_selinux_context": "system_u:system_r:init_t:s0",
|
36
|
+
"_boot_id": "9b18b8b74048480aa9f6fc268c51a96d",
|
37
|
+
"_machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
38
|
+
"_cap_effective": "0",
|
39
|
+
"_syslog_identifier": "consul-start.sh",
|
40
|
+
"_pid": "3388",
|
41
|
+
"_uid": "997",
|
42
|
+
"_gid": "994",
|
43
|
+
"_comm": "consul",
|
44
|
+
"_exe": "/usr/local/bin/consul",
|
45
|
+
"_cmdline": "/usr/local/bin/consul agent -config-dir /usr/local/etc/consul -protocol 3 -bind 10.42.9.114 -datacenter eu-central-1 -retry-join provider=aws tag_key=Consul_environment tag_value=unstable -recursor localhost:53 -pid-file /run/consul/consul.pid -enable-script-checks -raft-protocol 2",
|
46
|
+
"_systemd_cgroup": "/system.slice/consul.service",
|
47
|
+
"_systemd_unit": "consul.service",
|
48
|
+
"short_message": "2017/12/03 21:10:08 [INFO] agent: Synced check 'service:loga.4b859ca4-d6e2-11e7-a79a-02a64f45f7a2_loga_7403'"
|
49
|
+
}
|
50
|
+
}
|
@@ -0,0 +1,52 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2018-01-09 15:16:47.693Z UTC",
|
5
|
+
"record": {
|
6
|
+
"host": "mesosslave-private-0690bffd1e39d7462",
|
7
|
+
"transport": "stdout",
|
8
|
+
"priority": "6",
|
9
|
+
"syslog_facility": "3",
|
10
|
+
"systemd_slice": "system.slice",
|
11
|
+
"selinux_context": "system_u:system_r:container_runtime_t:s0",
|
12
|
+
"boot_id": "a83a6f2a9aa545d18188387fcff0dea9",
|
13
|
+
"machine_id": "f073c429a7456b53ec3e2c53460c5c8f",
|
14
|
+
"cap_effective": "1fffffffff",
|
15
|
+
"syslog_identifier": "kafka-connect-server-kafka-connect-clj",
|
16
|
+
"pid": "1158",
|
17
|
+
"uid": "0",
|
18
|
+
"gid": "0",
|
19
|
+
"comm": "dockerd",
|
20
|
+
"exe": "/usr/bin/dockerd",
|
21
|
+
"cmdline": "/usr/bin/dockerd --exec-opt native.cgroupdriver=cgroupfs --log-opt tag=docker.{{.ID}} --log-opt fluentd-address=unix:///var/run/td-agent/td-agent.sock --log-driver=fluentd --log-opt fluentd-async-connect=true --dns=10.42.8.190",
|
22
|
+
"systemd_cgroup": "/system.slice/docker.service",
|
23
|
+
"systemd_unit": "docker.service",
|
24
|
+
"message": "{\"timestamp\":\"2018-01-09T15:16:47.692+00:00\",\"logger\":\"kafka.connect.SinkTask\",\"level\":\"DEBUG\",\"thread\":\"pool-1-thread-30\",\"message\":\"put 0 records to sink\"}"
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"expected": {
|
28
|
+
"timestamp": 1515511007.692,
|
29
|
+
"_fluentd_tag": "journald.log",
|
30
|
+
"host": "mesosslave-private-0690bffd1e39d7462",
|
31
|
+
"_transport": "stdout",
|
32
|
+
"level": 7,
|
33
|
+
"facility": "daemon",
|
34
|
+
"_systemd_slice": "system.slice",
|
35
|
+
"_selinux_context": "system_u:system_r:container_runtime_t:s0",
|
36
|
+
"_boot_id": "a83a6f2a9aa545d18188387fcff0dea9",
|
37
|
+
"_machine_id": "f073c429a7456b53ec3e2c53460c5c8f",
|
38
|
+
"_cap_effective": "1fffffffff",
|
39
|
+
"_syslog_identifier": "kafka-connect-server-kafka-connect-clj",
|
40
|
+
"_pid": "1158",
|
41
|
+
"_uid": "0",
|
42
|
+
"_gid": "0",
|
43
|
+
"_comm": "dockerd",
|
44
|
+
"_exe": "/usr/bin/dockerd",
|
45
|
+
"_cmdline": "/usr/bin/dockerd --exec-opt native.cgroupdriver=cgroupfs --log-opt tag=docker.{{.ID}} --log-opt fluentd-address=unix:///var/run/td-agent/td-agent.sock --log-driver=fluentd --log-opt fluentd-async-connect=true --dns=10.42.8.190",
|
46
|
+
"_systemd_cgroup": "/system.slice/docker.service",
|
47
|
+
"_systemd_unit": "docker.service",
|
48
|
+
"_logger": "kafka.connect.SinkTask",
|
49
|
+
"_thread": "pool-1-thread-30",
|
50
|
+
"short_message": "put 0 records to sink"
|
51
|
+
}
|
52
|
+
}
|
@@ -0,0 +1,53 @@
|
|
1
|
+
{
|
2
|
+
"event": {
|
3
|
+
"tag": "journald.log",
|
4
|
+
"time": "2017-12-03 21:10:08.558Z UTC",
|
5
|
+
"record": {
|
6
|
+
"uid": "0",
|
7
|
+
"gid": "0",
|
8
|
+
"cap_effective": "1fffffffff",
|
9
|
+
"systemd_slice": "system.slice",
|
10
|
+
"boot_id": "652998235194436283ff15d225e268cf",
|
11
|
+
"machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
12
|
+
"hostname": "mesosslave-private-015e1a3477fa81e5e",
|
13
|
+
"transport": "syslog",
|
14
|
+
"priority": "3",
|
15
|
+
"syslog_facility": "1",
|
16
|
+
"syslog_identifier": "mesos-slave",
|
17
|
+
"syslog_pid": "3445",
|
18
|
+
"pid": "3596",
|
19
|
+
"comm": "logger",
|
20
|
+
"exe": "/usr/bin/logger",
|
21
|
+
"cmdline": "logger -p user.err -t mesos-slave[3445]",
|
22
|
+
"systemd_cgroup": "/system.slice/mesos-slave.service",
|
23
|
+
"systemd_unit": "mesos-slave.service",
|
24
|
+
"selinux_context": "system_u:system_r:unconfined_service_t:s0",
|
25
|
+
"message": "I1204 10:25:20.448935 3677 slave.cpp:5920] Current disk usage 5.31%. Max allowed age: 4.234456934750000mins",
|
26
|
+
"source_realtime_timestamp": "1512383120449042"
|
27
|
+
}
|
28
|
+
},
|
29
|
+
"expected": {
|
30
|
+
"timestamp": 1512383120.449,
|
31
|
+
"_fluentd_tag": "journald.log",
|
32
|
+
"_uid": "0",
|
33
|
+
"_gid": "0",
|
34
|
+
"_cap_effective": "1fffffffff",
|
35
|
+
"_systemd_slice": "system.slice",
|
36
|
+
"_boot_id": "652998235194436283ff15d225e268cf",
|
37
|
+
"_machine_id": "d1f142097d497f24c021d7de9b81cab4",
|
38
|
+
"host": "mesosslave-private-015e1a3477fa81e5e",
|
39
|
+
"_transport": "syslog",
|
40
|
+
"level": 3,
|
41
|
+
"facility": "user",
|
42
|
+
"_syslog_identifier": "mesos-slave",
|
43
|
+
"_syslog_pid": "3445",
|
44
|
+
"_pid": "3596",
|
45
|
+
"_comm": "logger",
|
46
|
+
"_exe": "/usr/bin/logger",
|
47
|
+
"_cmdline": "logger -p user.err -t mesos-slave[3445]",
|
48
|
+
"_systemd_cgroup": "/system.slice/mesos-slave.service",
|
49
|
+
"_systemd_unit": "mesos-slave.service",
|
50
|
+
"_selinux_context": "system_u:system_r:unconfined_service_t:s0",
|
51
|
+
"short_message": "I1204 10:25:20.448935 3677 slave.cpp:5920] Current disk usage 5.31%. Max allowed age: 4.234456934750000mins"
|
52
|
+
}
|
53
|
+
}
|
data/test/helper.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
4
|
+
require "test-unit"
|
5
|
+
require "fluent/test"
|
6
|
+
require "fluent/test/driver/output"
|
7
|
+
require "fluent/test/driver/formatter"
|
8
|
+
require "fluent/test/helpers"
|
9
|
+
|
10
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
11
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
12
|
+
|
13
|
+
def fixtures
|
14
|
+
path = File.expand_path("../fixtures", __FILE__)
|
15
|
+
all_fixture_files = Dir["#{path}/*.json"]
|
16
|
+
all_fixtures = {}
|
17
|
+
all_fixture_files.each do |file|
|
18
|
+
all_fixtures[File.basename(file)] = JSON.parse(File.read(file))
|
19
|
+
end
|
20
|
+
all_fixtures
|
21
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
require "fluent/plugin/formatter_gelf.rb"
|
5
|
+
|
6
|
+
class GelfFormatterTest < Test::Unit::TestCase
|
7
|
+
setup do
|
8
|
+
Fluent::Test.setup
|
9
|
+
@d = create_driver("")
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def create_driver(conf)
|
15
|
+
Fluent::Test::Driver::Formatter.new(Fluent::Plugin::GelfFormatter).configure(conf)
|
16
|
+
end
|
17
|
+
|
18
|
+
fixtures.each_pair do |file, v|
|
19
|
+
test "format #{file}" do
|
20
|
+
time = v["event"]["time"]
|
21
|
+
parsed_time = time.is_a?(Integer) ? time : event_time(time)
|
22
|
+
formatted = @d.instance.format(v["event"]["tag"], parsed_time, v["event"]["record"])
|
23
|
+
assert_equal v["expected"].to_json, formatted
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
require "fluent/plugin/out_gelf.rb"
|
5
|
+
require "test/unit/rr"
|
6
|
+
|
7
|
+
class GelfOutputTest < Test::Unit::TestCase
|
8
|
+
CONFIG = %(
|
9
|
+
host localhost
|
10
|
+
)
|
11
|
+
|
12
|
+
MULTI_RECORD_WRITE = "{\"timestamp\":1512335408.558,\"_fluentd_tag\":\"test\",\
|
13
|
+
\"_foo\":\"bar\",\"short_message\":\"(no message)\",\"level\":6}\u0000{\"timestamp\":1512335409.123,\
|
14
|
+
\"_fluentd_tag\":\"test\",\"_bar\":\"baz\",\"short_message\":\"(no message)\",\"level\":6}\u0000"
|
15
|
+
|
16
|
+
setup do
|
17
|
+
Fluent::Test.setup
|
18
|
+
@d = create_driver(CONFIG)
|
19
|
+
@stubbed_tcp = Object.new
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def create_driver(conf)
|
25
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::GelfOutput).configure(conf)
|
26
|
+
end
|
27
|
+
|
28
|
+
test "test config" do
|
29
|
+
assert_equal "localhost", @d.instance.host
|
30
|
+
assert_equal 12_201, @d.instance.port
|
31
|
+
end
|
32
|
+
|
33
|
+
test "write of multiple records chunk" do
|
34
|
+
mock(TCPSocket).new(@d.instance.host, @d.instance.port) { @stubbed_tcp }
|
35
|
+
mock(@stubbed_tcp).write(MULTI_RECORD_WRITE)
|
36
|
+
mock(@stubbed_tcp).close
|
37
|
+
@d.run(default_tag: "test") do
|
38
|
+
@d.feed(event_time("2017-12-03 21:10:08.558Z UTC"), "foo": "bar")
|
39
|
+
@d.feed(event_time("2017-12-03 21:10:09.123Z UTC"), "bar": "baz")
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
fixtures.each_pair do |file, v|
|
44
|
+
test "format #{file}" do
|
45
|
+
stub(TCPSocket).new { @stubbed_tcp }
|
46
|
+
stub(@stubbed_tcp).write
|
47
|
+
stub(@stubbed_tcp).close
|
48
|
+
@d.run(default_tag: v["event"]["tag"]) do
|
49
|
+
time = v["event"]["time"]
|
50
|
+
parsed_time = time.is_a?(Integer) ? time : event_time(time)
|
51
|
+
@d.feed(parsed_time, v["event"]["record"])
|
52
|
+
end
|
53
|
+
assert_equal [v["expected"].to_msgpack], @d.formatted
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
metadata
CHANGED
@@ -1,43 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gelf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
- Funding Circle
|
7
|
+
- Funding Circle
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: oj
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 3.3.10
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 3.3.10
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 2.1.4
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 2.1.4
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 12.3.3
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 12.3.3
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: test-unit
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -100,13 +114,39 @@ dependencies:
|
|
100
114
|
- - "<"
|
101
115
|
- !ruby/object:Gem::Version
|
102
116
|
version: '2'
|
103
|
-
description:
|
117
|
+
description: Converts fluentd log events into GELF format and sends them to Graylog
|
104
118
|
email:
|
105
119
|
- engineering+fluent-plugin-gelf@fundingcircle.com
|
106
120
|
executables: []
|
107
121
|
extensions: []
|
108
122
|
extra_rdoc_files: []
|
109
|
-
files:
|
123
|
+
files:
|
124
|
+
- ".circleci/config.yml"
|
125
|
+
- ".gitignore"
|
126
|
+
- ".rubocop.yml"
|
127
|
+
- ".ruby-version"
|
128
|
+
- Gemfile
|
129
|
+
- LICENSE
|
130
|
+
- README.md
|
131
|
+
- Rakefile
|
132
|
+
- fluent-plugin-gelf.gemspec
|
133
|
+
- lib/fluent/plugin/formatter_gelf.rb
|
134
|
+
- lib/fluent/plugin/gelf_plugin_util.rb
|
135
|
+
- lib/fluent/plugin/out_gelf.rb
|
136
|
+
- test/fixtures/docker_futureproof.json
|
137
|
+
- test/fixtures/docker_gelf_formatted.json
|
138
|
+
- test/fixtures/docker_gelf_formatted_already_extracted.json
|
139
|
+
- test/fixtures/docker_inner_json.json
|
140
|
+
- test/fixtures/docker_inner_json_with_time.json
|
141
|
+
- test/fixtures/docker_unformatted.json
|
142
|
+
- test/fixtures/level_from_tag.json
|
143
|
+
- test/fixtures/message_json_parsable_to_nil.json
|
144
|
+
- test/fixtures/systemd.json
|
145
|
+
- test/fixtures/systemd_inner_json.json
|
146
|
+
- test/fixtures/systemd_with_real_timestamp.json
|
147
|
+
- test/helper.rb
|
148
|
+
- test/plugin/test_formatter_gelf.rb
|
149
|
+
- test/plugin/test_out_gelf.rb
|
110
150
|
homepage:
|
111
151
|
licenses:
|
112
152
|
- BSD-3-Clause
|
@@ -122,13 +162,26 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
122
162
|
version: '0'
|
123
163
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
164
|
requirements:
|
125
|
-
- - "
|
165
|
+
- - ">="
|
126
166
|
- !ruby/object:Gem::Version
|
127
|
-
version:
|
167
|
+
version: '0'
|
128
168
|
requirements: []
|
129
|
-
|
130
|
-
rubygems_version: 2.6.14
|
169
|
+
rubygems_version: 3.0.3
|
131
170
|
signing_key:
|
132
171
|
specification_version: 4
|
133
|
-
summary:
|
134
|
-
test_files:
|
172
|
+
summary: Graylog output plugin for fluentd
|
173
|
+
test_files:
|
174
|
+
- test/fixtures/docker_futureproof.json
|
175
|
+
- test/fixtures/docker_gelf_formatted.json
|
176
|
+
- test/fixtures/docker_gelf_formatted_already_extracted.json
|
177
|
+
- test/fixtures/docker_inner_json.json
|
178
|
+
- test/fixtures/docker_inner_json_with_time.json
|
179
|
+
- test/fixtures/docker_unformatted.json
|
180
|
+
- test/fixtures/level_from_tag.json
|
181
|
+
- test/fixtures/message_json_parsable_to_nil.json
|
182
|
+
- test/fixtures/systemd.json
|
183
|
+
- test/fixtures/systemd_inner_json.json
|
184
|
+
- test/fixtures/systemd_with_real_timestamp.json
|
185
|
+
- test/helper.rb
|
186
|
+
- test/plugin/test_formatter_gelf.rb
|
187
|
+
- test/plugin/test_out_gelf.rb
|