fluent-plugin-newrelic 1.1.8 → 1.1.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/examples/Windows-IIS.conf +85 -0
- data/examples/copy_output.conf +43 -0
- data/examples/custom_field.conf +32 -0
- data/examples/file_input.conf +29 -0
- data/examples/filter_logs.conf +52 -0
- data/examples/grok_parser.conf +43 -0
- data/examples/minimal_complete_config.conf +27 -0
- data/examples/multiline_log_parse.conf +11 -0
- data/examples/parse_nginx.conf +17 -0
- data/examples/readme.md +2 -2
- data/examples/syslog_input.conf +5 -0
- data/lib/fluent/plugin/out_newrelic.rb +59 -24
- data/lib/newrelic-fluentd-output/version.rb +1 -1
- metadata +12 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7982f3b3dc21e8b82c8016c6f7ee6d2d4dad24128d754ad4c03a41a7e99f97d0
|
4
|
+
data.tar.gz: 3b2f2b9fe1e80f538a11c0171db58f7268eeaef01f35abd4130db6168946797d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4376c990dad117a8a4f64c1a469aae6018352584cbb375c7a88fea2d2f78c0474202fceea91d68d8be8a5e53e16d025f2dc0fdcae518f15ae4eb56080bc01de9
|
7
|
+
data.tar.gz: 91dec110d12f79f9104c5485f956477198cddad9fda0d0fddaae59800f895f4323d9fde1b4e6ac3f32fa4a9e50a5fdfdd67e08a4fc3cd7f132f54a8dc36730b2
|
@@ -0,0 +1,85 @@
|
|
1
|
+
####
|
2
|
+
## New Relic Logs - Basic Windows Event + IIS config v1.5
|
3
|
+
## Built in FluentD v1.7.4
|
4
|
+
## by AI of NR 2/4/2020
|
5
|
+
##
|
6
|
+
## Don't forget to add your NR license key to the <match> statement at the bottom of this file.
|
7
|
+
##
|
8
|
+
|
9
|
+
## Windows Event Log Input
|
10
|
+
<source>
|
11
|
+
@type windows_eventlog
|
12
|
+
@id windows_eventlog
|
13
|
+
tag winevt.raw
|
14
|
+
channels application,system,security
|
15
|
+
<storage>
|
16
|
+
@type local
|
17
|
+
persistent true
|
18
|
+
path c:/opt/td-agent/winevt.pos
|
19
|
+
</storage>
|
20
|
+
</source>
|
21
|
+
|
22
|
+
#
|
23
|
+
# Windows IIS log parsing. This config uses the standard W3C format and the standard location for IIS logs.
|
24
|
+
# It expects the log timestamp to be UTC.
|
25
|
+
# Change the path below if you store your logs elsewhere.
|
26
|
+
#
|
27
|
+
<source>
|
28
|
+
@type tail
|
29
|
+
tag iislog.raw
|
30
|
+
path c:/inetpub/logs/logfiles/*/*
|
31
|
+
pos_file c:/opt/td-agent/iislog.pos
|
32
|
+
<parse>
|
33
|
+
@type regexp
|
34
|
+
expression /(?<time>\d{4}-\d{2}-\d{2} [\d:]+) (?<message>.+)/
|
35
|
+
time_format %Y-%m-%d %H:%M:%S
|
36
|
+
utc true
|
37
|
+
</parse>
|
38
|
+
</source>
|
39
|
+
|
40
|
+
<filter iislog.raw>
|
41
|
+
@type parser
|
42
|
+
key_name message
|
43
|
+
remove_key_name_field false
|
44
|
+
reserve_data true
|
45
|
+
reserve_time true
|
46
|
+
<parse>
|
47
|
+
@type csv
|
48
|
+
delimiter ' '
|
49
|
+
keys hostname,req_method,req_uri,cs-uri-query,s_port,cs-username,c_ip,req_ua,req_referer,http_code,sc-substatus,sc-win32-status,time-taken
|
50
|
+
null_value_pattern -
|
51
|
+
null_empty_string true
|
52
|
+
</parse>
|
53
|
+
</filter>
|
54
|
+
|
55
|
+
#
|
56
|
+
# For a slightly nicer experience, add Service Name (s-sitename) to your log output, comment out the filter above and use this one instead.
|
57
|
+
#
|
58
|
+
#<filter iislog.raw>
|
59
|
+
# @type parser
|
60
|
+
# key_name message
|
61
|
+
# remove_key_name_field false
|
62
|
+
# reserve_data true
|
63
|
+
# reserve_time true
|
64
|
+
# <parse>
|
65
|
+
# @type csv
|
66
|
+
# delimiter ' '
|
67
|
+
# keys service_name,hostname,req_method,req_uri,cs-uri-query,s_port,cs-username,c_ip,req_ua,req_referer,http_code,sc-substatus,sc-win32-status,time-taken
|
68
|
+
# null_value_pattern -
|
69
|
+
# null_empty_string true
|
70
|
+
# </parse>
|
71
|
+
#</filter>
|
72
|
+
|
73
|
+
<filter winevt.raw>
|
74
|
+
@type record_transformer
|
75
|
+
<record>
|
76
|
+
message ${record["description"]}
|
77
|
+
hostname ${record["computer_name"]}
|
78
|
+
</record>
|
79
|
+
</filter>
|
80
|
+
|
81
|
+
# New Relic output
|
82
|
+
<match **>
|
83
|
+
@type newrelic
|
84
|
+
api_key <YOUR INSERT KEY>
|
85
|
+
</match>
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#Tail and parse Apache access logs
|
2
|
+
|
3
|
+
<source>
|
4
|
+
@type tail
|
5
|
+
@id input_tail_apache
|
6
|
+
tag apache_access
|
7
|
+
path /var/log/apache2/access.log
|
8
|
+
pos_file /var/log/apache2/access.pos
|
9
|
+
path_key filename
|
10
|
+
<parse>
|
11
|
+
@type apache2
|
12
|
+
</parse>
|
13
|
+
</source>
|
14
|
+
|
15
|
+
#Add hostname and tag fields to all events ("records") with a Fluentd tag of apache_access
|
16
|
+
|
17
|
+
<filter apache_access>
|
18
|
+
@type record_transformer
|
19
|
+
<record>
|
20
|
+
hostname "#{Socket.gethostname}"
|
21
|
+
tag ${tag}
|
22
|
+
</record>
|
23
|
+
</filter>
|
24
|
+
|
25
|
+
#Output (https://docs.fluentd.org/output/copy) events to both New Relic and a local file.
|
26
|
+
|
27
|
+
<match **>
|
28
|
+
@type copy
|
29
|
+
<store>
|
30
|
+
@type newrelic
|
31
|
+
api_key <YOUR INSERT KEY>
|
32
|
+
</store>
|
33
|
+
<store>
|
34
|
+
@type file
|
35
|
+
path /var/log/apacheout.log
|
36
|
+
<buffer>
|
37
|
+
#Buffer settings are for testing and not recommended for use in a production environment.
|
38
|
+
timekey 10s
|
39
|
+
timekey_use_utc true
|
40
|
+
timekey_wait 15s
|
41
|
+
</buffer>
|
42
|
+
</store>
|
43
|
+
</match>
|
@@ -0,0 +1,32 @@
|
|
1
|
+
#Tail and parse Docker log files
|
2
|
+
|
3
|
+
<source>
|
4
|
+
@type tail
|
5
|
+
path /var/lib/docker/containers/*/*-json.log
|
6
|
+
pos_file /var/log/docker-log.pos
|
7
|
+
read_from_head true
|
8
|
+
tag containers
|
9
|
+
<parse>
|
10
|
+
@type json
|
11
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
12
|
+
keep_time_key true
|
13
|
+
time_key time
|
14
|
+
</parse>
|
15
|
+
</source>
|
16
|
+
|
17
|
+
<filter containers>
|
18
|
+
@type record_transformer
|
19
|
+
enable_ruby true
|
20
|
+
<record>
|
21
|
+
#Add hostname and tag fields to all records
|
22
|
+
fluentd_host "#{Socket.gethostname}"
|
23
|
+
tag ${tag}
|
24
|
+
</record>
|
25
|
+
</filter>
|
26
|
+
|
27
|
+
# Forward events to New Relic
|
28
|
+
|
29
|
+
<match containers>
|
30
|
+
@type newrelic
|
31
|
+
api_key <YOUR INSERT KEY>
|
32
|
+
</match>
|
@@ -0,0 +1,29 @@
|
|
1
|
+
#Tail arbitrary text/log file
|
2
|
+
|
3
|
+
<source>
|
4
|
+
@type tail
|
5
|
+
<parse>
|
6
|
+
@type none
|
7
|
+
</parse>
|
8
|
+
path /var/log/backend-app*.log
|
9
|
+
pos_file /var/log/backend.application.pos
|
10
|
+
path_key filename # Add watched file path to path_key field for every event/record.
|
11
|
+
tag backend.application
|
12
|
+
</source>
|
13
|
+
|
14
|
+
#Add hostname and tag fields to all events ("records") with a Fluentd tag of backend.application
|
15
|
+
|
16
|
+
<filter backend.application>
|
17
|
+
@type record_transformer
|
18
|
+
<record>
|
19
|
+
hostname "#{Socket.gethostname}"
|
20
|
+
tag ${tag}
|
21
|
+
</record>
|
22
|
+
</filter>
|
23
|
+
|
24
|
+
#Write events to New Relic
|
25
|
+
|
26
|
+
<match backend.application>
|
27
|
+
@type newrelic
|
28
|
+
api_key <YOUR INSERT KEY>
|
29
|
+
</match>
|
@@ -0,0 +1,52 @@
|
|
1
|
+
#Tail and parse arbitrary text/log file
|
2
|
+
|
3
|
+
<source>
|
4
|
+
@type tail
|
5
|
+
<parse> #Parse timestamp, everything else to be stored in message field
|
6
|
+
@type regexp
|
7
|
+
expression /^\[(?<logtime>[^\]]*)\] (?<message>.*)$/
|
8
|
+
time_key logtime
|
9
|
+
time_format %Y-%m-%d %H:%M:%S %z
|
10
|
+
</parse>
|
11
|
+
path /var/log/backend-app*.log
|
12
|
+
pos_file /var/log/backend.application.pos
|
13
|
+
path_key filename # Add watched file path to path_key field for every event/record.
|
14
|
+
tag backend.application
|
15
|
+
</source>
|
16
|
+
|
17
|
+
#Add hostname and service_name fields to all events ("records") with a Fluentd tag of backend.application
|
18
|
+
|
19
|
+
<filter backend.application>
|
20
|
+
@type record_transformer
|
21
|
+
<record>
|
22
|
+
hostname "#{Socket.gethostname}"
|
23
|
+
service_name ${tag}
|
24
|
+
</record>
|
25
|
+
</filter>
|
26
|
+
|
27
|
+
# For all events with a tag of backend.application:
|
28
|
+
# Keep ONLY events where service_name field contains a value matching /backend.application/ AND where message field contains a value matching /Cannot connect to/
|
29
|
+
# Discard any events where value of hostname field matches /staging/
|
30
|
+
|
31
|
+
<filter backend.application>
|
32
|
+
@type grep
|
33
|
+
<regexp>
|
34
|
+
key service_name
|
35
|
+
pattern /backend.application/
|
36
|
+
</regexp>
|
37
|
+
<regexp>
|
38
|
+
key message
|
39
|
+
pattern /Cannot connect to/
|
40
|
+
</regexp>
|
41
|
+
<exclude>
|
42
|
+
key hostname
|
43
|
+
pattern /staging/
|
44
|
+
</exclude>
|
45
|
+
</filter>
|
46
|
+
|
47
|
+
#Write events with backend.application tag to New Relic
|
48
|
+
|
49
|
+
<match backend.application>
|
50
|
+
@type newrelic
|
51
|
+
api_key <YOUR INSERT KEY>
|
52
|
+
</match>
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#Tail arbitrary log file and parse using grok pattern
|
2
|
+
#Install the required plugin: fluent-gem install fluent-plugin-grok-parser
|
3
|
+
|
4
|
+
<source>
|
5
|
+
@type tail
|
6
|
+
<parse>
|
7
|
+
@type grok
|
8
|
+
<grok>
|
9
|
+
pattern %{SYSLOGTIMESTAMP:timestamp} %{LOGLEVEL:loglevel}: %{GREEDYDATA:message}
|
10
|
+
</grok>
|
11
|
+
</parse>
|
12
|
+
path /var/log/customapp.log
|
13
|
+
pos_file /var/log/customapp.pos
|
14
|
+
path_key filename
|
15
|
+
tag custom.application
|
16
|
+
</source>
|
17
|
+
|
18
|
+
# Drop events with custom.application tag where loglevel field contains "debug" or "info" (case-insensitive match)
|
19
|
+
|
20
|
+
<filter custom.application>
|
21
|
+
@type grep
|
22
|
+
<exclude>
|
23
|
+
key loglevel
|
24
|
+
pattern /debug|info/i
|
25
|
+
</exclude>
|
26
|
+
</filter>
|
27
|
+
|
28
|
+
#Add hostname and tag fields to all events ("records") with a Fluentd tag of custom.application
|
29
|
+
|
30
|
+
<filter custom.application>
|
31
|
+
@type record_transformer
|
32
|
+
<record>
|
33
|
+
hostname "#{Socket.gethostname}"
|
34
|
+
tag ${tag}
|
35
|
+
</record>
|
36
|
+
</filter>
|
37
|
+
|
38
|
+
#Write custom.application events to New Relic
|
39
|
+
|
40
|
+
<match custom.application>
|
41
|
+
@type newrelic
|
42
|
+
api_key <YOUR INSERT KEY>
|
43
|
+
</match>
|
@@ -0,0 +1,27 @@
|
|
1
|
+
#Tail arbitrary text/log file
|
2
|
+
|
3
|
+
<source>
|
4
|
+
@type tail
|
5
|
+
<parse>
|
6
|
+
@type none
|
7
|
+
</parse>
|
8
|
+
path /home/logs/*
|
9
|
+
path_key file
|
10
|
+
tag sample.tag
|
11
|
+
</source>
|
12
|
+
|
13
|
+
#Add service_name field to all events ("records") with a Fluentd tag of sample.tag
|
14
|
+
|
15
|
+
<filter sample.tag>
|
16
|
+
@type record_transformer
|
17
|
+
<record>
|
18
|
+
service_name ${tag}
|
19
|
+
</record>
|
20
|
+
</filter>
|
21
|
+
|
22
|
+
#Write sample.tag events to New Relic
|
23
|
+
|
24
|
+
<match sample.tag>
|
25
|
+
@type newrelic
|
26
|
+
api_key <YOUR INSERT KEY>
|
27
|
+
</match>
|
@@ -0,0 +1,17 @@
|
|
1
|
+
#Tail and parse NGINX log file
|
2
|
+
|
3
|
+
<source>
|
4
|
+
@type tail
|
5
|
+
<parse>
|
6
|
+
@type nginx
|
7
|
+
</parse>
|
8
|
+
path /path/to/access.log
|
9
|
+
tag nginx.access
|
10
|
+
</source>
|
11
|
+
|
12
|
+
#Write events with tag matching nginx.* to New Relic
|
13
|
+
|
14
|
+
<match nginx.*>
|
15
|
+
@type newrelic
|
16
|
+
api_key <YOUR INSERT KEY>
|
17
|
+
</match>
|
data/examples/readme.md
CHANGED
@@ -61,9 +61,9 @@ sudo mkdir /etc/fluentd
|
|
61
61
|
sudo nano /etc/fluentd/fluentd.conf
|
62
62
|
```
|
63
63
|
|
64
|
-
#### 2. Add the contents from the [`syslog
|
64
|
+
#### 2. Add the contents from the [`syslog/fluentd.conf`](syslog/fluentd.config)
|
65
65
|
|
66
|
-
You can find the contents of the [`syslog
|
66
|
+
You can find the contents of the [`syslog/fluentd.conf`](syslog/fluentd.config) in the sub folder `syslog`. These contents should provide a quick start to getting started. In the provided
|
67
67
|
example, the syslog details are coming from the above mentioned devices. You may need to tweak the configuration according to the server sending the syslog traffic.
|
68
68
|
|
69
69
|
#### 3. Check New Relic for New Logs
|
@@ -33,6 +33,7 @@ module Fluent
|
|
33
33
|
config_param :license_key, :string, :default => nil
|
34
34
|
|
35
35
|
DEFAULT_BUFFER_TYPE = 'memory'.freeze
|
36
|
+
MAX_PAYLOAD_SIZE = 1048576 # 1 Megabyte in bytes
|
36
37
|
|
37
38
|
config_section :buffer do
|
38
39
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -50,7 +51,7 @@ module Fluent
|
|
50
51
|
def configure(conf)
|
51
52
|
super
|
52
53
|
if @api_key.nil? && @license_key.nil?
|
53
|
-
raise Fluent::ConfigError.new("'api_key' or 'license_key' parameter is required")
|
54
|
+
raise Fluent::ConfigError.new("'api_key' or 'license_key' parameter is required")
|
54
55
|
end
|
55
56
|
|
56
57
|
# create initial sockets hash and socket based on config param
|
@@ -89,39 +90,23 @@ module Fluent
|
|
89
90
|
packaged['message'] = record['log']
|
90
91
|
packaged['attributes'].delete('log')
|
91
92
|
end
|
92
|
-
|
93
|
+
|
93
94
|
packaged
|
94
95
|
end
|
95
96
|
|
96
97
|
def write(chunk)
|
97
|
-
|
98
|
-
'common' => {
|
99
|
-
'attributes' => {
|
100
|
-
'plugin' => {
|
101
|
-
'type' => 'fluentd',
|
102
|
-
'version' => NewrelicFluentdOutput::VERSION,
|
103
|
-
}
|
104
|
-
}
|
105
|
-
},
|
106
|
-
'logs' => []
|
107
|
-
}
|
98
|
+
logs = []
|
108
99
|
chunk.msgpack_each do |ts, record|
|
109
100
|
next unless record.is_a? Hash
|
110
101
|
next if record.empty?
|
111
|
-
|
102
|
+
logs.push(package_record(record, ts))
|
112
103
|
end
|
113
|
-
io = StringIO.new
|
114
|
-
gzip = Zlib::GzipWriter.new(io)
|
115
104
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
# See https://github.com/fluent/fluentd/issues/215
|
120
|
-
gzip << Yajl.dump([payload])
|
121
|
-
gzip.close
|
122
|
-
send_payload(io.string)
|
105
|
+
|
106
|
+
payloads = get_compressed_payloads(logs)
|
107
|
+
payloads.each { |payload| send_payload(payload) }
|
123
108
|
end
|
124
|
-
|
109
|
+
|
125
110
|
def handle_response(response)
|
126
111
|
if !(200 <= response.code.to_i && response.code.to_i < 300)
|
127
112
|
log.error("Response was " + response.code + " " + response.body)
|
@@ -137,6 +122,56 @@ module Fluent
|
|
137
122
|
handle_response(http.request(request))
|
138
123
|
end
|
139
124
|
|
125
|
+
private
|
126
|
+
|
127
|
+
def get_compressed_payloads(logs)
|
128
|
+
return [] if logs.length == 0
|
129
|
+
|
130
|
+
payload = create_payload(logs)
|
131
|
+
compressed_payload = compress(payload)
|
132
|
+
|
133
|
+
if compressed_payload.bytesize <= MAX_PAYLOAD_SIZE
|
134
|
+
return [compressed_payload]
|
135
|
+
end
|
136
|
+
|
137
|
+
if logs.length > 1 # we can split
|
138
|
+
# let's split logs array by half, and try to create payloads again
|
139
|
+
midpoint = logs.length / 2
|
140
|
+
first_half = get_compressed_payloads(logs.slice(0, midpoint))
|
141
|
+
second_half = get_compressed_payloads(logs.slice(midpoint, logs.length))
|
142
|
+
return first_half + second_half
|
143
|
+
else
|
144
|
+
log.error("Can't compress record below required maximum packet size and it will be discarded. Record: #{logs[0]}")
|
145
|
+
return []
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def create_payload(logs)
|
150
|
+
{
|
151
|
+
'common' => {
|
152
|
+
'attributes' => {
|
153
|
+
'plugin' => {
|
154
|
+
'type' => 'fluentd',
|
155
|
+
'version' => NewrelicFluentdOutput::VERSION,
|
156
|
+
}
|
157
|
+
}
|
158
|
+
},
|
159
|
+
'logs' => logs
|
160
|
+
}
|
161
|
+
end
|
162
|
+
|
163
|
+
def compress(payload)
|
164
|
+
io = StringIO.new
|
165
|
+
gzip = Zlib::GzipWriter.new(io)
|
166
|
+
|
167
|
+
# Fluentd can run with a version of Ruby (2.1.0) whose to_json method doesn't support non-ASCII characters.
|
168
|
+
# So we use Yajl, which can handle all Unicode characters. Apparently this library is what Fluentd uses
|
169
|
+
# internally, so it is installed by default with td-agent.
|
170
|
+
# See https://github.com/fluent/fluentd/issues/215
|
171
|
+
gzip << Yajl.dump([payload])
|
172
|
+
gzip.close
|
173
|
+
io.string
|
174
|
+
end
|
140
175
|
end
|
141
176
|
end
|
142
177
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-newrelic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- New Relic Logging Team
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -110,8 +110,18 @@ files:
|
|
110
110
|
- README.md
|
111
111
|
- Rakefile
|
112
112
|
- examples/Dockerfile
|
113
|
+
- examples/Windows-IIS.conf
|
114
|
+
- examples/copy_output.conf
|
115
|
+
- examples/custom_field.conf
|
116
|
+
- examples/file_input.conf
|
117
|
+
- examples/filter_logs.conf
|
118
|
+
- examples/grok_parser.conf
|
119
|
+
- examples/minimal_complete_config.conf
|
120
|
+
- examples/multiline_log_parse.conf
|
121
|
+
- examples/parse_nginx.conf
|
113
122
|
- examples/readme.md
|
114
123
|
- examples/syslog/fluentd.config
|
124
|
+
- examples/syslog_input.conf
|
115
125
|
- lib/fluent/plugin/out_newrelic.rb
|
116
126
|
- lib/newrelic-fluentd-output/version.rb
|
117
127
|
- merge-to-master-pipeline.yml
|