fluent-plugin-vmware-loginsight 0.1.10 → 0.1.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +71 -18
- data/VERSION +1 -0
- data/examples/fluent.conf +104 -52
- data/examples/fluentd-vrli-plugin-debian.dockerfile +24 -12
- data/examples/k8s-log-collector-ds.yaml +108 -49
- data/fluent-plugin-vmware-loginsight.gemspec +1 -1
- data/lib/fluent/plugin/out_vmware_loginsight.rb +6 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 10d3d5209686fe48c79d1182e193316588f8e29bb6567d9ebe54b0eb061f0e55
|
|
4
|
+
data.tar.gz: 53b75b154b7ca8ff95c8aa71dd0fe91d00deff9460761ac4ea1c13da02c724e7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2f41fbe184204f8100ebca73447f87d1cdce2640941bdae56d34fff216cbd554a7c22ed1e71f5e39f23bcdb72d8b766897f91917959f29e7346e6deecb0a2f7e
|
|
7
|
+
data.tar.gz: 1ff567aaf08344dc494fdedb6ed2efc21084e55c66da28b33cdbd40a17331c02519e94226959c291e487787a57e12bdad2fc6cff386573fb483b92dd34430e47
|
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# fluent-plugin-vmware-loginsight
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight)
|
|
4
|
+
|
|
3
5
|
## Overview
|
|
4
6
|
output plugin to do forward logs to VMware Log Insight
|
|
5
7
|
|
|
@@ -28,39 +30,61 @@ $ bundle
|
|
|
28
30
|
## Usage
|
|
29
31
|
|
|
30
32
|
```
|
|
33
|
+
# Collect all container logs
|
|
31
34
|
<source>
|
|
32
35
|
@type tail
|
|
36
|
+
@id in_tail_container_logs
|
|
33
37
|
path /var/log/containers/*.log
|
|
38
|
+
# One could exclude certain logs like:
|
|
39
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
|
34
40
|
pos_file /var/log/fluentd-docker.pos
|
|
35
|
-
time_format %Y-%m-%dT%H:%M:%S
|
|
36
|
-
tag kubernetes.*
|
|
37
|
-
format json
|
|
38
41
|
read_from_head true
|
|
42
|
+
# Set this watcher to false if you have many files to tail
|
|
43
|
+
enable_stat_watcher false
|
|
44
|
+
refresh_interval 5
|
|
45
|
+
tag kubernetes.*
|
|
46
|
+
<parse>
|
|
47
|
+
@type json
|
|
48
|
+
time_key time
|
|
49
|
+
keep_time_key true
|
|
50
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
51
|
+
</parse>
|
|
39
52
|
</source>
|
|
40
53
|
|
|
41
|
-
# Kubernetes metadata filter that tags additional meta data for each event
|
|
42
|
-
<filter kubernetes
|
|
54
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
|
55
|
+
<filter kubernetes.**>
|
|
43
56
|
@type kubernetes_metadata
|
|
57
|
+
@id filter_kube_metadata
|
|
58
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
|
59
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
|
60
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
|
61
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
|
62
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
|
63
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
|
64
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
|
44
65
|
</filter>
|
|
45
66
|
|
|
67
|
+
# Match everything
|
|
46
68
|
<match **>
|
|
47
69
|
@type vmware_loginsight
|
|
70
|
+
@id out_vmw_li_all_container_logs
|
|
48
71
|
scheme https
|
|
49
72
|
ssl_verify true
|
|
50
|
-
# Loginsight host: One may use IP address or cname
|
|
51
|
-
#
|
|
52
|
-
host
|
|
53
|
-
port
|
|
54
|
-
path api/v1/events/ingest
|
|
73
|
+
# Loginsight host: One may use IP address or cname
|
|
74
|
+
#host X.X.X.X
|
|
75
|
+
host MY_LOGINSIGHT_HOST
|
|
76
|
+
port 9543
|
|
55
77
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
tag_key tag
|
|
78
|
+
# Keys from log event whose values should be added as log message/text to
|
|
79
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
80
|
+
log_text_keys ["log","msg","message"]
|
|
81
|
+
# Use this flag if you want to enable http debug logs
|
|
82
|
+
http_conn_debug false
|
|
62
83
|
</match>
|
|
63
84
|
```
|
|
85
|
+
|
|
86
|
+
For more examples look at [examples](./examples/)
|
|
87
|
+
|
|
64
88
|
### Configuration options
|
|
65
89
|
|
|
66
90
|
```
|
|
@@ -104,6 +128,9 @@ request_timeout, :time, :default => 5
|
|
|
104
128
|
# If set, enables debug logs for http connection
|
|
105
129
|
http_conn_debug, :bool, :default => false :: Valid Value: true | false
|
|
106
130
|
|
|
131
|
+
# Number of bytes per post request
|
|
132
|
+
max_batch_size, :integer, :default => 512000
|
|
133
|
+
|
|
107
134
|
# Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
|
|
108
135
|
rate_limit_msec, :integer, :default => 0
|
|
109
136
|
|
|
@@ -125,9 +152,35 @@ flatten_hashes, :bool, :default => true :: Valid Value: true | false
|
|
|
125
152
|
|
|
126
153
|
# Seperator to use for joining flattened keys
|
|
127
154
|
flatten_hashes_separator, :string, :default => "_"
|
|
128
|
-
```
|
|
129
155
|
|
|
130
|
-
|
|
156
|
+
# Rename fields names
|
|
157
|
+
config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
|
|
158
|
+
|
|
159
|
+
# Keys from log event to rewrite
|
|
160
|
+
# for instance from 'kubernetes_namespace' to 'k8s_namespace'
|
|
161
|
+
# tags will be rewritten with substring substitution
|
|
162
|
+
# and applied in the order present in the hash
|
|
163
|
+
# (Hashes enumerate their values in the order that the
|
|
164
|
+
# corresponding keys were inserted
|
|
165
|
+
# see https://ruby-doc.org/core-2.2.2/Hash.html)
|
|
166
|
+
# example config:
|
|
167
|
+
# shorten_keys {
|
|
168
|
+
# "__":"_",
|
|
169
|
+
# "container_":"",
|
|
170
|
+
# "kubernetes_":"k8s_",
|
|
171
|
+
# "labels_":"",
|
|
172
|
+
# }
|
|
173
|
+
shorten_keys, :hash, value_type: :string, default:
|
|
174
|
+
{
|
|
175
|
+
'kubernetes_':'k8s_',
|
|
176
|
+
'namespace':'ns',
|
|
177
|
+
'labels_':'',
|
|
178
|
+
'_name':'',
|
|
179
|
+
'_hash':'',
|
|
180
|
+
'container_':''
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
```
|
|
131
184
|
|
|
132
185
|
## Contributing
|
|
133
186
|
|
data/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.1.11
|
data/examples/fluent.conf
CHANGED
|
@@ -8,18 +8,25 @@
|
|
|
8
8
|
#
|
|
9
9
|
# SPDX-License-Identifier: MIT
|
|
10
10
|
|
|
11
|
+
# Sample Fluentd config, edit as per your needs.
|
|
12
|
+
# https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/templates/conf has some good fluentd config examples
|
|
11
13
|
|
|
14
|
+
# System level configs
|
|
12
15
|
<system>
|
|
13
16
|
log_level info
|
|
14
17
|
</system>
|
|
15
18
|
|
|
16
19
|
# Prevent fluentd from handling records containing its own logs to handle cycles.
|
|
17
|
-
<
|
|
18
|
-
|
|
19
|
-
|
|
20
|
+
<label @FLUENT_LOG>
|
|
21
|
+
<match fluent.**>
|
|
22
|
+
@type null
|
|
23
|
+
</match>
|
|
24
|
+
</label>
|
|
20
25
|
|
|
26
|
+
# Collect all journal logs
|
|
21
27
|
<source>
|
|
22
28
|
@type systemd
|
|
29
|
+
@id in_systemd_logs
|
|
23
30
|
path /run/log/journal
|
|
24
31
|
# Can filter logs if we want, e.g.
|
|
25
32
|
#filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
|
|
@@ -33,76 +40,121 @@
|
|
|
33
40
|
strip_underscores true
|
|
34
41
|
</source>
|
|
35
42
|
|
|
43
|
+
# Collect all container logs
|
|
36
44
|
<source>
|
|
37
45
|
@type tail
|
|
46
|
+
@id in_tail_container_logs
|
|
38
47
|
path /var/log/containers/*.log
|
|
39
48
|
# One could exclude certain logs like:
|
|
40
|
-
#
|
|
49
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
|
41
50
|
pos_file /var/log/fluentd-docker.pos
|
|
42
|
-
time_format %Y-%m-%dT%H:%M:%S
|
|
43
|
-
tag kubernetes.*
|
|
44
|
-
format json
|
|
45
51
|
read_from_head true
|
|
52
|
+
# Set this watcher to false if you have many files to tail
|
|
53
|
+
enable_stat_watcher false
|
|
54
|
+
refresh_interval 5
|
|
55
|
+
tag kubernetes.*
|
|
56
|
+
<parse>
|
|
57
|
+
@type json
|
|
58
|
+
time_key time
|
|
59
|
+
keep_time_key true
|
|
60
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
61
|
+
</parse>
|
|
46
62
|
</source>
|
|
47
63
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
#<source>
|
|
51
|
-
# @type tail
|
|
52
|
-
# path /var/log/containers/javaapp**.log
|
|
53
|
-
# pos_file /var/log/fluentd-dockerlog.pos
|
|
54
|
-
# time_format %b %d %H:%M:%S
|
|
55
|
-
# tag kubernetes.*
|
|
56
|
-
# format multiline
|
|
57
|
-
# format_firstline /\d{4}-\d{1,2}-\d{1,2}/
|
|
58
|
-
# format1 /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
|
|
59
|
-
# read_from_head true
|
|
60
|
-
#</source>
|
|
61
|
-
|
|
62
|
-
# Kubernetes metadata filter that tags additional meta data for each event
|
|
63
|
-
<filter kubernetes.var.log.containers.**.log>
|
|
64
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
|
65
|
+
<filter kubernetes.**>
|
|
64
66
|
@type kubernetes_metadata
|
|
67
|
+
@id filter_kube_metadata
|
|
68
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
|
69
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
|
70
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
|
71
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
|
72
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
|
73
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
|
74
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
|
65
75
|
</filter>
|
|
66
76
|
|
|
67
|
-
#
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
#</filter>
|
|
77
|
+
# Prefix the tag by namespace. This would make it easy to match logs by namespaces
|
|
78
|
+
<match kubernetes.**>
|
|
79
|
+
@type rewrite_tag_filter
|
|
80
|
+
<rule>
|
|
81
|
+
key $.kubernetes.namespace_name
|
|
82
|
+
pattern ^(.+)$
|
|
83
|
+
tag $1.${tag}
|
|
84
|
+
</rule>
|
|
85
|
+
</match>
|
|
77
86
|
|
|
78
|
-
|
|
79
|
-
|
|
87
|
+
# Collect all kube apiserver audit logs
|
|
88
|
+
<source>
|
|
89
|
+
@type tail
|
|
90
|
+
@id in_tail_kube_audit_logs
|
|
91
|
+
# audit log path of kube-apiserver
|
|
92
|
+
path "/var/log/kube-audit/audit.log"
|
|
93
|
+
pos_file /var/log/kube-audit.pos
|
|
94
|
+
tag kube-audit
|
|
95
|
+
<parse>
|
|
96
|
+
@type json
|
|
97
|
+
time_key timestamp
|
|
98
|
+
keep_time_key false
|
|
99
|
+
time_format %Y-%m-%dT%H:%M:%SZ
|
|
100
|
+
</parse>
|
|
101
|
+
</source>
|
|
102
|
+
|
|
103
|
+
# Loginsight doesn't support ingesting `source` as a field name, get rid of it
|
|
104
|
+
<filter kube-audit>
|
|
105
|
+
@type record_transformer
|
|
106
|
+
@id filter_kube_audit_logs
|
|
107
|
+
enable_ruby
|
|
108
|
+
remove_keys source
|
|
109
|
+
<record>
|
|
110
|
+
log ${record}
|
|
111
|
+
</record>
|
|
112
|
+
</filter>
|
|
113
|
+
|
|
114
|
+
# You can catch and match logs by namespace
|
|
115
|
+
<match my-namespace-one.** my-namespace-two.**>
|
|
116
|
+
@type vmware_loginsight
|
|
117
|
+
@id out_vmw_li_my_namespace_logs
|
|
118
|
+
scheme http
|
|
119
|
+
ssl_verify false
|
|
120
|
+
# Loginsight host: One may use IP address or cname
|
|
121
|
+
#host X.X.X.X
|
|
122
|
+
host MY_LOGINSIGHT_HOST
|
|
123
|
+
port 9000
|
|
124
|
+
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
125
|
+
# Keys from log event whose values should be added as log message/text to
|
|
126
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
127
|
+
log_text_keys ["log","msg","message"]
|
|
128
|
+
# Use this flag if you want to enable http debug logs
|
|
129
|
+
http_conn_debug false
|
|
80
130
|
</match>
|
|
81
131
|
|
|
132
|
+
# Match everything else
|
|
82
133
|
<match **>
|
|
83
134
|
@type copy
|
|
84
135
|
<store>
|
|
85
136
|
@type vmware_loginsight
|
|
137
|
+
@id out_vmw_li_all_container_logs
|
|
86
138
|
scheme https
|
|
87
139
|
ssl_verify true
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
host
|
|
91
|
-
port
|
|
92
|
-
path api/v1/events/ingest
|
|
140
|
+
# Loginsight host: One may use IP address or cname
|
|
141
|
+
#host X.X.X.X
|
|
142
|
+
host MY_LOGINSIGHT_HOST
|
|
143
|
+
port 9543
|
|
93
144
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
rate_limit_msec 0
|
|
97
|
-
raise_on_error false
|
|
145
|
+
# Keys from log event whose values should be added as log message/text to
|
|
146
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
98
147
|
log_text_keys ["log","msg","message"]
|
|
99
|
-
|
|
100
|
-
|
|
148
|
+
# Use this flag if you want to enable http debug logs
|
|
149
|
+
http_conn_debug false
|
|
101
150
|
</store>
|
|
102
|
-
# copy plugin supports sending/copying logs to multiple plugins
|
|
103
|
-
# One may choose to send them to multiple LIs
|
|
104
|
-
# Or one may want send a copy to stdout for debugging
|
|
105
|
-
#
|
|
106
|
-
#
|
|
107
|
-
|
|
151
|
+
# copy plugin supports sending/copying logs to multiple plugins
|
|
152
|
+
# One may choose to send them to multiple LIs
|
|
153
|
+
# Or one may want send a copy to stdout for debugging
|
|
154
|
+
# Please note, if you use stdout along with LI, catch the logger's log to make
|
|
155
|
+
# sure they're not cyclic
|
|
156
|
+
#<store>
|
|
157
|
+
# @type stdout
|
|
158
|
+
#</store>
|
|
108
159
|
</match>
|
|
160
|
+
|
|
@@ -8,29 +8,41 @@
|
|
|
8
8
|
#
|
|
9
9
|
# SPDX-License-Identifier: MIT
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
#
|
|
11
|
+
|
|
12
|
+
# Sample Dockerfile to use as log collector
|
|
13
|
+
# Builds a debian-based fluentd image that has fluent-plugin-kubernetes_metadata_filter,
|
|
14
|
+
# fluent-plugin-rewrite-tag-filter, fluent-plugin-systemd and
|
|
15
|
+
# fluent-plugin-vmware-loginsight gem installed.
|
|
13
16
|
#
|
|
14
|
-
#
|
|
17
|
+
# This image will get preconfigured with the fluent.conf if avaialble at the
|
|
18
|
+
# same dir level. For fluentd config example, see
|
|
15
19
|
# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
|
|
16
|
-
FROM fluent/fluentd:v0.14.15-debian-onbuild
|
|
17
|
-
# Above image expects the loginsight plugin vmware_loginsight to be available under ./plugins/vmware_loginsight.rb
|
|
18
|
-
# and fluentd config under ./fluent.conf by default
|
|
19
20
|
|
|
21
|
+
# This base image is built from https://github.com/fluent/fluentd-kubernetes-daemonset
|
|
22
|
+
FROM fluent/fluentd:v1.11-debian-1
|
|
23
|
+
|
|
24
|
+
# Use root account to use apt
|
|
20
25
|
USER root
|
|
21
26
|
|
|
22
|
-
|
|
27
|
+
# You can install your plugins here
|
|
28
|
+
RUN buildDeps="sudo make gcc g++ libc-dev" \
|
|
23
29
|
&& apt-get update \
|
|
24
30
|
&& apt-get install -y --no-install-recommends $buildDeps \
|
|
25
31
|
&& sudo gem install \
|
|
26
|
-
fluent-plugin-
|
|
27
|
-
fluent-plugin-
|
|
28
|
-
fluent-plugin-
|
|
32
|
+
fluent-plugin-kubernetes_metadata_filter:2.4.6 \
|
|
33
|
+
fluent-plugin-rewrite-tag-filter:2.3.0 \
|
|
34
|
+
fluent-plugin-systemd:1.0.2 \
|
|
35
|
+
fluent-plugin-vmware-loginsight:0.1.10 \
|
|
29
36
|
&& sudo gem sources --clear-all \
|
|
30
37
|
&& SUDO_FORCE_REMOVE=yes \
|
|
31
38
|
apt-get purge -y --auto-remove \
|
|
32
39
|
-o APT::AutoRemove::RecommendsImportant=false \
|
|
33
40
|
$buildDeps \
|
|
34
41
|
&& rm -rf /var/lib/apt/lists/* \
|
|
35
|
-
|
|
36
|
-
|
|
42
|
+
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
|
|
43
|
+
|
|
44
|
+
# You can install the LI plugin using a gem or if you want to test your
|
|
45
|
+
# changes to plugin, you may add the .rb directly under `plugins` dir, then
|
|
46
|
+
# you don't need to install the gem
|
|
47
|
+
COPY plugins /fluentd/plugins/
|
|
48
|
+
|
|
@@ -23,30 +23,32 @@ data:
|
|
|
23
23
|
myapp-fluent.conf: |
|
|
24
24
|
# Input sources
|
|
25
25
|
@include general.conf
|
|
26
|
-
@include systemd
|
|
27
|
-
@include kubernetes
|
|
28
|
-
|
|
29
|
-
# Parsing/Filtering
|
|
30
|
-
@include kubernetes-filter.conf
|
|
26
|
+
@include systemd.conf
|
|
27
|
+
@include kubernetes.conf
|
|
28
|
+
@include kube-audit.conf
|
|
31
29
|
|
|
32
30
|
# Forwading - Be vigilant of the order in which these plugins are specified. Order matters!
|
|
33
|
-
@include
|
|
31
|
+
@include vmw-li.conf
|
|
34
32
|
|
|
35
33
|
general.conf: |
|
|
36
34
|
<system>
|
|
37
35
|
log_level info
|
|
38
36
|
</system>
|
|
39
37
|
# Prevent fluentd from handling records containing its own logs to handle cycles.
|
|
40
|
-
<
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
<label @FLUENT_LOG>
|
|
39
|
+
<match fluent.**>
|
|
40
|
+
@type null
|
|
41
|
+
</match>
|
|
42
|
+
</label>
|
|
43
43
|
|
|
44
|
-
systemd
|
|
44
|
+
systemd.conf: |
|
|
45
|
+
# Journal logs
|
|
45
46
|
<source>
|
|
46
47
|
@type systemd
|
|
48
|
+
@id in_systemd_logs
|
|
47
49
|
path /run/log/journal
|
|
48
50
|
# Can filter logs if we want, e.g.
|
|
49
|
-
#
|
|
51
|
+
#filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
|
|
50
52
|
<storage>
|
|
51
53
|
@type local
|
|
52
54
|
persistent true
|
|
@@ -57,70 +59,114 @@ data:
|
|
|
57
59
|
strip_underscores true
|
|
58
60
|
</source>
|
|
59
61
|
|
|
60
|
-
kubernetes
|
|
62
|
+
kubernetes.conf: |
|
|
63
|
+
# Container logs
|
|
64
|
+
# Kubernetes docker logs are stored under /var/lib/docker/containers for
|
|
65
|
+
# which kubernetes creates a symlink at /var/log/containers
|
|
61
66
|
<source>
|
|
62
67
|
@type tail
|
|
68
|
+
@id in_tail_container_logs
|
|
63
69
|
path /var/log/containers/*.log
|
|
64
70
|
# One could exclude certain logs like:
|
|
65
|
-
#
|
|
71
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
|
66
72
|
pos_file /var/log/fluentd-docker.pos
|
|
67
|
-
time_format %Y-%m-%dT%H:%M:%S
|
|
68
|
-
tag kubernetes.*
|
|
69
|
-
format json
|
|
70
73
|
read_from_head true
|
|
74
|
+
# Set this watcher to false if you have many files to tail
|
|
75
|
+
enable_stat_watcher false
|
|
76
|
+
refresh_interval 5
|
|
77
|
+
tag kubernetes.*
|
|
78
|
+
<parse>
|
|
79
|
+
@type json
|
|
80
|
+
time_key time
|
|
81
|
+
keep_time_key true
|
|
82
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
83
|
+
</parse>
|
|
71
84
|
</source>
|
|
72
|
-
|
|
73
|
-
kubernetes-filter.conf: |
|
|
85
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
|
74
86
|
<filter kubernetes.**>
|
|
75
87
|
@type kubernetes_metadata
|
|
76
|
-
|
|
77
|
-
|
|
88
|
+
@id filter_kube_metadata
|
|
89
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV. fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
|
90
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
|
91
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
|
92
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
|
93
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
|
94
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
|
95
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
|
96
|
+
</filter>
|
|
97
|
+
|
|
98
|
+
# Prefix the tag by namespace. This would make it easy to match logs by namespaces
|
|
99
|
+
<match kubernetes.**>
|
|
100
|
+
@type rewrite_tag_filter
|
|
101
|
+
<rule>
|
|
102
|
+
key $.kubernetes.namespace_name
|
|
103
|
+
pattern ^(.+)$
|
|
104
|
+
tag $1.${tag}
|
|
105
|
+
</rule>
|
|
106
|
+
</match>
|
|
107
|
+
|
|
108
|
+
kube-audit.conf: |
|
|
109
|
+
# Kube-apiserver audit logs
|
|
110
|
+
<source>
|
|
111
|
+
@type tail
|
|
112
|
+
@id in_tail_kube_audit_logs
|
|
113
|
+
# path to audit logs for kube-apiserver
|
|
114
|
+
path "/var/log/kube-audit/audit.log"
|
|
115
|
+
pos_file /var/log/kube-audit.pos
|
|
116
|
+
tag kube-audit
|
|
117
|
+
<parse>
|
|
118
|
+
@type json
|
|
119
|
+
time_key timestamp
|
|
120
|
+
keep_time_key false
|
|
121
|
+
time_format %Y-%m-%dT%H:%M:%SZ
|
|
122
|
+
</parse>
|
|
123
|
+
</source>
|
|
124
|
+
# Loginsight doesn't support ingesting `source` as a field name, get rid of it
|
|
125
|
+
<filter kube-audit>
|
|
126
|
+
@type record_transformer
|
|
127
|
+
@id filter_kube_audit_logs
|
|
128
|
+
enable_ruby
|
|
129
|
+
remove_keys source
|
|
130
|
+
<record>
|
|
131
|
+
log ${record}
|
|
132
|
+
</record>
|
|
78
133
|
</filter>
|
|
79
134
|
|
|
80
|
-
|
|
135
|
+
vmw-li.conf: |
|
|
136
|
+
# Match everything
|
|
81
137
|
# We are capturing all log messages and redirecting them to endpoints mentioned in each <store> tag.
|
|
82
138
|
# One may redirect these logs to muliple endpoints (including multiple LI instances).
|
|
83
139
|
# Or one may chose to tag their specific logs and add their own config to capture those tagged logs and redirect
|
|
84
|
-
# them to appropriate endpoint.
|
|
140
|
+
# them to appropriate endpoint. That specific config needs to preceed this generic one.
|
|
85
141
|
<match **>
|
|
86
142
|
@type copy
|
|
87
143
|
<store>
|
|
88
144
|
@type vmware_loginsight
|
|
145
|
+
@id out_vmw_li_all_container_logs
|
|
89
146
|
scheme https
|
|
90
147
|
ssl_verify true
|
|
91
148
|
# Loginsight host: One may use IP address or cname
|
|
92
|
-
#
|
|
93
|
-
host
|
|
94
|
-
port
|
|
95
|
-
path api/v1/events/ingest
|
|
149
|
+
#host X.X.X.X
|
|
150
|
+
host MY_LOGINSIGHT_HOST
|
|
151
|
+
port 9543
|
|
96
152
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
tag_key tag
|
|
153
|
+
# Keys from log event whose values should be added as log message/text to
|
|
154
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
155
|
+
log_text_keys ["log","msg","message"]
|
|
156
|
+
# Use this flag if you want to enable http debug logs
|
|
157
|
+
http_conn_debug false
|
|
103
158
|
</store>
|
|
104
|
-
#
|
|
105
|
-
#
|
|
106
|
-
#
|
|
107
|
-
#
|
|
159
|
+
# copy plugin supports sending/copying logs to multiple plugins
|
|
160
|
+
# One may choose to send them to multiple LIs
|
|
161
|
+
# Or one may want send a copy to stdout for debugging
|
|
162
|
+
# Please note, if you use stdout along with LI, catch the logger's log to make
|
|
163
|
+
# sure they're not cyclic
|
|
164
|
+
#<store>
|
|
165
|
+
# @type stdout
|
|
166
|
+
#</store>
|
|
108
167
|
</match>
|
|
109
168
|
|
|
110
169
|
|
|
111
|
-
extra.conf: |
|
|
112
|
-
# If we want to transform events we could use:
|
|
113
|
-
#<filter **>
|
|
114
|
-
# @type record_transformer
|
|
115
|
-
# enable_ruby
|
|
116
|
-
# auto_typecast
|
|
117
|
-
# <record>
|
|
118
|
-
# hostname "#{Socket.gethostname}"
|
|
119
|
-
# mykey ${["message"=>record.to_json]}
|
|
120
|
-
# </record>
|
|
121
|
-
#</filter>
|
|
122
|
-
|
|
123
|
-
|
|
124
170
|
---
|
|
125
171
|
kind: DaemonSet
|
|
126
172
|
apiVersion: extensions/v1beta1
|
|
@@ -131,8 +177,21 @@ metadata:
|
|
|
131
177
|
app: "log-collector"
|
|
132
178
|
version: v1
|
|
133
179
|
spec:
|
|
180
|
+
selector:
|
|
181
|
+
matchLabels:
|
|
182
|
+
app: "log-collector"
|
|
183
|
+
revisionHistoryLimit: 3
|
|
184
|
+
minReadySeconds: 10
|
|
185
|
+
updateStrategy:
|
|
186
|
+
type: RollingUpdate
|
|
187
|
+
rollingUpdate:
|
|
188
|
+
# How many pods can be unavailable during the rolling update.
|
|
189
|
+
maxUnavailable: 3
|
|
134
190
|
template:
|
|
135
191
|
metadata:
|
|
192
|
+
annotations:
|
|
193
|
+
# One may use this annotation to trigger rollout whenever fluentd config changes
|
|
194
|
+
configHash: GENERATED_HASH
|
|
136
195
|
labels:
|
|
137
196
|
app: "log-collector"
|
|
138
197
|
version: v1
|
|
@@ -14,7 +14,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
14
14
|
|
|
15
15
|
Gem::Specification.new do |spec|
|
|
16
16
|
spec.name = "fluent-plugin-vmware-loginsight"
|
|
17
|
-
spec.version = "
|
|
17
|
+
spec.version = File.read("VERSION").strip
|
|
18
18
|
spec.authors = ["Vishal Mohite", "Chris Todd"]
|
|
19
19
|
spec.email = ["vmohite@vmware.com", "toddc@vmware.com"]
|
|
20
20
|
|
|
@@ -70,6 +70,8 @@ module Fluent
|
|
|
70
70
|
config_param :flatten_hashes, :bool, :default => true
|
|
71
71
|
# Seperator to use for joining flattened keys
|
|
72
72
|
config_param :flatten_hashes_separator, :string, :default => "_"
|
|
73
|
+
# Rename fields names
|
|
74
|
+
config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
|
|
73
75
|
|
|
74
76
|
# Keys from log event to rewrite
|
|
75
77
|
# for instance from 'kubernetes_namespace' to 'k8s_namespace'
|
|
@@ -164,6 +166,10 @@ module Fluent
|
|
|
164
166
|
flattened_records.each do |key, value|
|
|
165
167
|
begin
|
|
166
168
|
next if value.nil?
|
|
169
|
+
# check if name of the key should be replaced
|
|
170
|
+
if @rename_fields.has_key?(key)
|
|
171
|
+
key = @rename_fields[key]
|
|
172
|
+
end
|
|
167
173
|
# LI doesn't support duplicate fields, make unique names by appending underscore
|
|
168
174
|
key = shorten_key(key)
|
|
169
175
|
while keys.include?(key)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-vmware-loginsight
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.11
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Vishal Mohite
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2021-03-31 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: bundler
|
|
@@ -87,6 +87,7 @@ files:
|
|
|
87
87
|
- LICENSE
|
|
88
88
|
- README.md
|
|
89
89
|
- Rakefile
|
|
90
|
+
- VERSION
|
|
90
91
|
- examples/fluent.conf
|
|
91
92
|
- examples/fluentd-vrli-plugin-debian.dockerfile
|
|
92
93
|
- examples/fluentd-vrli-plugin-photon-tdnf.dockerfile
|