fluent-plugin-vmware-loginsight 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +71 -18
- data/VERSION +1 -0
- data/examples/fluent.conf +104 -52
- data/examples/fluentd-vrli-plugin-debian.dockerfile +24 -12
- data/examples/k8s-log-collector-ds.yaml +108 -49
- data/fluent-plugin-vmware-loginsight.gemspec +1 -1
- data/lib/fluent/plugin/out_vmware_loginsight.rb +6 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 10d3d5209686fe48c79d1182e193316588f8e29bb6567d9ebe54b0eb061f0e55
|
|
4
|
+
data.tar.gz: 53b75b154b7ca8ff95c8aa71dd0fe91d00deff9460761ac4ea1c13da02c724e7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2f41fbe184204f8100ebca73447f87d1cdce2640941bdae56d34fff216cbd554a7c22ed1e71f5e39f23bcdb72d8b766897f91917959f29e7346e6deecb0a2f7e
|
|
7
|
+
data.tar.gz: 1ff567aaf08344dc494fdedb6ed2efc21084e55c66da28b33cdbd40a17331c02519e94226959c291e487787a57e12bdad2fc6cff386573fb483b92dd34430e47
|
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# fluent-plugin-vmware-loginsight
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight)
|
|
4
|
+
|
|
3
5
|
## Overview
|
|
4
6
|
output plugin to do forward logs to VMware Log Insight
|
|
5
7
|
|
|
@@ -28,39 +30,61 @@ $ bundle
|
|
|
28
30
|
## Usage
|
|
29
31
|
|
|
30
32
|
```
|
|
33
|
+
# Collect all container logs
|
|
31
34
|
<source>
|
|
32
35
|
@type tail
|
|
36
|
+
@id in_tail_container_logs
|
|
33
37
|
path /var/log/containers/*.log
|
|
38
|
+
# One could exclude certain logs like:
|
|
39
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
|
34
40
|
pos_file /var/log/fluentd-docker.pos
|
|
35
|
-
time_format %Y-%m-%dT%H:%M:%S
|
|
36
|
-
tag kubernetes.*
|
|
37
|
-
format json
|
|
38
41
|
read_from_head true
|
|
42
|
+
# Set this watcher to false if you have many files to tail
|
|
43
|
+
enable_stat_watcher false
|
|
44
|
+
refresh_interval 5
|
|
45
|
+
tag kubernetes.*
|
|
46
|
+
<parse>
|
|
47
|
+
@type json
|
|
48
|
+
time_key time
|
|
49
|
+
keep_time_key true
|
|
50
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
51
|
+
</parse>
|
|
39
52
|
</source>
|
|
40
53
|
|
|
41
|
-
# Kubernetes metadata filter that tags additional meta data for each event
|
|
42
|
-
<filter kubernetes
|
|
54
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
|
55
|
+
<filter kubernetes.**>
|
|
43
56
|
@type kubernetes_metadata
|
|
57
|
+
@id filter_kube_metadata
|
|
58
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
|
59
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
|
60
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
|
61
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
|
62
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
|
63
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
|
64
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
|
44
65
|
</filter>
|
|
45
66
|
|
|
67
|
+
# Match everything
|
|
46
68
|
<match **>
|
|
47
69
|
@type vmware_loginsight
|
|
70
|
+
@id out_vmw_li_all_container_logs
|
|
48
71
|
scheme https
|
|
49
72
|
ssl_verify true
|
|
50
|
-
# Loginsight host: One may use IP address or cname
|
|
51
|
-
#
|
|
52
|
-
host
|
|
53
|
-
port
|
|
54
|
-
path api/v1/events/ingest
|
|
73
|
+
# Loginsight host: One may use IP address or cname
|
|
74
|
+
#host X.X.X.X
|
|
75
|
+
host MY_LOGINSIGHT_HOST
|
|
76
|
+
port 9543
|
|
55
77
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
tag_key tag
|
|
78
|
+
# Keys from log event whose values should be added as log message/text to
|
|
79
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
80
|
+
log_text_keys ["log","msg","message"]
|
|
81
|
+
# Use this flag if you want to enable http debug logs
|
|
82
|
+
http_conn_debug false
|
|
62
83
|
</match>
|
|
63
84
|
```
|
|
85
|
+
|
|
86
|
+
For more examples look at [examples](./examples/)
|
|
87
|
+
|
|
64
88
|
### Configuration options
|
|
65
89
|
|
|
66
90
|
```
|
|
@@ -104,6 +128,9 @@ request_timeout, :time, :default => 5
|
|
|
104
128
|
# If set, enables debug logs for http connection
|
|
105
129
|
http_conn_debug, :bool, :default => false :: Valid Value: true | false
|
|
106
130
|
|
|
131
|
+
# Number of bytes per post request
|
|
132
|
+
max_batch_size, :integer, :default => 512000
|
|
133
|
+
|
|
107
134
|
# Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
|
|
108
135
|
rate_limit_msec, :integer, :default => 0
|
|
109
136
|
|
|
@@ -125,9 +152,35 @@ flatten_hashes, :bool, :default => true :: Valid Value: true | false
|
|
|
125
152
|
|
|
126
153
|
# Seperator to use for joining flattened keys
|
|
127
154
|
flatten_hashes_separator, :string, :default => "_"
|
|
128
|
-
```
|
|
129
155
|
|
|
130
|
-
|
|
156
|
+
# Rename fields names
|
|
157
|
+
config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
|
|
158
|
+
|
|
159
|
+
# Keys from log event to rewrite
|
|
160
|
+
# for instance from 'kubernetes_namespace' to 'k8s_namespace'
|
|
161
|
+
# tags will be rewritten with substring substitution
|
|
162
|
+
# and applied in the order present in the hash
|
|
163
|
+
# (Hashes enumerate their values in the order that the
|
|
164
|
+
# corresponding keys were inserted
|
|
165
|
+
# see https://ruby-doc.org/core-2.2.2/Hash.html)
|
|
166
|
+
# example config:
|
|
167
|
+
# shorten_keys {
|
|
168
|
+
# "__":"_",
|
|
169
|
+
# "container_":"",
|
|
170
|
+
# "kubernetes_":"k8s_",
|
|
171
|
+
# "labels_":"",
|
|
172
|
+
# }
|
|
173
|
+
shorten_keys, :hash, value_type: :string, default:
|
|
174
|
+
{
|
|
175
|
+
'kubernetes_':'k8s_',
|
|
176
|
+
'namespace':'ns',
|
|
177
|
+
'labels_':'',
|
|
178
|
+
'_name':'',
|
|
179
|
+
'_hash':'',
|
|
180
|
+
'container_':''
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
```
|
|
131
184
|
|
|
132
185
|
## Contributing
|
|
133
186
|
|
data/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.1.11
|
data/examples/fluent.conf
CHANGED
|
@@ -8,18 +8,25 @@
|
|
|
8
8
|
#
|
|
9
9
|
# SPDX-License-Identifier: MIT
|
|
10
10
|
|
|
11
|
+
# Sample Fluentd config, edit as per your needs.
|
|
12
|
+
# https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/templates/conf has some good fluentd config examples
|
|
11
13
|
|
|
14
|
+
# System level configs
|
|
12
15
|
<system>
|
|
13
16
|
log_level info
|
|
14
17
|
</system>
|
|
15
18
|
|
|
16
19
|
# Prevent fluentd from handling records containing its own logs to handle cycles.
|
|
17
|
-
<
|
|
18
|
-
|
|
19
|
-
|
|
20
|
+
<label @FLUENT_LOG>
|
|
21
|
+
<match fluent.**>
|
|
22
|
+
@type null
|
|
23
|
+
</match>
|
|
24
|
+
</label>
|
|
20
25
|
|
|
26
|
+
# Collect all journal logs
|
|
21
27
|
<source>
|
|
22
28
|
@type systemd
|
|
29
|
+
@id in_systemd_logs
|
|
23
30
|
path /run/log/journal
|
|
24
31
|
# Can filter logs if we want, e.g.
|
|
25
32
|
#filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
|
|
@@ -33,76 +40,121 @@
|
|
|
33
40
|
strip_underscores true
|
|
34
41
|
</source>
|
|
35
42
|
|
|
43
|
+
# Collect all container logs
|
|
36
44
|
<source>
|
|
37
45
|
@type tail
|
|
46
|
+
@id in_tail_container_logs
|
|
38
47
|
path /var/log/containers/*.log
|
|
39
48
|
# One could exclude certain logs like:
|
|
40
|
-
#
|
|
49
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
|
41
50
|
pos_file /var/log/fluentd-docker.pos
|
|
42
|
-
time_format %Y-%m-%dT%H:%M:%S
|
|
43
|
-
tag kubernetes.*
|
|
44
|
-
format json
|
|
45
51
|
read_from_head true
|
|
52
|
+
# Set this watcher to false if you have many files to tail
|
|
53
|
+
enable_stat_watcher false
|
|
54
|
+
refresh_interval 5
|
|
55
|
+
tag kubernetes.*
|
|
56
|
+
<parse>
|
|
57
|
+
@type json
|
|
58
|
+
time_key time
|
|
59
|
+
keep_time_key true
|
|
60
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
61
|
+
</parse>
|
|
46
62
|
</source>
|
|
47
63
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
#<source>
|
|
51
|
-
# @type tail
|
|
52
|
-
# path /var/log/containers/javaapp**.log
|
|
53
|
-
# pos_file /var/log/fluentd-dockerlog.pos
|
|
54
|
-
# time_format %b %d %H:%M:%S
|
|
55
|
-
# tag kubernetes.*
|
|
56
|
-
# format multiline
|
|
57
|
-
# format_firstline /\d{4}-\d{1,2}-\d{1,2}/
|
|
58
|
-
# format1 /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
|
|
59
|
-
# read_from_head true
|
|
60
|
-
#</source>
|
|
61
|
-
|
|
62
|
-
# Kubernetes metadata filter that tags additional meta data for each event
|
|
63
|
-
<filter kubernetes.var.log.containers.**.log>
|
|
64
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
|
65
|
+
<filter kubernetes.**>
|
|
64
66
|
@type kubernetes_metadata
|
|
67
|
+
@id filter_kube_metadata
|
|
68
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
|
69
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
|
70
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
|
71
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
|
72
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
|
73
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
|
74
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
|
65
75
|
</filter>
|
|
66
76
|
|
|
67
|
-
#
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
#</filter>
|
|
77
|
+
# Prefix the tag by namespace. This would make it easy to match logs by namespaces
|
|
78
|
+
<match kubernetes.**>
|
|
79
|
+
@type rewrite_tag_filter
|
|
80
|
+
<rule>
|
|
81
|
+
key $.kubernetes.namespace_name
|
|
82
|
+
pattern ^(.+)$
|
|
83
|
+
tag $1.${tag}
|
|
84
|
+
</rule>
|
|
85
|
+
</match>
|
|
77
86
|
|
|
78
|
-
|
|
79
|
-
|
|
87
|
+
# Collect all kube apiserver audit logs
|
|
88
|
+
<source>
|
|
89
|
+
@type tail
|
|
90
|
+
@id in_tail_kube_audit_logs
|
|
91
|
+
# audit log path of kube-apiserver
|
|
92
|
+
path "/var/log/kube-audit/audit.log"
|
|
93
|
+
pos_file /var/log/kube-audit.pos
|
|
94
|
+
tag kube-audit
|
|
95
|
+
<parse>
|
|
96
|
+
@type json
|
|
97
|
+
time_key timestamp
|
|
98
|
+
keep_time_key false
|
|
99
|
+
time_format %Y-%m-%dT%H:%M:%SZ
|
|
100
|
+
</parse>
|
|
101
|
+
</source>
|
|
102
|
+
|
|
103
|
+
# Loginsight doesn't support ingesting `source` as a field name, get rid of it
|
|
104
|
+
<filter kube-audit>
|
|
105
|
+
@type record_transformer
|
|
106
|
+
@id filter_kube_audit_logs
|
|
107
|
+
enable_ruby
|
|
108
|
+
remove_keys source
|
|
109
|
+
<record>
|
|
110
|
+
log ${record}
|
|
111
|
+
</record>
|
|
112
|
+
</filter>
|
|
113
|
+
|
|
114
|
+
# You can catch and match logs by namespace
|
|
115
|
+
<match my-namespace-one.** my-namespace-two.**>
|
|
116
|
+
@type vmware_loginsight
|
|
117
|
+
@id out_vmw_li_my_namespace_logs
|
|
118
|
+
scheme http
|
|
119
|
+
ssl_verify false
|
|
120
|
+
# Loginsight host: One may use IP address or cname
|
|
121
|
+
#host X.X.X.X
|
|
122
|
+
host MY_LOGINSIGHT_HOST
|
|
123
|
+
port 9000
|
|
124
|
+
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
125
|
+
# Keys from log event whose values should be added as log message/text to
|
|
126
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
127
|
+
log_text_keys ["log","msg","message"]
|
|
128
|
+
# Use this flag if you want to enable http debug logs
|
|
129
|
+
http_conn_debug false
|
|
80
130
|
</match>
|
|
81
131
|
|
|
132
|
+
# Match everything else
|
|
82
133
|
<match **>
|
|
83
134
|
@type copy
|
|
84
135
|
<store>
|
|
85
136
|
@type vmware_loginsight
|
|
137
|
+
@id out_vmw_li_all_container_logs
|
|
86
138
|
scheme https
|
|
87
139
|
ssl_verify true
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
host
|
|
91
|
-
port
|
|
92
|
-
path api/v1/events/ingest
|
|
140
|
+
# Loginsight host: One may use IP address or cname
|
|
141
|
+
#host X.X.X.X
|
|
142
|
+
host MY_LOGINSIGHT_HOST
|
|
143
|
+
port 9543
|
|
93
144
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
rate_limit_msec 0
|
|
97
|
-
raise_on_error false
|
|
145
|
+
# Keys from log event whose values should be added as log message/text to
|
|
146
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
98
147
|
log_text_keys ["log","msg","message"]
|
|
99
|
-
|
|
100
|
-
|
|
148
|
+
# Use this flag if you want to enable http debug logs
|
|
149
|
+
http_conn_debug false
|
|
101
150
|
</store>
|
|
102
|
-
# copy plugin supports sending/copying logs to multiple plugins
|
|
103
|
-
# One may choose to send them to multiple LIs
|
|
104
|
-
# Or one may want send a copy to stdout for debugging
|
|
105
|
-
#
|
|
106
|
-
#
|
|
107
|
-
|
|
151
|
+
# copy plugin supports sending/copying logs to multiple plugins
|
|
152
|
+
# One may choose to send them to multiple LIs
|
|
153
|
+
# Or one may want send a copy to stdout for debugging
|
|
154
|
+
# Please note, if you use stdout along with LI, catch the logger's log to make
|
|
155
|
+
# sure they're not cyclic
|
|
156
|
+
#<store>
|
|
157
|
+
# @type stdout
|
|
158
|
+
#</store>
|
|
108
159
|
</match>
|
|
160
|
+
|
|
@@ -8,29 +8,41 @@
|
|
|
8
8
|
#
|
|
9
9
|
# SPDX-License-Identifier: MIT
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
#
|
|
11
|
+
|
|
12
|
+
# Sample Dockerfile to use as log collector
|
|
13
|
+
# Builds a debian-based fluentd image that has fluent-plugin-kubernetes_metadata_filter,
|
|
14
|
+
# fluent-plugin-rewrite-tag-filter, fluent-plugin-systemd and
|
|
15
|
+
# fluent-plugin-vmware-loginsight gem installed.
|
|
13
16
|
#
|
|
14
|
-
#
|
|
17
|
+
# This image will get preconfigured with the fluent.conf if avaialble at the
|
|
18
|
+
# same dir level. For fluentd config example, see
|
|
15
19
|
# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
|
|
16
|
-
FROM fluent/fluentd:v0.14.15-debian-onbuild
|
|
17
|
-
# Above image expects the loginsight plugin vmware_loginsight to be available under ./plugins/vmware_loginsight.rb
|
|
18
|
-
# and fluentd config under ./fluent.conf by default
|
|
19
20
|
|
|
21
|
+
# This base image is built from https://github.com/fluent/fluentd-kubernetes-daemonset
|
|
22
|
+
FROM fluent/fluentd:v1.11-debian-1
|
|
23
|
+
|
|
24
|
+
# Use root account to use apt
|
|
20
25
|
USER root
|
|
21
26
|
|
|
22
|
-
|
|
27
|
+
# You can install your plugins here
|
|
28
|
+
RUN buildDeps="sudo make gcc g++ libc-dev" \
|
|
23
29
|
&& apt-get update \
|
|
24
30
|
&& apt-get install -y --no-install-recommends $buildDeps \
|
|
25
31
|
&& sudo gem install \
|
|
26
|
-
fluent-plugin-
|
|
27
|
-
fluent-plugin-
|
|
28
|
-
fluent-plugin-
|
|
32
|
+
fluent-plugin-kubernetes_metadata_filter:2.4.6 \
|
|
33
|
+
fluent-plugin-rewrite-tag-filter:2.3.0 \
|
|
34
|
+
fluent-plugin-systemd:1.0.2 \
|
|
35
|
+
fluent-plugin-vmware-loginsight:0.1.10 \
|
|
29
36
|
&& sudo gem sources --clear-all \
|
|
30
37
|
&& SUDO_FORCE_REMOVE=yes \
|
|
31
38
|
apt-get purge -y --auto-remove \
|
|
32
39
|
-o APT::AutoRemove::RecommendsImportant=false \
|
|
33
40
|
$buildDeps \
|
|
34
41
|
&& rm -rf /var/lib/apt/lists/* \
|
|
35
|
-
|
|
36
|
-
|
|
42
|
+
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
|
|
43
|
+
|
|
44
|
+
# You can install the LI plugin using a gem or if you want to test your
|
|
45
|
+
# changes to plugin, you may add the .rb directly under `plugins` dir, then
|
|
46
|
+
# you don't need to install the gem
|
|
47
|
+
COPY plugins /fluentd/plugins/
|
|
48
|
+
|
|
@@ -23,30 +23,32 @@ data:
|
|
|
23
23
|
myapp-fluent.conf: |
|
|
24
24
|
# Input sources
|
|
25
25
|
@include general.conf
|
|
26
|
-
@include systemd
|
|
27
|
-
@include kubernetes
|
|
28
|
-
|
|
29
|
-
# Parsing/Filtering
|
|
30
|
-
@include kubernetes-filter.conf
|
|
26
|
+
@include systemd.conf
|
|
27
|
+
@include kubernetes.conf
|
|
28
|
+
@include kube-audit.conf
|
|
31
29
|
|
|
32
30
|
# Forwading - Be vigilant of the order in which these plugins are specified. Order matters!
|
|
33
|
-
@include
|
|
31
|
+
@include vmw-li.conf
|
|
34
32
|
|
|
35
33
|
general.conf: |
|
|
36
34
|
<system>
|
|
37
35
|
log_level info
|
|
38
36
|
</system>
|
|
39
37
|
# Prevent fluentd from handling records containing its own logs to handle cycles.
|
|
40
|
-
<
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
<label @FLUENT_LOG>
|
|
39
|
+
<match fluent.**>
|
|
40
|
+
@type null
|
|
41
|
+
</match>
|
|
42
|
+
</label>
|
|
43
43
|
|
|
44
|
-
systemd
|
|
44
|
+
systemd.conf: |
|
|
45
|
+
# Journal logs
|
|
45
46
|
<source>
|
|
46
47
|
@type systemd
|
|
48
|
+
@id in_systemd_logs
|
|
47
49
|
path /run/log/journal
|
|
48
50
|
# Can filter logs if we want, e.g.
|
|
49
|
-
#
|
|
51
|
+
#filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
|
|
50
52
|
<storage>
|
|
51
53
|
@type local
|
|
52
54
|
persistent true
|
|
@@ -57,70 +59,114 @@ data:
|
|
|
57
59
|
strip_underscores true
|
|
58
60
|
</source>
|
|
59
61
|
|
|
60
|
-
kubernetes
|
|
62
|
+
kubernetes.conf: |
|
|
63
|
+
# Container logs
|
|
64
|
+
# Kubernetes docker logs are stored under /var/lib/docker/containers for
|
|
65
|
+
# which kubernetes creates a symlink at /var/log/containers
|
|
61
66
|
<source>
|
|
62
67
|
@type tail
|
|
68
|
+
@id in_tail_container_logs
|
|
63
69
|
path /var/log/containers/*.log
|
|
64
70
|
# One could exclude certain logs like:
|
|
65
|
-
#
|
|
71
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
|
66
72
|
pos_file /var/log/fluentd-docker.pos
|
|
67
|
-
time_format %Y-%m-%dT%H:%M:%S
|
|
68
|
-
tag kubernetes.*
|
|
69
|
-
format json
|
|
70
73
|
read_from_head true
|
|
74
|
+
# Set this watcher to false if you have many files to tail
|
|
75
|
+
enable_stat_watcher false
|
|
76
|
+
refresh_interval 5
|
|
77
|
+
tag kubernetes.*
|
|
78
|
+
<parse>
|
|
79
|
+
@type json
|
|
80
|
+
time_key time
|
|
81
|
+
keep_time_key true
|
|
82
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
83
|
+
</parse>
|
|
71
84
|
</source>
|
|
72
|
-
|
|
73
|
-
kubernetes-filter.conf: |
|
|
85
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
|
74
86
|
<filter kubernetes.**>
|
|
75
87
|
@type kubernetes_metadata
|
|
76
|
-
|
|
77
|
-
|
|
88
|
+
@id filter_kube_metadata
|
|
89
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV. fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
|
90
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
|
91
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
|
92
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
|
93
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
|
94
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
|
95
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
|
96
|
+
</filter>
|
|
97
|
+
|
|
98
|
+
# Prefix the tag by namespace. This would make it easy to match logs by namespaces
|
|
99
|
+
<match kubernetes.**>
|
|
100
|
+
@type rewrite_tag_filter
|
|
101
|
+
<rule>
|
|
102
|
+
key $.kubernetes.namespace_name
|
|
103
|
+
pattern ^(.+)$
|
|
104
|
+
tag $1.${tag}
|
|
105
|
+
</rule>
|
|
106
|
+
</match>
|
|
107
|
+
|
|
108
|
+
kube-audit.conf: |
|
|
109
|
+
# Kube-apiserver audit logs
|
|
110
|
+
<source>
|
|
111
|
+
@type tail
|
|
112
|
+
@id in_tail_kube_audit_logs
|
|
113
|
+
# path to audit logs for kube-apiserver
|
|
114
|
+
path "/var/log/kube-audit/audit.log"
|
|
115
|
+
pos_file /var/log/kube-audit.pos
|
|
116
|
+
tag kube-audit
|
|
117
|
+
<parse>
|
|
118
|
+
@type json
|
|
119
|
+
time_key timestamp
|
|
120
|
+
keep_time_key false
|
|
121
|
+
time_format %Y-%m-%dT%H:%M:%SZ
|
|
122
|
+
</parse>
|
|
123
|
+
</source>
|
|
124
|
+
# Loginsight doesn't support ingesting `source` as a field name, get rid of it
|
|
125
|
+
<filter kube-audit>
|
|
126
|
+
@type record_transformer
|
|
127
|
+
@id filter_kube_audit_logs
|
|
128
|
+
enable_ruby
|
|
129
|
+
remove_keys source
|
|
130
|
+
<record>
|
|
131
|
+
log ${record}
|
|
132
|
+
</record>
|
|
78
133
|
</filter>
|
|
79
134
|
|
|
80
|
-
|
|
135
|
+
vmw-li.conf: |
|
|
136
|
+
# Match everything
|
|
81
137
|
# We are capturing all log messages and redirecting them to endpoints mentioned in each <store> tag.
|
|
82
138
|
# One may redirect these logs to muliple endpoints (including multiple LI instances).
|
|
83
139
|
# Or one may chose to tag their specific logs and add their own config to capture those tagged logs and redirect
|
|
84
|
-
# them to appropriate endpoint.
|
|
140
|
+
# them to appropriate endpoint. That specific config needs to preceed this generic one.
|
|
85
141
|
<match **>
|
|
86
142
|
@type copy
|
|
87
143
|
<store>
|
|
88
144
|
@type vmware_loginsight
|
|
145
|
+
@id out_vmw_li_all_container_logs
|
|
89
146
|
scheme https
|
|
90
147
|
ssl_verify true
|
|
91
148
|
# Loginsight host: One may use IP address or cname
|
|
92
|
-
#
|
|
93
|
-
host
|
|
94
|
-
port
|
|
95
|
-
path api/v1/events/ingest
|
|
149
|
+
#host X.X.X.X
|
|
150
|
+
host MY_LOGINSIGHT_HOST
|
|
151
|
+
port 9543
|
|
96
152
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
tag_key tag
|
|
153
|
+
# Keys from log event whose values should be added as log message/text to
|
|
154
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
|
155
|
+
log_text_keys ["log","msg","message"]
|
|
156
|
+
# Use this flag if you want to enable http debug logs
|
|
157
|
+
http_conn_debug false
|
|
103
158
|
</store>
|
|
104
|
-
#
|
|
105
|
-
#
|
|
106
|
-
#
|
|
107
|
-
#
|
|
159
|
+
# copy plugin supports sending/copying logs to multiple plugins
|
|
160
|
+
# One may choose to send them to multiple LIs
|
|
161
|
+
# Or one may want send a copy to stdout for debugging
|
|
162
|
+
# Please note, if you use stdout along with LI, catch the logger's log to make
|
|
163
|
+
# sure they're not cyclic
|
|
164
|
+
#<store>
|
|
165
|
+
# @type stdout
|
|
166
|
+
#</store>
|
|
108
167
|
</match>
|
|
109
168
|
|
|
110
169
|
|
|
111
|
-
extra.conf: |
|
|
112
|
-
# If we want to transform events we could use:
|
|
113
|
-
#<filter **>
|
|
114
|
-
# @type record_transformer
|
|
115
|
-
# enable_ruby
|
|
116
|
-
# auto_typecast
|
|
117
|
-
# <record>
|
|
118
|
-
# hostname "#{Socket.gethostname}"
|
|
119
|
-
# mykey ${["message"=>record.to_json]}
|
|
120
|
-
# </record>
|
|
121
|
-
#</filter>
|
|
122
|
-
|
|
123
|
-
|
|
124
170
|
---
|
|
125
171
|
kind: DaemonSet
|
|
126
172
|
apiVersion: extensions/v1beta1
|
|
@@ -131,8 +177,21 @@ metadata:
|
|
|
131
177
|
app: "log-collector"
|
|
132
178
|
version: v1
|
|
133
179
|
spec:
|
|
180
|
+
selector:
|
|
181
|
+
matchLabels:
|
|
182
|
+
app: "log-collector"
|
|
183
|
+
revisionHistoryLimit: 3
|
|
184
|
+
minReadySeconds: 10
|
|
185
|
+
updateStrategy:
|
|
186
|
+
type: RollingUpdate
|
|
187
|
+
rollingUpdate:
|
|
188
|
+
# How many pods can be unavailable during the rolling update.
|
|
189
|
+
maxUnavailable: 3
|
|
134
190
|
template:
|
|
135
191
|
metadata:
|
|
192
|
+
annotations:
|
|
193
|
+
# One may use this annotation to trigger rollout whenever fluentd config changes
|
|
194
|
+
configHash: GENERATED_HASH
|
|
136
195
|
labels:
|
|
137
196
|
app: "log-collector"
|
|
138
197
|
version: v1
|
|
@@ -14,7 +14,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
14
14
|
|
|
15
15
|
Gem::Specification.new do |spec|
|
|
16
16
|
spec.name = "fluent-plugin-vmware-loginsight"
|
|
17
|
-
spec.version = "
|
|
17
|
+
spec.version = File.read("VERSION").strip
|
|
18
18
|
spec.authors = ["Vishal Mohite", "Chris Todd"]
|
|
19
19
|
spec.email = ["vmohite@vmware.com", "toddc@vmware.com"]
|
|
20
20
|
|
|
@@ -70,6 +70,8 @@ module Fluent
|
|
|
70
70
|
config_param :flatten_hashes, :bool, :default => true
|
|
71
71
|
# Seperator to use for joining flattened keys
|
|
72
72
|
config_param :flatten_hashes_separator, :string, :default => "_"
|
|
73
|
+
# Rename fields names
|
|
74
|
+
config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
|
|
73
75
|
|
|
74
76
|
# Keys from log event to rewrite
|
|
75
77
|
# for instance from 'kubernetes_namespace' to 'k8s_namespace'
|
|
@@ -164,6 +166,10 @@ module Fluent
|
|
|
164
166
|
flattened_records.each do |key, value|
|
|
165
167
|
begin
|
|
166
168
|
next if value.nil?
|
|
169
|
+
# check if name of the key should be replaced
|
|
170
|
+
if @rename_fields.has_key?(key)
|
|
171
|
+
key = @rename_fields[key]
|
|
172
|
+
end
|
|
167
173
|
# LI doesn't support duplicate fields, make unique names by appending underscore
|
|
168
174
|
key = shorten_key(key)
|
|
169
175
|
while keys.include?(key)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-vmware-loginsight
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.11
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Vishal Mohite
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2021-03-31 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: bundler
|
|
@@ -87,6 +87,7 @@ files:
|
|
|
87
87
|
- LICENSE
|
|
88
88
|
- README.md
|
|
89
89
|
- Rakefile
|
|
90
|
+
- VERSION
|
|
90
91
|
- examples/fluent.conf
|
|
91
92
|
- examples/fluentd-vrli-plugin-debian.dockerfile
|
|
92
93
|
- examples/fluentd-vrli-plugin-photon-tdnf.dockerfile
|