fluent-plugin-vmware-loginsight 0.1.6 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/gem-push.yml +38 -0
- data/CHANGELOG.md +63 -0
- data/README.md +71 -18
- data/VERSION +1 -0
- data/examples/fluent.conf +104 -52
- data/examples/fluentd-vrli-plugin-debian.dockerfile +24 -12
- data/examples/k8s-log-collector-ds.yaml +108 -49
- data/fluent-plugin-vmware-loginsight.gemspec +1 -1
- data/lib/fluent/plugin/out_vmware_loginsight.rb +273 -272
- metadata +9 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f7e7d17226c807f4fffdd8723c9f260730af5de29334a1b10271993b21709185
|
4
|
+
data.tar.gz: 768c2724b2b84ac8ae8c1e4473dbe850c37a87102aaed95d6c52fc4f06f9a0df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 912e7dfa34b63dc9da92635603e88fea94d340ce8fb9aef99827bfd6896690a89533fe523416c2eafd34da2470016b13d23c290c74d93dc05fdb320beec78b55
|
7
|
+
data.tar.gz: 5998ef3e2869d13cd30dc934347b3dad6ebe2337de5678a7fb12f641b15a5b0a5a60daaa8bd9b5490ecdf58e5fb5f3529dd313ea2606126aaae8456dad023ffb
|
@@ -0,0 +1,38 @@
|
|
1
|
+
name: Ruby Gem
|
2
|
+
|
3
|
+
on: workflow_dispatch
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
name: Build + Publish
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
|
10
|
+
steps:
|
11
|
+
- uses: actions/checkout@v2
|
12
|
+
- name: Set up Ruby 2.6
|
13
|
+
uses: actions/setup-ruby@v1
|
14
|
+
with:
|
15
|
+
ruby-version: 2.6.x
|
16
|
+
|
17
|
+
- name: Publish to GPR
|
18
|
+
run: |
|
19
|
+
mkdir -p $HOME/.gem
|
20
|
+
touch $HOME/.gem/credentials
|
21
|
+
chmod 0600 $HOME/.gem/credentials
|
22
|
+
printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
23
|
+
gem build *.gemspec
|
24
|
+
gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
|
25
|
+
env:
|
26
|
+
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
27
|
+
OWNER: ${{ github.repository_owner }}
|
28
|
+
|
29
|
+
- name: Publish to RubyGems
|
30
|
+
run: |
|
31
|
+
mkdir -p $HOME/.gem
|
32
|
+
touch $HOME/.gem/credentials
|
33
|
+
chmod 0600 $HOME/.gem/credentials
|
34
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
35
|
+
gem build *.gemspec
|
36
|
+
gem push *.gem
|
37
|
+
env:
|
38
|
+
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
# Changelog
|
2
|
+
|
3
|
+
## v1.0.0 - April 19, 2021
|
4
|
+
|
5
|
+
* Update plugin structure to use Fluentd 1.x syntax
|
6
|
+
|
7
|
+
## v0.1.11 - March 31, 2021
|
8
|
+
|
9
|
+
* Add an option to rename Loginsight fields. This option could be used to rename certain fields that are reserved by Loginsight
|
10
|
+
|
11
|
+
## v0.1.10 - May 13, 2020
|
12
|
+
|
13
|
+
* Escape `@` char from Loginsight field
|
14
|
+
|
15
|
+
## v0.1.9 - May 07, 2020
|
16
|
+
|
17
|
+
* No change
|
18
|
+
|
19
|
+
## v0.1.8 - May 06, 2020 yanked, Not available
|
20
|
+
|
21
|
+
* Parameterize and add an option to shorten Loginsight field names
|
22
|
+
|
23
|
+
## v0.1.7 - December 10, 2019
|
24
|
+
|
25
|
+
* Fix basic authentication #8
|
26
|
+
|
27
|
+
## v0.1.6 - September 13, 2019
|
28
|
+
|
29
|
+
* For immutable log fields, use a copy to utf encode. This should fix 'can't modify frozen String' error in #5
|
30
|
+
|
31
|
+
## v0.1.5 - October 22, 2018
|
32
|
+
|
33
|
+
* Add option to display debug logs for http connection, default false
|
34
|
+
* Flatten Lists/Arrays for LI fields
|
35
|
+
* Convert LI field value to String to ensure no utf encoding errors
|
36
|
+
* Update help doc/examples with sample use of @log_text_keys and @http_conn_debug options
|
37
|
+
|
38
|
+
## v0.1.4 - October 17, 2018
|
39
|
+
|
40
|
+
* Add option to specify a list of keys that plugin should treat as log messages and forward them as text to Loginsight. Plugin should not flatten these fields
|
41
|
+
* If user specifies flatten_hashes option as false, plugin should try to add record key/values as is
|
42
|
+
|
43
|
+
## v0.1.3 - September 13, 2018
|
44
|
+
|
45
|
+
* Reorder namespace and name fields to be shorten
|
46
|
+
|
47
|
+
## v0.1.2 - September 10, 2018
|
48
|
+
|
49
|
+
* Republished yanked gem
|
50
|
+
|
51
|
+
## v0.1.1 - August 30, 2018 yanked, Not available
|
52
|
+
|
53
|
+
* Send log messages in batches, add max_batch_size parameter
|
54
|
+
* Shorten common kubernetes Loginsight field names
|
55
|
+
* Convert time to milliseconds
|
56
|
+
|
57
|
+
|
58
|
+
## 0.1.0 - August 30, 2018
|
59
|
+
|
60
|
+
### Initial release
|
61
|
+
|
62
|
+
* Fluentd output plugin to push logs to VMware Log Insight
|
63
|
+
|
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# fluent-plugin-vmware-loginsight
|
2
2
|
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight.svg)](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight)
|
4
|
+
|
3
5
|
## Overview
|
4
6
|
output plugin to do forward logs to VMware Log Insight
|
5
7
|
|
@@ -28,39 +30,61 @@ $ bundle
|
|
28
30
|
## Usage
|
29
31
|
|
30
32
|
```
|
33
|
+
# Collect all container logs
|
31
34
|
<source>
|
32
35
|
@type tail
|
36
|
+
@id in_tail_container_logs
|
33
37
|
path /var/log/containers/*.log
|
38
|
+
# One could exclude certain logs like:
|
39
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
34
40
|
pos_file /var/log/fluentd-docker.pos
|
35
|
-
time_format %Y-%m-%dT%H:%M:%S
|
36
|
-
tag kubernetes.*
|
37
|
-
format json
|
38
41
|
read_from_head true
|
42
|
+
# Set this watcher to false if you have many files to tail
|
43
|
+
enable_stat_watcher false
|
44
|
+
refresh_interval 5
|
45
|
+
tag kubernetes.*
|
46
|
+
<parse>
|
47
|
+
@type json
|
48
|
+
time_key time
|
49
|
+
keep_time_key true
|
50
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
51
|
+
</parse>
|
39
52
|
</source>
|
40
53
|
|
41
|
-
# Kubernetes metadata filter that tags additional meta data for each event
|
42
|
-
<filter kubernetes
|
54
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
55
|
+
<filter kubernetes.**>
|
43
56
|
@type kubernetes_metadata
|
57
|
+
@id filter_kube_metadata
|
58
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
59
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
60
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
61
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
62
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
63
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
64
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
44
65
|
</filter>
|
45
66
|
|
67
|
+
# Match everything
|
46
68
|
<match **>
|
47
69
|
@type vmware_loginsight
|
70
|
+
@id out_vmw_li_all_container_logs
|
48
71
|
scheme https
|
49
72
|
ssl_verify true
|
50
|
-
# Loginsight host: One may use IP address or cname
|
51
|
-
#
|
52
|
-
host
|
53
|
-
port
|
54
|
-
path api/v1/events/ingest
|
73
|
+
# Loginsight host: One may use IP address or cname
|
74
|
+
#host X.X.X.X
|
75
|
+
host MY_LOGINSIGHT_HOST
|
76
|
+
port 9543
|
55
77
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
tag_key tag
|
78
|
+
# Keys from log event whose values should be added as log message/text to
|
79
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
80
|
+
log_text_keys ["log","msg","message"]
|
81
|
+
# Use this flag if you want to enable http debug logs
|
82
|
+
http_conn_debug false
|
62
83
|
</match>
|
63
84
|
```
|
85
|
+
|
86
|
+
For more examples look at [examples](./examples/)
|
87
|
+
|
64
88
|
### Configuration options
|
65
89
|
|
66
90
|
```
|
@@ -104,6 +128,9 @@ request_timeout, :time, :default => 5
|
|
104
128
|
# If set, enables debug logs for http connection
|
105
129
|
http_conn_debug, :bool, :default => false :: Valid Value: true | false
|
106
130
|
|
131
|
+
# Number of bytes per post request
|
132
|
+
max_batch_size, :integer, :default => 512000
|
133
|
+
|
107
134
|
# Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
|
108
135
|
rate_limit_msec, :integer, :default => 0
|
109
136
|
|
@@ -125,9 +152,35 @@ flatten_hashes, :bool, :default => true :: Valid Value: true | false
|
|
125
152
|
|
126
153
|
# Seperator to use for joining flattened keys
|
127
154
|
flatten_hashes_separator, :string, :default => "_"
|
128
|
-
```
|
129
155
|
|
130
|
-
|
156
|
+
# Rename fields names
|
157
|
+
config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
|
158
|
+
|
159
|
+
# Keys from log event to rewrite
|
160
|
+
# for instance from 'kubernetes_namespace' to 'k8s_namespace'
|
161
|
+
# tags will be rewritten with substring substitution
|
162
|
+
# and applied in the order present in the hash
|
163
|
+
# (Hashes enumerate their values in the order that the
|
164
|
+
# corresponding keys were inserted
|
165
|
+
# see https://ruby-doc.org/core-2.2.2/Hash.html)
|
166
|
+
# example config:
|
167
|
+
# shorten_keys {
|
168
|
+
# "__":"_",
|
169
|
+
# "container_":"",
|
170
|
+
# "kubernetes_":"k8s_",
|
171
|
+
# "labels_":"",
|
172
|
+
# }
|
173
|
+
shorten_keys, :hash, value_type: :string, default:
|
174
|
+
{
|
175
|
+
'kubernetes_':'k8s_',
|
176
|
+
'namespace':'ns',
|
177
|
+
'labels_':'',
|
178
|
+
'_name':'',
|
179
|
+
'_hash':'',
|
180
|
+
'container_':''
|
181
|
+
}
|
182
|
+
|
183
|
+
```
|
131
184
|
|
132
185
|
## Contributing
|
133
186
|
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.0.0
|
data/examples/fluent.conf
CHANGED
@@ -8,18 +8,25 @@
|
|
8
8
|
#
|
9
9
|
# SPDX-License-Identifier: MIT
|
10
10
|
|
11
|
+
# Sample Fluentd config, edit as per your needs.
|
12
|
+
# https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/templates/conf has some good fluentd config examples
|
11
13
|
|
14
|
+
# System level configs
|
12
15
|
<system>
|
13
16
|
log_level info
|
14
17
|
</system>
|
15
18
|
|
16
19
|
# Prevent fluentd from handling records containing its own logs to handle cycles.
|
17
|
-
<
|
18
|
-
|
19
|
-
|
20
|
+
<label @FLUENT_LOG>
|
21
|
+
<match fluent.**>
|
22
|
+
@type null
|
23
|
+
</match>
|
24
|
+
</label>
|
20
25
|
|
26
|
+
# Collect all journal logs
|
21
27
|
<source>
|
22
28
|
@type systemd
|
29
|
+
@id in_systemd_logs
|
23
30
|
path /run/log/journal
|
24
31
|
# Can filter logs if we want, e.g.
|
25
32
|
#filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
|
@@ -33,76 +40,121 @@
|
|
33
40
|
strip_underscores true
|
34
41
|
</source>
|
35
42
|
|
43
|
+
# Collect all container logs
|
36
44
|
<source>
|
37
45
|
@type tail
|
46
|
+
@id in_tail_container_logs
|
38
47
|
path /var/log/containers/*.log
|
39
48
|
# One could exclude certain logs like:
|
40
|
-
#
|
49
|
+
#exclude_path ["/var/log/containers/log-collector*.log"]
|
41
50
|
pos_file /var/log/fluentd-docker.pos
|
42
|
-
time_format %Y-%m-%dT%H:%M:%S
|
43
|
-
tag kubernetes.*
|
44
|
-
format json
|
45
51
|
read_from_head true
|
52
|
+
# Set this watcher to false if you have many files to tail
|
53
|
+
enable_stat_watcher false
|
54
|
+
refresh_interval 5
|
55
|
+
tag kubernetes.*
|
56
|
+
<parse>
|
57
|
+
@type json
|
58
|
+
time_key time
|
59
|
+
keep_time_key true
|
60
|
+
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
61
|
+
</parse>
|
46
62
|
</source>
|
47
63
|
|
48
|
-
|
49
|
-
|
50
|
-
#<source>
|
51
|
-
# @type tail
|
52
|
-
# path /var/log/containers/javaapp**.log
|
53
|
-
# pos_file /var/log/fluentd-dockerlog.pos
|
54
|
-
# time_format %b %d %H:%M:%S
|
55
|
-
# tag kubernetes.*
|
56
|
-
# format multiline
|
57
|
-
# format_firstline /\d{4}-\d{1,2}-\d{1,2}/
|
58
|
-
# format1 /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
|
59
|
-
# read_from_head true
|
60
|
-
#</source>
|
61
|
-
|
62
|
-
# Kubernetes metadata filter that tags additional meta data for each event
|
63
|
-
<filter kubernetes.var.log.containers.**.log>
|
64
|
+
# Kubernetes metadata filter that tags additional meta data for each container event
|
65
|
+
<filter kubernetes.**>
|
64
66
|
@type kubernetes_metadata
|
67
|
+
@id filter_kube_metadata
|
68
|
+
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
|
69
|
+
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
|
70
|
+
ca_file "#{ENV['KUBERNETES_CA_FILE']}"
|
71
|
+
skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
|
72
|
+
skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
|
73
|
+
skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
|
74
|
+
skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
|
65
75
|
</filter>
|
66
76
|
|
67
|
-
#
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
#</filter>
|
77
|
+
# Prefix the tag by namespace. This would make it easy to match logs by namespaces
|
78
|
+
<match kubernetes.**>
|
79
|
+
@type rewrite_tag_filter
|
80
|
+
<rule>
|
81
|
+
key $.kubernetes.namespace_name
|
82
|
+
pattern ^(.+)$
|
83
|
+
tag $1.${tag}
|
84
|
+
</rule>
|
85
|
+
</match>
|
77
86
|
|
78
|
-
|
79
|
-
|
87
|
+
# Collect all kube apiserver audit logs
|
88
|
+
<source>
|
89
|
+
@type tail
|
90
|
+
@id in_tail_kube_audit_logs
|
91
|
+
# audit log path of kube-apiserver
|
92
|
+
path "/var/log/kube-audit/audit.log"
|
93
|
+
pos_file /var/log/kube-audit.pos
|
94
|
+
tag kube-audit
|
95
|
+
<parse>
|
96
|
+
@type json
|
97
|
+
time_key timestamp
|
98
|
+
keep_time_key false
|
99
|
+
time_format %Y-%m-%dT%H:%M:%SZ
|
100
|
+
</parse>
|
101
|
+
</source>
|
102
|
+
|
103
|
+
# Loginsight doesn't support ingesting `source` as a field name, get rid of it
|
104
|
+
<filter kube-audit>
|
105
|
+
@type record_transformer
|
106
|
+
@id filter_kube_audit_logs
|
107
|
+
enable_ruby
|
108
|
+
remove_keys source
|
109
|
+
<record>
|
110
|
+
log ${record}
|
111
|
+
</record>
|
112
|
+
</filter>
|
113
|
+
|
114
|
+
# You can catch and match logs by namespace
|
115
|
+
<match my-namespace-one.** my-namespace-two.**>
|
116
|
+
@type vmware_loginsight
|
117
|
+
@id out_vmw_li_my_namespace_logs
|
118
|
+
scheme http
|
119
|
+
ssl_verify false
|
120
|
+
# Loginsight host: One may use IP address or cname
|
121
|
+
#host X.X.X.X
|
122
|
+
host MY_LOGINSIGHT_HOST
|
123
|
+
port 9000
|
124
|
+
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
125
|
+
# Keys from log event whose values should be added as log message/text to
|
126
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
127
|
+
log_text_keys ["log","msg","message"]
|
128
|
+
# Use this flag if you want to enable http debug logs
|
129
|
+
http_conn_debug false
|
80
130
|
</match>
|
81
131
|
|
132
|
+
# Match everything else
|
82
133
|
<match **>
|
83
134
|
@type copy
|
84
135
|
<store>
|
85
136
|
@type vmware_loginsight
|
137
|
+
@id out_vmw_li_all_container_logs
|
86
138
|
scheme https
|
87
139
|
ssl_verify true
|
88
|
-
#
|
89
|
-
#
|
90
|
-
host
|
91
|
-
port
|
92
|
-
path api/v1/events/ingest
|
140
|
+
# Loginsight host: One may use IP address or cname
|
141
|
+
#host X.X.X.X
|
142
|
+
host MY_LOGINSIGHT_HOST
|
143
|
+
port 9543
|
93
144
|
agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
94
|
-
|
95
|
-
|
96
|
-
rate_limit_msec 0
|
97
|
-
raise_on_error false
|
145
|
+
# Keys from log event whose values should be added as log message/text to
|
146
|
+
# Loginsight. Note these key/value pairs won't be added as metadata/fields
|
98
147
|
log_text_keys ["log","msg","message"]
|
99
|
-
|
100
|
-
|
148
|
+
# Use this flag if you want to enable http debug logs
|
149
|
+
http_conn_debug false
|
101
150
|
</store>
|
102
|
-
# copy plugin supports sending/copying logs to multiple plugins
|
103
|
-
# One may choose to send them to multiple LIs
|
104
|
-
# Or one may want send a copy to stdout for debugging
|
105
|
-
#
|
106
|
-
#
|
107
|
-
|
151
|
+
# copy plugin supports sending/copying logs to multiple plugins
|
152
|
+
# One may choose to send them to multiple LIs
|
153
|
+
# Or one may want send a copy to stdout for debugging
|
154
|
+
# Please note, if you use stdout along with LI, catch the logger's log to make
|
155
|
+
# sure they're not cyclic
|
156
|
+
#<store>
|
157
|
+
# @type stdout
|
158
|
+
#</store>
|
108
159
|
</match>
|
160
|
+
|