fluent-plugin-vmware-loginsight 0.1.5 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +71 -18
  3. data/VERSION +1 -0
  4. data/examples/fluent.conf +104 -52
  5. data/examples/fluentd-vrli-plugin-debian.dockerfile +48 -0
  6. data/examples/fluentd-vrli-plugin-photon-tdnf.dockerfile +45 -0
  7. data/examples/fluentd-vrli-plugin-photon.dockerfile +68 -0
  8. data/examples/k8s-log-collector-ds.yaml +108 -49
  9. data/fluent-plugin-vmware-loginsight.gemspec +1 -1
  10. data/lib/fluent/plugin/out_vmware_loginsight.rb +39 -11
  11. metadata +7 -5
  12. data/examples/Dockerfile +0 -32
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 769cbf3caa38634e432618e399254da2f53d6ba18e8b2def4b9d172aaed29c63
4
- data.tar.gz: ba0ef75193473229d2d1d47de0866160e261dcd19672f5fde1c7bdfd9b423ea1
3
+ metadata.gz: 10d3d5209686fe48c79d1182e193316588f8e29bb6567d9ebe54b0eb061f0e55
4
+ data.tar.gz: 53b75b154b7ca8ff95c8aa71dd0fe91d00deff9460761ac4ea1c13da02c724e7
5
5
  SHA512:
6
- metadata.gz: 3e6a40c4f69297840d6da7cab368a22df065080e0e3706e351686f67ea440d04d8a550d1b4554bac46a25d922be4c40584afb42ffe958c3f48b1cdb18d19822f
7
- data.tar.gz: 1f954795a9d5064904df57469ebe071f94400a39644b14101767542a13bab312a6091ca0348855c8c392be385edf030374e3588c1d012da7208d0c8a6d74b335
6
+ metadata.gz: 2f41fbe184204f8100ebca73447f87d1cdce2640941bdae56d34fff216cbd554a7c22ed1e71f5e39f23bcdb72d8b766897f91917959f29e7346e6deecb0a2f7e
7
+ data.tar.gz: 1ff567aaf08344dc494fdedb6ed2efc21084e55c66da28b33cdbd40a17331c02519e94226959c291e487787a57e12bdad2fc6cff386573fb483b92dd34430e47
data/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # fluent-plugin-vmware-loginsight
2
2
 
3
+ [![Gem Version](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight.svg)](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight)
4
+
3
5
  ## Overview
4
6
  output plugin to do forward logs to VMware Log Insight
5
7
 
@@ -28,39 +30,61 @@ $ bundle
28
30
  ## Usage
29
31
 
30
32
  ```
33
+ # Collect all container logs
31
34
  <source>
32
35
  @type tail
36
+ @id in_tail_container_logs
33
37
  path /var/log/containers/*.log
38
+ # One could exclude certain logs like:
39
+ #exclude_path ["/var/log/containers/log-collector*.log"]
34
40
  pos_file /var/log/fluentd-docker.pos
35
- time_format %Y-%m-%dT%H:%M:%S
36
- tag kubernetes.*
37
- format json
38
41
  read_from_head true
42
+ # Set this watcher to false if you have many files to tail
43
+ enable_stat_watcher false
44
+ refresh_interval 5
45
+ tag kubernetes.*
46
+ <parse>
47
+ @type json
48
+ time_key time
49
+ keep_time_key true
50
+ time_format %Y-%m-%dT%H:%M:%S.%NZ
51
+ </parse>
39
52
  </source>
40
53
 
41
- # Kubernetes metadata filter that tags additional meta data for each event
42
- <filter kubernetes.var.log.containers.**.log>
54
+ # Kubernetes metadata filter that tags additional meta data for each container event
55
+ <filter kubernetes.**>
43
56
  @type kubernetes_metadata
57
+ @id filter_kube_metadata
58
+ kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
59
+ verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
60
+ ca_file "#{ENV['KUBERNETES_CA_FILE']}"
61
+ skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
62
+ skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
63
+ skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
64
+ skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
44
65
  </filter>
45
66
 
67
+ # Match everything
46
68
  <match **>
47
69
  @type vmware_loginsight
70
+ @id out_vmw_li_all_container_logs
48
71
  scheme https
49
72
  ssl_verify true
50
- # Loginsight host: One may use IP address or cname
51
- # host X.X.X.X
52
- host my-loginsight.mycompany.com
53
- port 9000
54
- path api/v1/events/ingest
73
+ # Loginsight host: One may use IP address or cname
74
+ #host X.X.X.X
75
+ host MY_LOGINSIGHT_HOST
76
+ port 9543
55
77
  agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
56
- http_method post
57
- serializer json
58
- rate_limit_msec 0
59
- raise_on_error false
60
- include_tag_key true
61
- tag_key tag
78
+ # Keys from log event whose values should be added as log message/text to
79
+ # Loginsight. Note these key/value pairs won't be added as metadata/fields
80
+ log_text_keys ["log","msg","message"]
81
+ # Use this flag if you want to enable http debug logs
82
+ http_conn_debug false
62
83
  </match>
63
84
  ```
85
+
86
+ For more examples look at [examples](./examples/)
87
+
64
88
  ### Configuration options
65
89
 
66
90
  ```
@@ -104,6 +128,9 @@ request_timeout, :time, :default => 5
104
128
  # If set, enables debug logs for http connection
105
129
  http_conn_debug, :bool, :default => false :: Valid Value: true | false
106
130
 
131
+ # Number of bytes per post request
132
+ max_batch_size, :integer, :default => 512000
133
+
107
134
  # Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
108
135
  rate_limit_msec, :integer, :default => 0
109
136
 
@@ -125,9 +152,35 @@ flatten_hashes, :bool, :default => true :: Valid Value: true | false
125
152
 
126
153
  # Seperator to use for joining flattened keys
127
154
  flatten_hashes_separator, :string, :default => "_"
128
- ```
129
155
 
130
- For more examples look at [examples](./examples/)
156
+ # Rename fields names
157
+ config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
158
+
159
+ # Keys from log event to rewrite
160
+ # for instance from 'kubernetes_namespace' to 'k8s_namespace'
161
+ # tags will be rewritten with substring substitution
162
+ # and applied in the order present in the hash
163
+ # (Hashes enumerate their values in the order that the
164
+ # corresponding keys were inserted
165
+ # see https://ruby-doc.org/core-2.2.2/Hash.html)
166
+ # example config:
167
+ # shorten_keys {
168
+ # "__":"_",
169
+ # "container_":"",
170
+ # "kubernetes_":"k8s_",
171
+ # "labels_":"",
172
+ # }
173
+ shorten_keys, :hash, value_type: :string, default:
174
+ {
175
+ 'kubernetes_':'k8s_',
176
+ 'namespace':'ns',
177
+ 'labels_':'',
178
+ '_name':'',
179
+ '_hash':'',
180
+ 'container_':''
181
+ }
182
+
183
+ ```
131
184
 
132
185
  ## Contributing
133
186
 
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.11
data/examples/fluent.conf CHANGED
@@ -8,18 +8,25 @@
8
8
  #
9
9
  # SPDX-License-Identifier: MIT
10
10
 
11
+ # Sample Fluentd config, edit as per your needs.
12
+ # https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/templates/conf has some good fluentd config examples
11
13
 
14
+ # System level configs
12
15
  <system>
13
16
  log_level info
14
17
  </system>
15
18
 
16
19
  # Prevent fluentd from handling records containing its own logs to handle cycles.
17
- <match fluent.**>
18
- @type null
19
- </match>
20
+ <label @FLUENT_LOG>
21
+ <match fluent.**>
22
+ @type null
23
+ </match>
24
+ </label>
20
25
 
26
+ # Collect all journal logs
21
27
  <source>
22
28
  @type systemd
29
+ @id in_systemd_logs
23
30
  path /run/log/journal
24
31
  # Can filter logs if we want, e.g.
25
32
  #filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
@@ -33,76 +40,121 @@
33
40
  strip_underscores true
34
41
  </source>
35
42
 
43
+ # Collect all container logs
36
44
  <source>
37
45
  @type tail
46
+ @id in_tail_container_logs
38
47
  path /var/log/containers/*.log
39
48
  # One could exclude certain logs like:
40
- # exclude_path ["/var/log/containers/log-collector*.log"]
49
+ #exclude_path ["/var/log/containers/log-collector*.log"]
41
50
  pos_file /var/log/fluentd-docker.pos
42
- time_format %Y-%m-%dT%H:%M:%S
43
- tag kubernetes.*
44
- format json
45
51
  read_from_head true
52
+ # Set this watcher to false if you have many files to tail
53
+ enable_stat_watcher false
54
+ refresh_interval 5
55
+ tag kubernetes.*
56
+ <parse>
57
+ @type json
58
+ time_key time
59
+ keep_time_key true
60
+ time_format %Y-%m-%dT%H:%M:%S.%NZ
61
+ </parse>
46
62
  </source>
47
63
 
48
-
49
- # Sample rule for services that generate java like stack trace
50
- #<source>
51
- # @type tail
52
- # path /var/log/containers/javaapp**.log
53
- # pos_file /var/log/fluentd-dockerlog.pos
54
- # time_format %b %d %H:%M:%S
55
- # tag kubernetes.*
56
- # format multiline
57
- # format_firstline /\d{4}-\d{1,2}-\d{1,2}/
58
- # format1 /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
59
- # read_from_head true
60
- #</source>
61
-
62
- # Kubernetes metadata filter that tags additional meta data for each event
63
- <filter kubernetes.var.log.containers.**.log>
64
+ # Kubernetes metadata filter that tags additional meta data for each container event
65
+ <filter kubernetes.**>
64
66
  @type kubernetes_metadata
67
+ @id filter_kube_metadata
68
+ kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
69
+ verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
70
+ ca_file "#{ENV['KUBERNETES_CA_FILE']}"
71
+ skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
72
+ skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
73
+ skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
74
+ skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
65
75
  </filter>
66
76
 
67
- # If we want to transform events we could use:
68
- #<filter **>
69
- # @type record_transformer
70
- # enable_ruby
71
- # auto_typecast
72
- # <record>
73
- # hostname "#{Socket.gethostname}"
74
- # mykey ${["message"=>record.to_json]}
75
- # </record>
76
- #</filter>
77
+ # Prefix the tag by namespace. This would make it easy to match logs by namespaces
78
+ <match kubernetes.**>
79
+ @type rewrite_tag_filter
80
+ <rule>
81
+ key $.kubernetes.namespace_name
82
+ pattern ^(.+)$
83
+ tag $1.${tag}
84
+ </rule>
85
+ </match>
77
86
 
78
- <match fluent.**>
79
- @type null
87
+ # Collect all kube apiserver audit logs
88
+ <source>
89
+ @type tail
90
+ @id in_tail_kube_audit_logs
91
+ # audit log path of kube-apiserver
92
+ path "/var/log/kube-audit/audit.log"
93
+ pos_file /var/log/kube-audit.pos
94
+ tag kube-audit
95
+ <parse>
96
+ @type json
97
+ time_key timestamp
98
+ keep_time_key false
99
+ time_format %Y-%m-%dT%H:%M:%SZ
100
+ </parse>
101
+ </source>
102
+
103
+ # Loginsight doesn't support ingesting `source` as a field name, get rid of it
104
+ <filter kube-audit>
105
+ @type record_transformer
106
+ @id filter_kube_audit_logs
107
+ enable_ruby
108
+ remove_keys source
109
+ <record>
110
+ log ${record}
111
+ </record>
112
+ </filter>
113
+
114
+ # You can catch and match logs by namespace
115
+ <match my-namespace-one.** my-namespace-two.**>
116
+ @type vmware_loginsight
117
+ @id out_vmw_li_my_namespace_logs
118
+ scheme http
119
+ ssl_verify false
120
+ # Loginsight host: One may use IP address or cname
121
+ #host X.X.X.X
122
+ host MY_LOGINSIGHT_HOST
123
+ port 9000
124
+ agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
125
+ # Keys from log event whose values should be added as log message/text to
126
+ # Loginsight. Note these key/value pairs won't be added as metadata/fields
127
+ log_text_keys ["log","msg","message"]
128
+ # Use this flag if you want to enable http debug logs
129
+ http_conn_debug false
80
130
  </match>
81
131
 
132
+ # Match everything else
82
133
  <match **>
83
134
  @type copy
84
135
  <store>
85
136
  @type vmware_loginsight
137
+ @id out_vmw_li_all_container_logs
86
138
  scheme https
87
139
  ssl_verify true
88
- # Loginsight host: One may use IP address or cname
89
- # host X.X.X.X
90
- host my-loginsight.mycompany.com
91
- port 9000
92
- path api/v1/events/ingest
140
+ # Loginsight host: One may use IP address or cname
141
+ #host X.X.X.X
142
+ host MY_LOGINSIGHT_HOST
143
+ port 9543
93
144
  agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
94
- http_method post
95
- serializer json
96
- rate_limit_msec 0
97
- raise_on_error false
145
+ # Keys from log event whose values should be added as log message/text to
146
+ # Loginsight. Note these key/value pairs won't be added as metadata/fields
98
147
  log_text_keys ["log","msg","message"]
99
- include_tag_key true
100
- tag_key tag
148
+ # Use this flag if you want to enable http debug logs
149
+ http_conn_debug false
101
150
  </store>
102
- # copy plugin supports sending/copying logs to multiple plugins
103
- # One may choose to send them to multiple LIs
104
- # Or one may want send a copy to stdout for debugging
105
- # <store>
106
- # @type stdout
107
- # </store>
151
+ # copy plugin supports sending/copying logs to multiple plugins
152
+ # One may choose to send them to multiple LIs
153
+ # Or one may want send a copy to stdout for debugging
154
+ # Please note, if you use stdout along with LI, catch the logger's log to make
155
+ # sure they're not cyclic
156
+ #<store>
157
+ # @type stdout
158
+ #</store>
108
159
  </match>
160
+
data/examples/fluentd-vrli-plugin-debian.dockerfile ADDED
@@ -0,0 +1,48 @@
1
+ # Fluentd plugin for VMware Log Insight
2
+ #
3
+ # Copyright 2018-2019 VMware, Inc. All Rights Reserved.
4
+ #
5
+ # This product is licensed to you under the MIT license (the "License"). You may not use this product except in compliance with the MIT License.
6
+ #
7
+ # This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file.
8
+ #
9
+ # SPDX-License-Identifier: MIT
10
+
11
+
12
+ # Sample Dockerfile to use as log collector
13
+ # Builds a debian-based fluentd image that has fluent-plugin-kubernetes_metadata_filter,
14
+ # fluent-plugin-rewrite-tag-filter, fluent-plugin-systemd and
15
+ # fluent-plugin-vmware-loginsight gem installed.
16
+ #
17
+ # This image will get preconfigured with the fluent.conf if avaialble at the
18
+ # same dir level. For fluentd config example, see
19
+ # https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
20
+
21
+ # This base image is built from https://github.com/fluent/fluentd-kubernetes-daemonset
22
+ FROM fluent/fluentd:v1.11-debian-1
23
+
24
+ # Use root account to use apt
25
+ USER root
26
+
27
+ # You can install your plugins here
28
+ RUN buildDeps="sudo make gcc g++ libc-dev" \
29
+ && apt-get update \
30
+ && apt-get install -y --no-install-recommends $buildDeps \
31
+ && sudo gem install \
32
+ fluent-plugin-kubernetes_metadata_filter:2.4.6 \
33
+ fluent-plugin-rewrite-tag-filter:2.3.0 \
34
+ fluent-plugin-systemd:1.0.2 \
35
+ fluent-plugin-vmware-loginsight:0.1.10 \
36
+ && sudo gem sources --clear-all \
37
+ && SUDO_FORCE_REMOVE=yes \
38
+ apt-get purge -y --auto-remove \
39
+ -o APT::AutoRemove::RecommendsImportant=false \
40
+ $buildDeps \
41
+ && rm -rf /var/lib/apt/lists/* \
42
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
43
+
44
+ # You can install the LI plugin using a gem or if you want to test your
45
+ # changes to plugin, you may add the .rb directly under `plugins` dir, then
46
+ # you don't need to install the gem
47
+ COPY plugins /fluentd/plugins/
48
+
data/examples/fluentd-vrli-plugin-photon-tdnf.dockerfile ADDED
@@ -0,0 +1,45 @@
1
+ # Fluentd plugin for VMware Log Insight
2
+ #
3
+ # Copyright 2019 VMware, Inc. All Rights Reserved.
4
+ #
5
+ # This product is licensed to you under the MIT license (the "License"). You may not use this product except in compliance with the MIT License.
6
+ #
7
+ # This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file.
8
+ #
9
+ # SPDX-License-Identifier: MIT
10
+
11
+ # Builds a photon-based image that contains fluentd, fluent-plugin-vmware-loginsight some of the tools recommended by fluent
12
+ # (libjemalloc, oj, assync-http). This image is based on the minimalistic VMware Photon OS so the result is smaller in size.
13
+ # Furthermore, all of the needed components are installed from the trusted Photon repository by using the tdnf package manager.
14
+ #
15
+ # Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
16
+ # a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
17
+ # https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
18
+
19
+ FROM photon:3.0-20190705
20
+
21
+ USER root
22
+
23
+ # Distro sync and install components
24
+ RUN tdnf distro-sync --refresh -y \
25
+ && tdnf install -y \
26
+ rubygem-fluentd-1.6.3 \
27
+ #
28
+ # optional but used by fluentd
29
+ rubygem-oj-3.3.10 \
30
+ rubygem-async-http-0.48.2 \
31
+ jemalloc-4.5.0 \
32
+ #
33
+ # Install Log Insight plugin
34
+ rubygem-fluent-plugin-vmware-loginsight-0.1.5
35
+
36
+ RUN ln -s /usr/lib/ruby/gems/2.5.0/bin/fluentd /usr/bin/fluentd \
37
+ && fluentd --setup
38
+
39
+ # Make sure fluentd picks jemalloc
40
+ ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
41
+
42
+ # Standard fluentd ports
43
+ EXPOSE 24224 5140
44
+
45
+ ENTRYPOINT ["/usr/bin/fluentd"]
data/examples/fluentd-vrli-plugin-photon.dockerfile ADDED
@@ -0,0 +1,68 @@
1
+ # Fluentd plugin for VMware Log Insight
2
+ #
3
+ # Copyright 2019 VMware, Inc. All Rights Reserved.
4
+ #
5
+ # This product is licensed to you under the MIT license (the "License"). You may not use this product except in compliance with the MIT License.
6
+ #
7
+ # This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file.
8
+ #
9
+ # SPDX-License-Identifier: MIT
10
+
11
+ # Builds a photon-based image that contains fluentd, fluent-plugin-vmware-loginsight some of the tools recommended by fluent
12
+ # (libjemalloc, oj, assync-http). This image is based on the minimalistic VMware Photon OS so the result is smaller in size.
13
+ #
14
+ # Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
15
+ # a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
16
+ # https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
17
+ FROM photon:3.0-20190705
18
+
19
+ USER root
20
+
21
+ RUN buildDeps="\
22
+ binutils linux-api-headers glibc-devel \
23
+ make gcc gmp-devel libffi-devel \
24
+ tar bzip2 sed gawk" \
25
+ #
26
+ # Distro sync and install build dependencies
27
+ && tdnf distro-sync --refresh -y \
28
+ # Toybox conflicts with bzip2. The latter is needed to unpack libjemalloc
29
+ && tdnf remove -y toybox \
30
+ && tdnf install -y $buildDeps ruby \
31
+ #
32
+ # These are not required but are used if available
33
+ && gem install oj -v 3.3.10 \
34
+ && gem install json -v 2.2.0 \
35
+ && gem install async-http -v 0.46.3 \
36
+ #
37
+ # Install fluentd
38
+ && gem install --norc --no-document fluentd -v 1.6.3 \
39
+ && mkdir -p /fluentd/etc /fluentd/plugins \
40
+ #
41
+ # Install Log Insight plugin
42
+ && gem install --norc --no-document -v 0.1.5 fluent-plugin-vmware-loginsight \
43
+ #
44
+ # Install jemalloc 4.5.0
45
+ && curl -L --output /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \
46
+ && tar -C /tmp/ -xjvf /tmp/jemalloc-4.5.0.tar.bz2 \
47
+ && cd /tmp/jemalloc-4.5.0 \
48
+ && ./configure && make \
49
+ && mv lib/libjemalloc.so.2 /usr/lib \
50
+ && cd / \
51
+ #
52
+ # Cleanup to reduce image size
53
+ && rm -rf /tmp/jemalloc-4.5.0* \
54
+ && tdnf remove -y $buildDeps \
55
+ && tdnf clean all \
56
+ && gem sources --clear-all \
57
+ && gem cleanup
58
+
59
+ # Create default fluent.conf
60
+ RUN fluentd --setup
61
+
62
+ # Make sure fluentd picks jemalloc
63
+ ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
64
+
65
+ # Standard fluentd ports
66
+ EXPOSE 24224 5140
67
+
68
+ ENTRYPOINT ["/usr/bin/fluentd"]
data/examples/k8s-log-collector-ds.yaml CHANGED
@@ -23,30 +23,32 @@ data:
23
23
  myapp-fluent.conf: |
24
24
  # Input sources
25
25
  @include general.conf
26
- @include systemd-input.conf
27
- @include kubernetes-input.conf
28
-
29
- # Parsing/Filtering
30
- @include kubernetes-filter.conf
26
+ @include systemd.conf
27
+ @include kubernetes.conf
28
+ @include kube-audit.conf
31
29
 
32
30
  # Forwading - Be vigilant of the order in which these plugins are specified. Order matters!
33
- @include myapp-loginsight-output.conf
31
+ @include vmw-li.conf
34
32
 
35
33
  general.conf: |
36
34
  <system>
37
35
  log_level info
38
36
  </system>
39
37
  # Prevent fluentd from handling records containing its own logs to handle cycles.
40
- <match fluent.**>
41
- @type null
42
- </match>
38
+ <label @FLUENT_LOG>
39
+ <match fluent.**>
40
+ @type null
41
+ </match>
42
+ </label>
43
43
 
44
- systemd-input.conf: |
44
+ systemd.conf: |
45
+ # Journal logs
45
46
  <source>
46
47
  @type systemd
48
+ @id in_systemd_logs
47
49
  path /run/log/journal
48
50
  # Can filter logs if we want, e.g.
49
- # filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
51
+ #filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
50
52
  <storage>
51
53
  @type local
52
54
  persistent true
@@ -57,70 +59,114 @@ data:
57
59
  strip_underscores true
58
60
  </source>
59
61
 
60
- kubernetes-input.conf: |
62
+ kubernetes.conf: |
63
+ # Container logs
64
+ # Kubernetes docker logs are stored under /var/lib/docker/containers for
65
+ # which kubernetes creates a symlink at /var/log/containers
61
66
  <source>
62
67
  @type tail
68
+ @id in_tail_container_logs
63
69
  path /var/log/containers/*.log
64
70
  # One could exclude certain logs like:
65
- # exclude_path ["/var/log/containers/log-collector*.log"]
71
+ #exclude_path ["/var/log/containers/log-collector*.log"]
66
72
  pos_file /var/log/fluentd-docker.pos
67
- time_format %Y-%m-%dT%H:%M:%S
68
- tag kubernetes.*
69
- format json
70
73
  read_from_head true
74
+ # Set this watcher to false if you have many files to tail
75
+ enable_stat_watcher false
76
+ refresh_interval 5
77
+ tag kubernetes.*
78
+ <parse>
79
+ @type json
80
+ time_key time
81
+ keep_time_key true
82
+ time_format %Y-%m-%dT%H:%M:%S.%NZ
83
+ </parse>
71
84
  </source>
72
-
73
- kubernetes-filter.conf: |
85
+ # Kubernetes metadata filter that tags additional meta data for each container event
74
86
  <filter kubernetes.**>
75
87
  @type kubernetes_metadata
76
- merge_json_log true
77
- preserve_json_log true
88
+ @id filter_kube_metadata
89
+ kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV. fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
90
+ verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
91
+ ca_file "#{ENV['KUBERNETES_CA_FILE']}"
92
+ skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
93
+ skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
94
+ skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
95
+ skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
96
+ </filter>
97
+
98
+ # Prefix the tag by namespace. This would make it easy to match logs by namespaces
99
+ <match kubernetes.**>
100
+ @type rewrite_tag_filter
101
+ <rule>
102
+ key $.kubernetes.namespace_name
103
+ pattern ^(.+)$
104
+ tag $1.${tag}
105
+ </rule>
106
+ </match>
107
+
108
+ kube-audit.conf: |
109
+ # Kube-apiserver audit logs
110
+ <source>
111
+ @type tail
112
+ @id in_tail_kube_audit_logs
113
+ # path to audit logs for kube-apiserver
114
+ path "/var/log/kube-audit/audit.log"
115
+ pos_file /var/log/kube-audit.pos
116
+ tag kube-audit
117
+ <parse>
118
+ @type json
119
+ time_key timestamp
120
+ keep_time_key false
121
+ time_format %Y-%m-%dT%H:%M:%SZ
122
+ </parse>
123
+ </source>
124
+ # Loginsight doesn't support ingesting `source` as a field name, get rid of it
125
+ <filter kube-audit>
126
+ @type record_transformer
127
+ @id filter_kube_audit_logs
128
+ enable_ruby
129
+ remove_keys source
130
+ <record>
131
+ log ${record}
132
+ </record>
78
133
  </filter>
79
134
 
80
- myapp-loginsight-output.conf: |
135
+ vmw-li.conf: |
136
+ # Match everything
81
137
  # We are capturing all log messages and redirecting them to endpoints mentioned in each <store> tag.
82
138
  # One may redirect these logs to muliple endpoints (including multiple LI instances).
83
139
  # Or one may chose to tag their specific logs and add their own config to capture those tagged logs and redirect
84
- # them to appropriate endpoint. This specific config needs to preceed this generic one.
140
+ # them to appropriate endpoint. That specific config needs to preceed this generic one.
85
141
  <match **>
86
142
  @type copy
87
143
  <store>
88
144
  @type vmware_loginsight
145
+ @id out_vmw_li_all_container_logs
89
146
  scheme https
90
147
  ssl_verify true
91
148
  # Loginsight host: One may use IP address or cname
92
- # host X.X.X.X
93
- host my-loginsight.mycompany.com
94
- port 9000
95
- path api/v1/events/ingest
149
+ #host X.X.X.X
150
+ host MY_LOGINSIGHT_HOST
151
+ port 9543
96
152
  agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
97
- http_method post
98
- serializer json
99
- rate_limit_msec 0
100
- raise_on_error false
101
- include_tag_key true
102
- tag_key tag
153
+ # Keys from log event whose values should be added as log message/text to
154
+ # Loginsight. Note these key/value pairs won't be added as metadata/fields
155
+ log_text_keys ["log","msg","message"]
156
+ # Use this flag if you want to enable http debug logs
157
+ http_conn_debug false
103
158
  </store>
104
- # If we want to debug and send logs to stdout as well
105
- # <store>
106
- # @type stdout
107
- # </store>
159
+ # copy plugin supports sending/copying logs to multiple plugins
160
+ # One may choose to send them to multiple LIs
161
+ # Or one may want send a copy to stdout for debugging
162
+ # Please note, if you use stdout along with LI, catch the logger's log to make
163
+ # sure they're not cyclic
164
+ #<store>
165
+ # @type stdout
166
+ #</store>
108
167
  </match>
109
168
 
110
169
 
111
- extra.conf: |
112
- # If we want to transform events we could use:
113
- #<filter **>
114
- # @type record_transformer
115
- # enable_ruby
116
- # auto_typecast
117
- # <record>
118
- # hostname "#{Socket.gethostname}"
119
- # mykey ${["message"=>record.to_json]}
120
- # </record>
121
- #</filter>
122
-
123
-
124
170
  ---
125
171
  kind: DaemonSet
126
172
  apiVersion: extensions/v1beta1
@@ -131,8 +177,21 @@ metadata:
131
177
  app: "log-collector"
132
178
  version: v1
133
179
  spec:
180
+ selector:
181
+ matchLabels:
182
+ app: "log-collector"
183
+ revisionHistoryLimit: 3
184
+ minReadySeconds: 10
185
+ updateStrategy:
186
+ type: RollingUpdate
187
+ rollingUpdate:
188
+ # How many pods can be unavailable during the rolling update.
189
+ maxUnavailable: 3
134
190
  template:
135
191
  metadata:
192
+ annotations:
193
+ # One may use this annotation to trigger rollout whenever fluentd config changes
194
+ configHash: GENERATED_HASH
136
195
  labels:
137
196
  app: "log-collector"
138
197
  version: v1
data/fluent-plugin-vmware-loginsight.gemspec CHANGED
@@ -14,7 +14,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
14
14
 
15
15
  Gem::Specification.new do |spec|
16
16
  spec.name = "fluent-plugin-vmware-loginsight"
17
- spec.version = "0.1.5"
17
+ spec.version = File.read("VERSION").strip
18
18
  spec.authors = ["Vishal Mohite", "Chris Todd"]
19
19
  spec.email = ["vmohite@vmware.com", "toddc@vmware.com"]
20
20
 
data/lib/fluent/plugin/out_vmware_loginsight.rb CHANGED
@@ -70,7 +70,32 @@ module Fluent
70
70
  config_param :flatten_hashes, :bool, :default => true
71
71
  # Seperator to use for joining flattened keys
72
72
  config_param :flatten_hashes_separator, :string, :default => "_"
73
-
73
+ # Rename fields names
74
+ config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
75
+
76
+ # Keys from log event to rewrite
77
+ # for instance from 'kubernetes_namespace' to 'k8s_namespace'
78
+ # tags will be rewritten with substring substitution
79
+ # and applied in the order present in the hash
80
+ # (Hashes enumerate their values in the order that the
81
+ # corresponding keys were inserted
82
+ # see https://ruby-doc.org/core-2.2.2/Hash.html)
83
+ # example config:
84
+ # shorten_keys {
85
+ # "__":"_",
86
+ # "container_":"",
87
+ # "kubernetes_":"k8s_",
88
+ # "labels_":"",
89
+ # }
90
+ config_param :shorten_keys, :hash, value_type: :string, default:
91
+ {
92
+ 'kubernetes_':'k8s_',
93
+ 'namespace':'ns',
94
+ 'labels_':'',
95
+ '_name':'',
96
+ '_hash':'',
97
+ 'container_':''
98
+ }
74
99
 
75
100
  def initialize
76
101
  super
@@ -118,14 +143,11 @@ module Fluent
118
143
  def shorten_key(key)
119
144
  # LI doesn't allow some characters in field 'name'
120
145
  # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
121
- key = key.gsub(/[\/\.\-\\]/,@flatten_hashes_separator).downcase
122
- # shorten field names
123
- key = key.gsub(/kubernetes_/,'k8s_')
124
- key = key.gsub(/namespace/,'ns')
125
- key = key.gsub(/labels_/,'')
126
- key = key.gsub(/_name/,'')
127
- key = key.gsub(/_hash/,'')
128
- key = key.gsub(/container_/,'')
146
+ key = key.gsub(/[\/\.\-\\\@]/,@flatten_hashes_separator).downcase
147
+ # shorten field names using provided shorten_keys parameters
148
+ @shorten_keys.each do | match, replace |
149
+ key = key.gsub(match.to_s,replace)
150
+ end
129
151
  key
130
152
  end
131
153
 
@@ -136,13 +158,18 @@ module Fluent
136
158
  else
137
159
  flattened_records = record
138
160
  end
139
- flattened_records[@tag_key] = tag if @include_tag_key
161
+ # tag can be immutable in some cases, use a copy.
162
+ flattened_records[@tag_key] = tag.dup if @include_tag_key
140
163
  fields = []
141
164
  keys = []
142
165
  log = ''
143
166
  flattened_records.each do |key, value|
144
167
  begin
145
168
  next if value.nil?
169
+ # check if name of the key should be replaced
170
+ if @rename_fields.has_key?(key)
171
+ key = @rename_fields[key]
172
+ end
146
173
  # LI doesn't support duplicate fields, make unique names by appending underscore
147
174
  key = shorten_key(key)
148
175
  while keys.include?(key)
@@ -154,6 +181,7 @@ module Fluent
154
181
  begin
155
182
  value = value.to_json if value.is_a?(Hash)
156
183
  value = value.to_s
184
+ value = value.frozen? ? value.dup : value # if value is immutable, use a copy.
157
185
  value.force_encoding("utf-8")
158
186
  rescue Exception=>e
159
187
  $log.warn "force_encoding exception: " "#{e.class}, '#{e.message}', " \
@@ -228,7 +256,7 @@ module Fluent
228
256
  return
229
257
  end
230
258
 
231
- if @auth and @auth == 'basic'
259
+ if @auth and @auth.to_s.eql? "basic"
232
260
  req.basic_auth(@username, @password)
233
261
  end
234
262
  begin
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-vmware-loginsight
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.1.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Vishal Mohite
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-10-22 00:00:00.000000000 Z
12
+ date: 2021-03-31 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -87,8 +87,11 @@ files:
87
87
  - LICENSE
88
88
  - README.md
89
89
  - Rakefile
90
- - examples/Dockerfile
90
+ - VERSION
91
91
  - examples/fluent.conf
92
+ - examples/fluentd-vrli-plugin-debian.dockerfile
93
+ - examples/fluentd-vrli-plugin-photon-tdnf.dockerfile
94
+ - examples/fluentd-vrli-plugin-photon.dockerfile
92
95
  - examples/k8s-log-collector-ds.yaml
93
96
  - fluent-plugin-vmware-loginsight.gemspec
94
97
  - lib/fluent/plugin/out_vmware_loginsight.rb
@@ -113,8 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
113
116
  - !ruby/object:Gem::Version
114
117
  version: '0'
115
118
  requirements: []
116
- rubyforge_project:
117
- rubygems_version: 2.7.7
119
+ rubygems_version: 3.0.3
118
120
  signing_key:
119
121
  specification_version: 4
120
122
  summary: Fluend output plugin to forward logs to VMware Log Insight
data/examples/Dockerfile DELETED
@@ -1,32 +0,0 @@
1
- # Fluentd plugin for VMware Log Insight
2
- #
3
- # Copyright 2018 VMware, Inc. All Rights Reserved.
4
- #
5
- # This product is licensed to you under the MIT license (the "License"). You may not use this product except in compliance with the MIT License.
6
- #
7
- # This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file.
8
- #
9
- # SPDX-License-Identifier: MIT
10
-
11
-
12
- FROM fluent/fluentd:v0.14.15-debian-onbuild
13
- # Above image expects the loginsight plugin vmware_loginsight to be available under ./plugins/vmware_loginsight.rb
14
- # and fluentd config under ./fluent.conf by default
15
-
16
- USER root
17
-
18
- RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \
19
- && apt-get update \
20
- && apt-get install -y --no-install-recommends $buildDeps \
21
- && sudo gem install \
22
- fluent-plugin-systemd \
23
- fluent-plugin-kubernetes_metadata_filter \
24
- fluent-plugin-vmware-loginsight \
25
- && sudo gem sources --clear-all \
26
- && SUDO_FORCE_REMOVE=yes \
27
- apt-get purge -y --auto-remove \
28
- -o APT::AutoRemove::RecommendsImportant=false \
29
- $buildDeps \
30
- && rm -rf /var/lib/apt/lists/* \
31
- /home/fluent/.gem/ruby/2.3.0/cache/*.gem \
32
- /home/root/.gem/ruby/2.3.0/cache/*.gem