fluent-plugin-vmware-loginsight 0.1.11 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/gem-push.yml +38 -0
- data/CHANGELOG.md +63 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_vmware_loginsight.rb +273 -298
- metadata +8 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f7e7d17226c807f4fffdd8723c9f260730af5de29334a1b10271993b21709185
|
4
|
+
data.tar.gz: 768c2724b2b84ac8ae8c1e4473dbe850c37a87102aaed95d6c52fc4f06f9a0df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 912e7dfa34b63dc9da92635603e88fea94d340ce8fb9aef99827bfd6896690a89533fe523416c2eafd34da2470016b13d23c290c74d93dc05fdb320beec78b55
|
7
|
+
data.tar.gz: 5998ef3e2869d13cd30dc934347b3dad6ebe2337de5678a7fb12f641b15a5b0a5a60daaa8bd9b5490ecdf58e5fb5f3529dd313ea2606126aaae8456dad023ffb
|
@@ -0,0 +1,38 @@
|
|
1
|
+
name: Ruby Gem
|
2
|
+
|
3
|
+
on: workflow_dispatch
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
name: Build + Publish
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
|
10
|
+
steps:
|
11
|
+
- uses: actions/checkout@v2
|
12
|
+
- name: Set up Ruby 2.6
|
13
|
+
uses: actions/setup-ruby@v1
|
14
|
+
with:
|
15
|
+
ruby-version: 2.6.x
|
16
|
+
|
17
|
+
- name: Publish to GPR
|
18
|
+
run: |
|
19
|
+
mkdir -p $HOME/.gem
|
20
|
+
touch $HOME/.gem/credentials
|
21
|
+
chmod 0600 $HOME/.gem/credentials
|
22
|
+
printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
23
|
+
gem build *.gemspec
|
24
|
+
gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
|
25
|
+
env:
|
26
|
+
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
27
|
+
OWNER: ${{ github.repository_owner }}
|
28
|
+
|
29
|
+
- name: Publish to RubyGems
|
30
|
+
run: |
|
31
|
+
mkdir -p $HOME/.gem
|
32
|
+
touch $HOME/.gem/credentials
|
33
|
+
chmod 0600 $HOME/.gem/credentials
|
34
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
35
|
+
gem build *.gemspec
|
36
|
+
gem push *.gem
|
37
|
+
env:
|
38
|
+
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
# Changelog
|
2
|
+
|
3
|
+
## v1.0.0 - April 19, 2021
|
4
|
+
|
5
|
+
* Update plugin structure to use Fluentd 1.x syntax
|
6
|
+
|
7
|
+
## v0.1.11 - March 31, 2021
|
8
|
+
|
9
|
+
* Add an option to rename Loginsight fields. This option could be used to rename certain fields that are reserved by Loginsight
|
10
|
+
|
11
|
+
## v0.1.10 - May 13, 2020
|
12
|
+
|
13
|
+
* Escape `@` char from Loginsight field
|
14
|
+
|
15
|
+
## v0.1.9 - May 07, 2020
|
16
|
+
|
17
|
+
* No change
|
18
|
+
|
19
|
+
## v0.1.8 - May 06, 2020 yanked, Not available
|
20
|
+
|
21
|
+
* Parameterize and add an option to shorten Loginsight field names
|
22
|
+
|
23
|
+
## v0.1.7 - December 10, 2019
|
24
|
+
|
25
|
+
* Fix basic authentication #8
|
26
|
+
|
27
|
+
## v0.1.6 - September 13, 2019
|
28
|
+
|
29
|
+
* For immutable log fields, use a copy to utf encode. This should fix 'can't modify frozen String' error in #5
|
30
|
+
|
31
|
+
## v0.1.5 - October 22, 2018
|
32
|
+
|
33
|
+
* Add option to display debug logs for http connection, default false
|
34
|
+
* Flatten Lists/Arrays for LI fields
|
35
|
+
* Convert LI field value to String to ensure no utf encoding errors
|
36
|
+
* Update help doc/examples with sample use of @log_text_keys and @http_conn_debug options
|
37
|
+
|
38
|
+
## v0.1.4 - October 17, 2018
|
39
|
+
|
40
|
+
* Add option to specify a list of keys that plugin should treat as log messages and forward them as text to Loginsight. Plugin should not flatten these fields
|
41
|
+
* If user specifies flatten_hashes option as false, plugin should try to add record key/values as is
|
42
|
+
|
43
|
+
## v0.1.3 - September 13, 2018
|
44
|
+
|
45
|
+
* Reorder namespace and name fields to be shorten
|
46
|
+
|
47
|
+
## v0.1.2 - September 10, 2018
|
48
|
+
|
49
|
+
* Republished yanked gem
|
50
|
+
|
51
|
+
## v0.1.1 - August 30, 2018 yanked, Not available
|
52
|
+
|
53
|
+
* Send log messages in batches, add max_batch_size parameter
|
54
|
+
* Shorten common kubernetes Loginsight field names
|
55
|
+
* Convert time to milliseconds
|
56
|
+
|
57
|
+
|
58
|
+
## 0.1.0 - August 30, 2018
|
59
|
+
|
60
|
+
### Initial release
|
61
|
+
|
62
|
+
* Fluentd output plugin to push logs to VMware Log Insight
|
63
|
+
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
1.0.0
|
@@ -9,339 +9,314 @@
|
|
9
9
|
# SPDX-License-Identifier: MIT
|
10
10
|
|
11
11
|
|
12
|
-
require
|
12
|
+
require 'fluent/plugin/output'
|
13
13
|
require 'json'
|
14
14
|
require 'net/http'
|
15
15
|
require 'uri'
|
16
16
|
|
17
|
-
module Fluent
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
super
|
106
|
-
|
107
|
-
@ssl_verify_mode = @ssl_verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
108
|
-
@auth = case @authentication
|
109
|
-
when 'basic'
|
110
|
-
:basic
|
111
|
-
else
|
112
|
-
:none
|
113
|
-
end
|
114
|
-
|
115
|
-
@last_request_time = nil
|
116
|
-
end
|
17
|
+
module Fluent::Plugin
|
18
|
+
class VmwareLoginsightOutput < Output
|
19
|
+
Fluent::Plugin.register_output('vmware_loginsight', self)
|
20
|
+
|
21
|
+
### Connection Params ###
|
22
|
+
config_param :scheme, :string, :default => 'http'
|
23
|
+
# Loginsight Host ex. localhost
|
24
|
+
config_param :host, :string, :default => 'localhost'
|
25
|
+
# In case we want to post to multiple hosts. This is futuristic, Fluentd copy plugin can support this as is
|
26
|
+
#config_param :hosts, :string, :default => nil
|
27
|
+
# Loginsight port ex. 9000. Default 80
|
28
|
+
config_param :port, :integer, :default => 80
|
29
|
+
# Loginsight ingestion api path ex. 'api/v1/events/ingest'
|
30
|
+
config_param :path, :string, :default => 'api/v1/events/ingest'
|
31
|
+
# agent_id generated by your LI
|
32
|
+
config_param :agent_id, :string, :default => '0'
|
33
|
+
# Credentials if used
|
34
|
+
config_param :username, :string, :default => nil
|
35
|
+
config_param :password, :string, :default => nil, :secret => true
|
36
|
+
# Authentication nil | 'basic'
|
37
|
+
config_param :authentication, :string, :default => nil
|
38
|
+
|
39
|
+
# Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
|
40
|
+
config_param :ssl_verify, :bool, :default => true
|
41
|
+
config_param :ca_file, :string, :default => nil
|
42
|
+
|
43
|
+
### API Params ###
|
44
|
+
# HTTP method
|
45
|
+
# post | put
|
46
|
+
config_param :http_method, :string, :default => :post
|
47
|
+
# form | json
|
48
|
+
config_param :serializer, :string, :default => :json
|
49
|
+
config_param :request_retries, :integer, :default => 3
|
50
|
+
config_param :request_timeout, :time, :default => 5
|
51
|
+
config_param :http_conn_debug, :bool, :default => false
|
52
|
+
config_param :max_batch_size, :integer, :default => 512000
|
53
|
+
|
54
|
+
# Simple rate limiting: ignore any records within `rate_limit_msec`
|
55
|
+
# since the last one.
|
56
|
+
config_param :rate_limit_msec, :integer, :default => 0
|
57
|
+
# Raise errors that were rescued during HTTP requests?
|
58
|
+
config_param :raise_on_error, :bool, :default => false
|
59
|
+
### Additional Params
|
60
|
+
config_param :include_tag_key, :bool, :default => true
|
61
|
+
# Metadata key that identifies Fluentd tags
|
62
|
+
config_param :tag_key, :string, :default => 'tag'
|
63
|
+
# Keys from log event whose values should be added as log message/text
|
64
|
+
# to loginsight. Note these key/value pairs won't be added as metadata/fields
|
65
|
+
config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
|
66
|
+
# Flatten hashes to create one key/val pair w/o losing log data
|
67
|
+
config_param :flatten_hashes, :bool, :default => true
|
68
|
+
# Seperator to use for joining flattened keys
|
69
|
+
config_param :flatten_hashes_separator, :string, :default => "_"
|
70
|
+
|
71
|
+
# Keys from log event to rewrite
|
72
|
+
# for instance from 'kubernetes_namespace' to 'k8s_namespace'
|
73
|
+
# tags will be rewritten with substring substitution
|
74
|
+
# and applied in the order present in the hash
|
75
|
+
# (Hashes enumerate their values in the order that the
|
76
|
+
# corresponding keys were inserted
|
77
|
+
# see https://ruby-doc.org/core-2.2.2/Hash.html)
|
78
|
+
# example config:
|
79
|
+
# shorten_keys {
|
80
|
+
# "__":"_",
|
81
|
+
# "container_":"",
|
82
|
+
# "kubernetes_":"k8s_",
|
83
|
+
# "labels_":"",
|
84
|
+
# }
|
85
|
+
config_param :shorten_keys, :hash, value_type: :string, default:
|
86
|
+
{
|
87
|
+
'kubernetes_':'k8s_',
|
88
|
+
'namespace':'ns',
|
89
|
+
'labels_':'',
|
90
|
+
'_name':'',
|
91
|
+
'_hash':'',
|
92
|
+
'container_':''
|
93
|
+
}
|
94
|
+
|
95
|
+
def configure(conf)
|
96
|
+
super
|
97
|
+
|
98
|
+
@ssl_verify_mode = @ssl_verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
99
|
+
@auth = case @authentication
|
100
|
+
when 'basic'
|
101
|
+
:basic
|
102
|
+
else
|
103
|
+
:none
|
104
|
+
end
|
117
105
|
|
118
|
-
|
119
|
-
|
120
|
-
end
|
106
|
+
@last_request_time = nil
|
107
|
+
end
|
121
108
|
|
122
|
-
|
123
|
-
|
124
|
-
|
109
|
+
def format_url()
|
110
|
+
url = "#{@scheme}://#{host}:#{port}/#{path}/#{agent_id}"
|
111
|
+
url
|
112
|
+
end
|
125
113
|
|
126
|
-
|
127
|
-
|
128
|
-
|
114
|
+
def set_header(req)
|
115
|
+
if @serializer == 'json'
|
116
|
+
set_json_header(req)
|
129
117
|
end
|
118
|
+
req
|
119
|
+
end
|
130
120
|
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
req
|
136
|
-
end
|
121
|
+
def set_json_header(req)
|
122
|
+
req['Content-Type'] = 'application/json'
|
123
|
+
req
|
124
|
+
end
|
137
125
|
|
138
|
-
|
139
|
-
|
140
|
-
|
126
|
+
def shorten_key(key)
|
127
|
+
# LI doesn't allow some characters in field 'name'
|
128
|
+
# like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
|
129
|
+
key = key.gsub(/[\/\.\-\\\@]/,@flatten_hashes_separator).downcase
|
130
|
+
# shorten field names using provided shorten_keys parameters
|
131
|
+
@shorten_keys.each do | match, replace |
|
132
|
+
key = key.gsub(match.to_s,replace)
|
141
133
|
end
|
134
|
+
key
|
135
|
+
end
|
142
136
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
key = key.gsub(match.to_s,replace)
|
150
|
-
end
|
151
|
-
key
|
137
|
+
def create_loginsight_event(tag, time, record)
|
138
|
+
flattened_records = {}
|
139
|
+
if @flatten_hashes
|
140
|
+
flattened_records = flatten_record(record, [])
|
141
|
+
else
|
142
|
+
flattened_records = record
|
152
143
|
end
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
144
|
+
# tag can be immutable in some cases, use a copy.
|
145
|
+
flattened_records[@tag_key] = tag.dup if @include_tag_key
|
146
|
+
fields = []
|
147
|
+
keys = []
|
148
|
+
log = ''
|
149
|
+
flattened_records.each do |key, value|
|
150
|
+
begin
|
151
|
+
next if value.nil?
|
152
|
+
# LI doesn't support duplicate fields, make unique names by appending underscore
|
153
|
+
key = shorten_key(key)
|
154
|
+
while keys.include?(key)
|
155
|
+
key = key + '_'
|
156
|
+
end
|
157
|
+
keys.push(key)
|
158
|
+
key.force_encoding("utf-8")
|
159
|
+
# convert value to json string if its a hash and to string if not already a string
|
167
160
|
begin
|
168
|
-
|
169
|
-
|
170
|
-
if
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
key = key + '_'
|
177
|
-
end
|
178
|
-
keys.push(key)
|
179
|
-
key.force_encoding("utf-8")
|
180
|
-
# convert value to json string if its a hash and to string if not already a string
|
181
|
-
begin
|
182
|
-
value = value.to_json if value.is_a?(Hash)
|
183
|
-
value = value.to_s
|
184
|
-
value = value.frozen? ? value.dup : value # if value is immutable, use a copy.
|
185
|
-
value.force_encoding("utf-8")
|
186
|
-
rescue Exception=>e
|
187
|
-
$log.warn "force_encoding exception: " "#{e.class}, '#{e.message}', " \
|
188
|
-
"\n Request: #{key} #{record.to_json[1..1024]}"
|
189
|
-
value = "Exception during conversion: #{e.message}"
|
190
|
-
end
|
161
|
+
value = value.to_json if value.is_a?(Hash)
|
162
|
+
value = value.to_s
|
163
|
+
value = value.frozen? ? value.dup : value # if value is immutable, use a copy.
|
164
|
+
value.force_encoding("utf-8")
|
165
|
+
rescue Exception=>e
|
166
|
+
$log.warn "force_encoding exception: " "#{e.class}, '#{e.message}', " \
|
167
|
+
"\n Request: #{key} #{record.to_json[1..1024]}"
|
168
|
+
value = "Exception during conversion: #{e.message}"
|
191
169
|
end
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
170
|
+
end
|
171
|
+
if @log_text_keys.include?(key)
|
172
|
+
if log != "#{value}"
|
173
|
+
if log.empty?
|
174
|
+
log = "#{value}"
|
175
|
+
else
|
176
|
+
log += " #{value}"
|
199
177
|
end
|
200
|
-
else
|
201
|
-
# If there is time information available, update time for LI. LI ignores
|
202
|
-
# time if it is out of the error/adjusment window of 10 mins. in such
|
203
|
-
# cases we would still like to preserve time info, so add it as event.
|
204
|
-
# TODO Ignore the below block for now. Handle the case for time being in
|
205
|
-
# different formats than milliseconds
|
206
|
-
#if ['time', '_source_realtime_timestamp'].include?(key)
|
207
|
-
# time = value
|
208
|
-
#end
|
209
|
-
fields << {"name" => key, "content" => value}
|
210
178
|
end
|
179
|
+
else
|
180
|
+
# If there is time information available, update time for LI. LI ignores
|
181
|
+
# time if it is out of the error/adjusment window of 10 mins. in such
|
182
|
+
# cases we would still like to preserve time info, so add it as event.
|
183
|
+
# TODO Ignore the below block for now. Handle the case for time being in
|
184
|
+
# different formats than milliseconds
|
185
|
+
#if ['time', '_source_realtime_timestamp'].include?(key)
|
186
|
+
# time = value
|
187
|
+
#end
|
188
|
+
fields << {"name" => key, "content" => value}
|
211
189
|
end
|
212
|
-
event = {
|
213
|
-
"fields" => fields,
|
214
|
-
"text" => log.gsub(/^$\n/, ''),
|
215
|
-
"timestamp" => time * 1000
|
216
|
-
}
|
217
|
-
event
|
218
190
|
end
|
191
|
+
event = {
|
192
|
+
"fields" => fields,
|
193
|
+
"text" => log.gsub(/^$\n/, ''),
|
194
|
+
"timestamp" => time * 1000
|
195
|
+
}
|
196
|
+
event
|
197
|
+
end
|
219
198
|
|
220
|
-
|
221
|
-
|
199
|
+
def flatten_record(record, prefix=[])
|
200
|
+
ret = {}
|
222
201
|
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
end
|
231
|
-
end
|
232
|
-
when Array
|
233
|
-
record.each do |value|
|
234
|
-
ret.merge! flatten_record(value, prefix)
|
202
|
+
case record
|
203
|
+
when Hash
|
204
|
+
record.each do |key, value|
|
205
|
+
if @log_text_keys.include?(key)
|
206
|
+
ret.merge!({key.to_s => value})
|
207
|
+
else
|
208
|
+
ret.merge! flatten_record(value, prefix + [key.to_s])
|
235
209
|
end
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
210
|
+
end
|
211
|
+
when Array
|
212
|
+
record.each do |value|
|
213
|
+
ret.merge! flatten_record(value, prefix)
|
214
|
+
end
|
215
|
+
else
|
216
|
+
return {prefix.join(@flatten_hashes_separator) => record}
|
240
217
|
end
|
218
|
+
ret
|
219
|
+
end
|
241
220
|
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
221
|
+
def create_request(tag, time, record)
|
222
|
+
url = format_url()
|
223
|
+
uri = URI.parse(url)
|
224
|
+
req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
|
225
|
+
set_body(req, tag, time, record)
|
226
|
+
set_header(req)
|
227
|
+
return req, uri
|
228
|
+
end
|
250
229
|
|
230
|
+
def send_request(req, uri)
|
231
|
+
is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
|
232
|
+
if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
|
233
|
+
$log.info('Dropped request due to rate limiting')
|
234
|
+
return
|
235
|
+
end
|
251
236
|
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
237
|
+
if @auth and @auth.to_s.eql? "basic"
|
238
|
+
req.basic_auth(@username, @password)
|
239
|
+
end
|
240
|
+
begin
|
241
|
+
retries ||= 2
|
242
|
+
response = nil
|
243
|
+
@last_request_time = Time.now.to_f
|
244
|
+
|
245
|
+
http_conn = Net::HTTP.new(uri.host, uri.port)
|
246
|
+
# For debugging, set this
|
247
|
+
http_conn.set_debug_output($stdout) if @http_conn_debug
|
248
|
+
http_conn.use_ssl = (uri.scheme == 'https')
|
249
|
+
if http_conn.use_ssl?
|
250
|
+
http_conn.ca_file = @ca_file
|
257
251
|
end
|
252
|
+
http_conn.verify_mode = @ssl_verify_mode
|
258
253
|
|
259
|
-
|
260
|
-
|
254
|
+
response = http_conn.start do |http|
|
255
|
+
http.read_timeout = @request_timeout
|
256
|
+
http.request(req)
|
261
257
|
end
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
258
|
+
rescue => e # rescue all StandardErrors
|
259
|
+
# server didn't respond
|
260
|
+
# Be careful while turning on below log, if LI instance can't be reached and you're sending
|
261
|
+
# log-container logs to LI as well, you may end up in a cycle.
|
262
|
+
# TODO handle the cyclic case at plugin level if possible.
|
263
|
+
# $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: " \
|
264
|
+
# "#{e.class}, '#{e.message}', \n Request: #{req.body[1..1024]}"
|
265
|
+
retry unless (retries -= 1).zero?
|
266
|
+
raise e if @raise_on_error
|
267
|
+
else
|
268
|
+
unless response and response.is_a?(Net::HTTPSuccess)
|
269
|
+
res_summary = if response
|
270
|
+
"Response Code: #{response.code}\n"\
|
271
|
+
"Response Message: #{response.message}\n" \
|
272
|
+
"Response Body: #{response.body}"
|
273
|
+
else
|
274
|
+
"Response = nil"
|
275
|
+
end
|
276
|
+
# ditto cyclic warning
|
277
|
+
# $log.warn "Failed to #{req.method} #{uri}\n(#{res_summary})\n" \
|
278
|
+
# "Request Size: #{req.body.size} Request Body: #{req.body[1..1024]}"
|
279
|
+
end #end unless
|
280
|
+
end # end begin
|
281
|
+
end # end send_request
|
282
|
+
|
283
|
+
def send_events(uri, events)
|
284
|
+
req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
|
285
|
+
event_req = {
|
286
|
+
"events" => events
|
287
|
+
}
|
288
|
+
req.body = event_req.to_json
|
289
|
+
set_header(req)
|
290
|
+
send_request(req, uri)
|
291
|
+
end
|
275
292
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
# "#{e.class}, '#{e.message}', \n Request: #{req.body[1..1024]}"
|
287
|
-
retry unless (retries -= 1).zero?
|
288
|
-
raise e if @raise_on_error
|
293
|
+
def handle_records(tag, es)
|
294
|
+
url = format_url()
|
295
|
+
uri = URI.parse(url)
|
296
|
+
events = []
|
297
|
+
count = 0
|
298
|
+
es.each do |time, record|
|
299
|
+
new_event = create_loginsight_event(tag, time, record)
|
300
|
+
new_event_size = new_event.to_json.size
|
301
|
+
if new_event_size > @max_batch_size
|
302
|
+
$log.warn "dropping event larger than max_batch_size: #{new_event.to_json[1..1024]}"
|
289
303
|
else
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
"Response Body: #{response.body}"
|
295
|
-
else
|
296
|
-
"Response = nil"
|
297
|
-
end
|
298
|
-
# ditto cyclic warning
|
299
|
-
# $log.warn "Failed to #{req.method} #{uri}\n(#{res_summary})\n" \
|
300
|
-
# "Request Size: #{req.body.size} Request Body: #{req.body[1..1024]}"
|
301
|
-
end #end unless
|
302
|
-
end # end begin
|
303
|
-
end # end send_request
|
304
|
-
|
305
|
-
def send_events(uri, events)
|
306
|
-
req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
|
307
|
-
event_req = {
|
308
|
-
"events" => events
|
309
|
-
}
|
310
|
-
req.body = event_req.to_json
|
311
|
-
set_header(req)
|
312
|
-
send_request(req, uri)
|
313
|
-
end
|
314
|
-
|
315
|
-
def handle_records(tag, es)
|
316
|
-
url = format_url()
|
317
|
-
uri = URI.parse(url)
|
318
|
-
events = []
|
319
|
-
count = 0
|
320
|
-
es.each do |time, record|
|
321
|
-
new_event = create_loginsight_event(tag, time, record)
|
322
|
-
new_event_size = new_event.to_json.size
|
323
|
-
if new_event_size > @max_batch_size
|
324
|
-
$log.warn "dropping event larger than max_batch_size: #{new_event.to_json[1..1024]}"
|
325
|
-
else
|
326
|
-
if (count + new_event_size) > @max_batch_size
|
327
|
-
send_events(uri, events)
|
328
|
-
events = []
|
329
|
-
count = 0
|
330
|
-
end
|
331
|
-
count += new_event_size
|
332
|
-
events << new_event
|
304
|
+
if (count + new_event_size) > @max_batch_size
|
305
|
+
send_events(uri, events)
|
306
|
+
events = []
|
307
|
+
count = 0
|
333
308
|
end
|
334
|
-
|
335
|
-
|
336
|
-
send_events(uri, events)
|
309
|
+
count += new_event_size
|
310
|
+
events << new_event
|
337
311
|
end
|
338
312
|
end
|
339
|
-
|
340
|
-
|
341
|
-
handle_records(tag, es)
|
342
|
-
chain.next
|
313
|
+
if count > 0
|
314
|
+
send_events(uri, events)
|
343
315
|
end
|
344
316
|
end
|
317
|
+
|
318
|
+
def process(tag, es)
|
319
|
+
handle_records(tag, es)
|
320
|
+
end
|
345
321
|
end
|
346
322
|
end
|
347
|
-
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-vmware-loginsight
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Vishal Mohite
|
8
8
|
- Chris Todd
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-
|
12
|
+
date: 2021-04-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -81,7 +81,9 @@ executables: []
|
|
81
81
|
extensions: []
|
82
82
|
extra_rdoc_files: []
|
83
83
|
files:
|
84
|
+
- ".github/workflows/gem-push.yml"
|
84
85
|
- ".gitignore"
|
86
|
+
- CHANGELOG.md
|
85
87
|
- CONTRIBUTING.md
|
86
88
|
- Gemfile
|
87
89
|
- LICENSE
|
@@ -101,7 +103,7 @@ homepage: https://github.com/vmware/fluent-plugin-vmware-loginsight
|
|
101
103
|
licenses:
|
102
104
|
- MIT
|
103
105
|
metadata: {}
|
104
|
-
post_install_message:
|
106
|
+
post_install_message:
|
105
107
|
rdoc_options: []
|
106
108
|
require_paths:
|
107
109
|
- lib
|
@@ -116,8 +118,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
116
118
|
- !ruby/object:Gem::Version
|
117
119
|
version: '0'
|
118
120
|
requirements: []
|
119
|
-
rubygems_version: 3.
|
120
|
-
signing_key:
|
121
|
+
rubygems_version: 3.2.3
|
122
|
+
signing_key:
|
121
123
|
specification_version: 4
|
122
124
|
summary: Fluend output plugin to forward logs to VMware Log Insight
|
123
125
|
test_files:
|