fluent-plugin-vmware-loginsight 1.0.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/gem-push.yml +10 -10
- data/CHANGELOG.md +1 -1
- data/Dockerfile +68 -0
- data/README.md +2 -6
- data/VERSION +1 -1
- data/fluent-plugin-vmware-loginsight.gemspec +1 -1
- data/lib/fluent/plugin/out_vmware_loginsight.rb +14 -21
- metadata +11 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d8c6cb1363b6364811d6c9d9c1c80067c46ed222eaa2a70034a7f2cd69b2b043
|
4
|
+
data.tar.gz: cad2d46faa95261265dc7c8aa2119ce8a389ca641c8ca789079a34a90d32fb19
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 320f33af67ad7102ea3212f217b1bcfa23c0a27f37f0b4f4873a66fd716ec4556431b0e14005a732541d985b980de1cb44b7fbde5a7d3dac980944593156ea1e
|
7
|
+
data.tar.gz: 1c172ba0c3ea4af9ac2a3e0c28037ff589163dc4b5ff82fc3c70efb7829deca8f0c2c5d4bd9a580bafc104e670e44ad7c322625e60a937ca4b2cc54ebeabd6b7
|
@@ -26,13 +26,13 @@ jobs:
|
|
26
26
|
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
27
27
|
OWNER: ${{ github.repository_owner }}
|
28
28
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
29
|
+
# - name: Publish to RubyGems
|
30
|
+
# run: |
|
31
|
+
# mkdir -p $HOME/.gem
|
32
|
+
# touch $HOME/.gem/credentials
|
33
|
+
# chmod 0600 $HOME/.gem/credentials
|
34
|
+
# printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
35
|
+
# gem build *.gemspec
|
36
|
+
# gem push *.gem
|
37
|
+
# env:
|
38
|
+
# GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
data/CHANGELOG.md
CHANGED
data/Dockerfile
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
# Fluentd plugin for VMware Log Insight
|
2
|
+
#
|
3
|
+
# Copyright 2019 VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# This product is licensed to you under the MIT license (the "License"). You may not use this product except in compliance with the MIT License.
|
6
|
+
#
|
7
|
+
# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file.
|
8
|
+
#
|
9
|
+
# SPDX-License-Identifier: MIT
|
10
|
+
|
11
|
+
# Builds a photon-based image that contains fluentd, fluent-plugin-vmware-loginsight some of the tools recommended by fluent
|
12
|
+
# (libjemalloc, oj, assync-http). This image is based on the minimalistic VMware Photon OS so the result is smaller in size.
|
13
|
+
#
|
14
|
+
# Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
|
15
|
+
# a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
|
16
|
+
# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
|
17
|
+
FROM photon:4.0-20220722
|
18
|
+
|
19
|
+
USER root
|
20
|
+
|
21
|
+
RUN buildDeps="\
|
22
|
+
binutils linux-api-headers glibc-devel \
|
23
|
+
make gcc gmp-devel libffi-devel \
|
24
|
+
tar bzip2 sed gawk" \
|
25
|
+
#
|
26
|
+
# Distro sync and install build dependencies
|
27
|
+
&& tdnf distro-sync --refresh -y \
|
28
|
+
# Toybox conflicts with bzip2. The latter is needed to unpack libjemalloc
|
29
|
+
&& tdnf remove -y toybox \
|
30
|
+
&& tdnf install -y $buildDeps ruby \
|
31
|
+
#
|
32
|
+
# These are not required but are used if available
|
33
|
+
&& gem install oj -v 3.3.10 \
|
34
|
+
&& gem install json -v 2.2.0 \
|
35
|
+
&& gem install async-http -v 0.46.3 \
|
36
|
+
#
|
37
|
+
# Install fluentd
|
38
|
+
&& gem install --norc --no-document fluentd -v 1.14.2 \
|
39
|
+
&& mkdir -p /fluentd/etc /fluentd/plugins \
|
40
|
+
#
|
41
|
+
# Install Log Insight plugin
|
42
|
+
&& gem install --norc --no-document -v 0.1.5 fluent-plugin-vmware-loginsight \
|
43
|
+
#
|
44
|
+
# Install jemalloc 4.5.0
|
45
|
+
&& curl -L --output /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \
|
46
|
+
&& tar -C /tmp/ -xjvf /tmp/jemalloc-4.5.0.tar.bz2 \
|
47
|
+
&& cd /tmp/jemalloc-4.5.0 \
|
48
|
+
&& ./configure && make \
|
49
|
+
&& mv lib/libjemalloc.so.2 /usr/lib \
|
50
|
+
&& cd / \
|
51
|
+
#
|
52
|
+
# Cleanup to reduce image size
|
53
|
+
&& rm -rf /tmp/jemalloc-4.5.0* \
|
54
|
+
&& tdnf remove -y $buildDeps \
|
55
|
+
&& tdnf clean all \
|
56
|
+
&& gem sources --clear-all \
|
57
|
+
&& gem cleanup
|
58
|
+
|
59
|
+
# Create default fluent.conf
|
60
|
+
RUN fluentd --setup
|
61
|
+
|
62
|
+
# Make sure fluentd picks jemalloc
|
63
|
+
ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
|
64
|
+
|
65
|
+
# Standard fluentd ports
|
66
|
+
EXPOSE 24224 5140
|
67
|
+
|
68
|
+
ENTRYPOINT ["/usr/bin/fluentd"]
|
data/README.md
CHANGED
@@ -43,6 +43,8 @@ $ bundle
|
|
43
43
|
enable_stat_watcher false
|
44
44
|
refresh_interval 5
|
45
45
|
tag kubernetes.*
|
46
|
+
# Open below line if you need have filename as tag field (now without prefix kubernetes.)
|
47
|
+
# path_key tag
|
46
48
|
<parse>
|
47
49
|
@type json
|
48
50
|
time_key time
|
@@ -137,12 +139,6 @@ rate_limit_msec, :integer, :default => 0
|
|
137
139
|
# Raise errors that were rescued during HTTP requests?
|
138
140
|
raise_on_error, :bool, :default => false :: Valid Value: true | false
|
139
141
|
|
140
|
-
# Include tag key as log event?
|
141
|
-
include_tag_key, :bool, :default => true :: Valid Value: true | false
|
142
|
-
|
143
|
-
# Metadata key that identifies Fluentd tags
|
144
|
-
tag_key, :string, :default => 'tag'
|
145
|
-
|
146
142
|
# Keys from log event whose values should be added as log message/text to loginsight.
|
147
143
|
# These key/value pairs won't expanded/flattened and won't be added as metadata/fields.
|
148
144
|
log_text_keys, :array, :default => ["log", "message", "msg"] :: Valid Value: Array of strings
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.3.0
|
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.test_files = test_files
|
32
32
|
spec.require_paths = ["lib"]
|
33
33
|
|
34
|
-
spec.add_development_dependency "bundler", "
|
34
|
+
spec.add_development_dependency "bundler", ">= 2.1.0"
|
35
35
|
spec.add_development_dependency "rake", "~> 12.0"
|
36
36
|
spec.add_development_dependency "test-unit", "~> 3.0"
|
37
37
|
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
@@ -49,6 +49,7 @@ module Fluent::Plugin
|
|
49
49
|
config_param :request_retries, :integer, :default => 3
|
50
50
|
config_param :request_timeout, :time, :default => 5
|
51
51
|
config_param :http_conn_debug, :bool, :default => false
|
52
|
+
# in bytes
|
52
53
|
config_param :max_batch_size, :integer, :default => 512000
|
53
54
|
|
54
55
|
# Simple rate limiting: ignore any records within `rate_limit_msec`
|
@@ -56,10 +57,6 @@ module Fluent::Plugin
|
|
56
57
|
config_param :rate_limit_msec, :integer, :default => 0
|
57
58
|
# Raise errors that were rescued during HTTP requests?
|
58
59
|
config_param :raise_on_error, :bool, :default => false
|
59
|
-
### Additional Params
|
60
|
-
config_param :include_tag_key, :bool, :default => true
|
61
|
-
# Metadata key that identifies Fluentd tags
|
62
|
-
config_param :tag_key, :string, :default => 'tag'
|
63
60
|
# Keys from log event whose values should be added as log message/text
|
64
61
|
# to loginsight. Note these key/value pairs won't be added as metadata/fields
|
65
62
|
config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
|
@@ -92,6 +89,12 @@ module Fluent::Plugin
|
|
92
89
|
'container_':''
|
93
90
|
}
|
94
91
|
|
92
|
+
config_section :buffer do
|
93
|
+
config_set_default :@type, "memory"
|
94
|
+
config_set_default :chunk_keys, []
|
95
|
+
config_set_default :timekey_use_utc, true
|
96
|
+
end
|
97
|
+
|
95
98
|
def configure(conf)
|
96
99
|
super
|
97
100
|
|
@@ -134,15 +137,13 @@ module Fluent::Plugin
|
|
134
137
|
key
|
135
138
|
end
|
136
139
|
|
137
|
-
def create_loginsight_event(
|
140
|
+
def create_loginsight_event(time, record)
|
138
141
|
flattened_records = {}
|
139
142
|
if @flatten_hashes
|
140
143
|
flattened_records = flatten_record(record, [])
|
141
144
|
else
|
142
145
|
flattened_records = record
|
143
146
|
end
|
144
|
-
# tag can be immutable in some cases, use a copy.
|
145
|
-
flattened_records[@tag_key] = tag.dup if @include_tag_key
|
146
147
|
fields = []
|
147
148
|
keys = []
|
148
149
|
log = ''
|
@@ -218,15 +219,6 @@ module Fluent::Plugin
|
|
218
219
|
ret
|
219
220
|
end
|
220
221
|
|
221
|
-
def create_request(tag, time, record)
|
222
|
-
url = format_url()
|
223
|
-
uri = URI.parse(url)
|
224
|
-
req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
|
225
|
-
set_body(req, tag, time, record)
|
226
|
-
set_header(req)
|
227
|
-
return req, uri
|
228
|
-
end
|
229
|
-
|
230
222
|
def send_request(req, uri)
|
231
223
|
is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
|
232
224
|
if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
|
@@ -290,13 +282,13 @@ module Fluent::Plugin
|
|
290
282
|
send_request(req, uri)
|
291
283
|
end
|
292
284
|
|
293
|
-
def handle_records(
|
285
|
+
def handle_records(chunk)
|
294
286
|
url = format_url()
|
295
287
|
uri = URI.parse(url)
|
296
288
|
events = []
|
297
289
|
count = 0
|
298
|
-
|
299
|
-
new_event = create_loginsight_event(
|
290
|
+
chunk.each do |time, record|
|
291
|
+
new_event = create_loginsight_event(time, record)
|
300
292
|
new_event_size = new_event.to_json.size
|
301
293
|
if new_event_size > @max_batch_size
|
302
294
|
$log.warn "dropping event larger than max_batch_size: #{new_event.to_json[1..1024]}"
|
@@ -315,8 +307,9 @@ module Fluent::Plugin
|
|
315
307
|
end
|
316
308
|
end
|
317
309
|
|
318
|
-
|
319
|
-
|
310
|
+
# Sync Buffered Output
|
311
|
+
def write(chunk)
|
312
|
+
handle_records(chunk)
|
320
313
|
end
|
321
314
|
end
|
322
315
|
end
|
metadata
CHANGED
@@ -1,30 +1,30 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-vmware-loginsight
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Vishal Mohite
|
8
8
|
- Chris Todd
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-07-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
requirements:
|
18
|
-
- - "
|
18
|
+
- - ">="
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version:
|
20
|
+
version: 2.1.0
|
21
21
|
type: :development
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
|
-
- - "
|
25
|
+
- - ">="
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
27
|
+
version: 2.1.0
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: rake
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
@@ -85,6 +85,7 @@ files:
|
|
85
85
|
- ".gitignore"
|
86
86
|
- CHANGELOG.md
|
87
87
|
- CONTRIBUTING.md
|
88
|
+
- Dockerfile
|
88
89
|
- Gemfile
|
89
90
|
- LICENSE
|
90
91
|
- README.md
|
@@ -103,7 +104,7 @@ homepage: https://github.com/vmware/fluent-plugin-vmware-loginsight
|
|
103
104
|
licenses:
|
104
105
|
- MIT
|
105
106
|
metadata: {}
|
106
|
-
post_install_message:
|
107
|
+
post_install_message:
|
107
108
|
rdoc_options: []
|
108
109
|
require_paths:
|
109
110
|
- lib
|
@@ -118,8 +119,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
118
119
|
- !ruby/object:Gem::Version
|
119
120
|
version: '0'
|
120
121
|
requirements: []
|
121
|
-
rubygems_version: 3.
|
122
|
-
signing_key:
|
122
|
+
rubygems_version: 3.3.7
|
123
|
+
signing_key:
|
123
124
|
specification_version: 4
|
124
125
|
summary: Fluend output plugin to forward logs to VMware Log Insight
|
125
126
|
test_files:
|