fluent-plugin-k8s-metrics-agg 1.1.4 → 1.1.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci_build_test.yaml +192 -0
- data/Gemfile.lock +39 -36
- data/Makefile +6 -7
- data/README.md +0 -1
- data/VERSION +1 -1
- data/ci_scripts/deploy_connector.sh +29 -0
- data/{.circleci → ci_scripts}/install_dep.sh +0 -0
- data/ci_scripts/k8s-splunk.yml +31 -0
- data/docker/Dockerfile +22 -6
- data/docker/Gemfile +5 -4
- data/docker/Gemfile.lock +65 -57
- data/fluent-plugin-k8s-metrics-agg.gemspec +1 -1
- data/lib/fluent/plugin/in_kubernetes_metrics_aggregator.rb +78 -60
- metadata +9 -8
- data/.circleci/config.yml +0 -88
- data/.circleci/trigger_func_test.sh +0 -42
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 98a2dd49ad2df50f48934daa1f41f92d9466a0c87149b74a059755e2a12b55a1
|
4
|
+
data.tar.gz: 4549d0845d04a933c89a6599a18b15ef83e468c0e739c5b9f66458b0c5f8116f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c1acc729827a62b91a8cf8fa33f230d651eaf733e66d395ba4610ff2b55407013e9c3fdb8b4db6d8913a4e6320fbe37f04b910b62de3c894216c347bacb86be5
|
7
|
+
data.tar.gz: 79c4d760559ef7edb0789133aa7d9c7e1a89cfd28b3ac71df146dd755082338b1c480a9b95940fc15f54b1984e22cf81f72e6c48f4b7e8c93354a1aaed9985b6
|
@@ -0,0 +1,192 @@
|
|
1
|
+
name: CI Build Test
|
2
|
+
|
3
|
+
on:
|
4
|
+
pull_request:
|
5
|
+
branches-ignore:
|
6
|
+
- /^release\/.*/
|
7
|
+
- main
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
build:
|
11
|
+
runs-on: ubuntu-20.04
|
12
|
+
steps:
|
13
|
+
- name: Checkout code
|
14
|
+
uses: actions/checkout@v2
|
15
|
+
|
16
|
+
- name: Setup Ruby and install gems
|
17
|
+
uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
bundler-cache: true
|
20
|
+
ruby-version: 2.6.1
|
21
|
+
|
22
|
+
- name: Install dependencies
|
23
|
+
run: |
|
24
|
+
sudo ci_scripts/install_dep.sh
|
25
|
+
|
26
|
+
- name: Builder
|
27
|
+
run: |
|
28
|
+
rake build -t -v
|
29
|
+
cp -R pkg /tmp
|
30
|
+
|
31
|
+
- name: Cache pkg
|
32
|
+
uses: actions/cache@v1
|
33
|
+
with:
|
34
|
+
path: /tmp
|
35
|
+
key: ${{ runner.os }}-build
|
36
|
+
|
37
|
+
unit-test:
|
38
|
+
runs-on: ubuntu-20.04
|
39
|
+
needs:
|
40
|
+
- build
|
41
|
+
steps:
|
42
|
+
- name: Checkout
|
43
|
+
uses: actions/checkout@v2
|
44
|
+
|
45
|
+
- name: Install dependencies
|
46
|
+
run: |
|
47
|
+
sudo ci_scripts/install_dep.sh
|
48
|
+
|
49
|
+
- uses: actions/cache@v2
|
50
|
+
with:
|
51
|
+
path: /tmp
|
52
|
+
key: ${{ runner.os }}-build
|
53
|
+
|
54
|
+
- name: Run unit tests
|
55
|
+
run: |
|
56
|
+
bundle exec rake test -t -v
|
57
|
+
|
58
|
+
func-test:
|
59
|
+
needs:
|
60
|
+
- unit-test
|
61
|
+
runs-on: ubuntu-20.04
|
62
|
+
env:
|
63
|
+
CI_SPLUNK_PORT: 8089
|
64
|
+
CI_SPLUNK_USERNAME: admin
|
65
|
+
CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
|
66
|
+
CI_SPLUNK_PASSWORD: helloworld
|
67
|
+
CI_INDEX_EVENTS: ci_events
|
68
|
+
CI_INDEX_OBJECTS: ci_objects
|
69
|
+
CI_INDEX_METRICS: ci_metrics
|
70
|
+
KUBERNETES_VERSION: v1.15.2
|
71
|
+
MINIKUBE_VERSION: v1.21.0
|
72
|
+
GITHUB_ACTIONS: true
|
73
|
+
|
74
|
+
steps:
|
75
|
+
- name: Checkout
|
76
|
+
uses: actions/checkout@v2
|
77
|
+
|
78
|
+
- name: Prepare container build
|
79
|
+
id: prep
|
80
|
+
run: |
|
81
|
+
VERSION=`cat VERSION`
|
82
|
+
TAGS=splunk/k8s-metrics-aggr:recent
|
83
|
+
echo ::set-output name=tags::${TAGS}
|
84
|
+
echo ::set-output name=version::${VERSION}
|
85
|
+
|
86
|
+
- name: Set up QEMU
|
87
|
+
uses: docker/setup-qemu-action@master
|
88
|
+
with:
|
89
|
+
platforms: all
|
90
|
+
|
91
|
+
- name: Set up Docker Buildx
|
92
|
+
id: buildx
|
93
|
+
uses: docker/setup-buildx-action@master
|
94
|
+
|
95
|
+
- name: Build multi-arch kubernetes-metrics image
|
96
|
+
uses: docker/build-push-action@v2
|
97
|
+
with:
|
98
|
+
builder: ${{ steps.buildx.outputs.name }}
|
99
|
+
context: .
|
100
|
+
file: ./docker/Dockerfile
|
101
|
+
platforms: linux/amd64
|
102
|
+
push: false
|
103
|
+
load: true
|
104
|
+
tags: ${{ steps.prep.outputs.tags }}
|
105
|
+
build-args: VERSION=${{ steps.prep.outputs.version }}
|
106
|
+
|
107
|
+
- name: Build k8s-metrics-aggr image
|
108
|
+
run: |
|
109
|
+
docker image ls
|
110
|
+
|
111
|
+
- name: Setup Minikube
|
112
|
+
run: |
|
113
|
+
# Install Kubectl
|
114
|
+
curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
|
115
|
+
chmod +x kubectl
|
116
|
+
sudo mv kubectl /usr/local/bin/
|
117
|
+
mkdir -p ${HOME}/.kube
|
118
|
+
touch ${HOME}/.kube/config
|
119
|
+
# Install Minikube
|
120
|
+
curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
|
121
|
+
chmod +x minikube
|
122
|
+
sudo mv minikube /usr/local/bin/
|
123
|
+
# Start Minikube and Wait
|
124
|
+
minikube start --driver=docker --container-runtime=docker --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check
|
125
|
+
export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
|
126
|
+
until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
|
127
|
+
sleep 1;
|
128
|
+
done
|
129
|
+
|
130
|
+
- name: Install Splunk
|
131
|
+
run: |
|
132
|
+
# Wait until minikube is ready
|
133
|
+
kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml
|
134
|
+
export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
|
135
|
+
until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
|
136
|
+
echo "wait for minikube ready ..."
|
137
|
+
sleep 1;
|
138
|
+
done
|
139
|
+
kubectl get nodes
|
140
|
+
# Install Splunk on minikube
|
141
|
+
kubectl apply -f ci_scripts/k8s-splunk.yml
|
142
|
+
# Wait until splunk is ready
|
143
|
+
until kubectl logs splunk --tail=2 | grep -q 'Ansible playbook complete'; do
|
144
|
+
sleep 1;
|
145
|
+
done
|
146
|
+
export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
|
147
|
+
# Setup Indexes
|
148
|
+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_EVENTS -d datatype=event
|
149
|
+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_OBJECTS -d datatype=event
|
150
|
+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_METRICS -d datatype=metric
|
151
|
+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=default-events -d datatype=event
|
152
|
+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=ns-anno -d datatype=event
|
153
|
+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=pod-anno -d datatype=event
|
154
|
+
# Enable HEC services
|
155
|
+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
|
156
|
+
# Create new HEC token
|
157
|
+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=default-events&indexes=default-events,$CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,ns-anno,pod-anno" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
|
158
|
+
# Restart Splunk
|
159
|
+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST
|
160
|
+
|
161
|
+
- name: Deploy k8s connector
|
162
|
+
run: |
|
163
|
+
export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
|
164
|
+
ci_scripts/deploy_connector.sh
|
165
|
+
|
166
|
+
- name: Deploy log generator
|
167
|
+
run: |
|
168
|
+
cd /opt/splunk-connect-for-kubernetes
|
169
|
+
kubectl apply -f test/test_setup.yaml
|
170
|
+
sleep 65
|
171
|
+
|
172
|
+
- uses: actions/setup-python@v2
|
173
|
+
with:
|
174
|
+
python-version: 3.7
|
175
|
+
|
176
|
+
- name: Run functional tests
|
177
|
+
run: |
|
178
|
+
echo "check the pods"
|
179
|
+
kubectl get pods -A
|
180
|
+
cd /opt/splunk-connect-for-kubernetes
|
181
|
+
kubectl get nodes
|
182
|
+
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
|
183
|
+
export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
|
184
|
+
cd test
|
185
|
+
pip install --upgrade pip
|
186
|
+
pip install -r requirements.txt
|
187
|
+
echo "Running functional tests....."
|
188
|
+
python -m pytest \
|
189
|
+
--splunkd-url https://$CI_SPLUNK_HOST:8089 \
|
190
|
+
--splunk-user admin \
|
191
|
+
--splunk-password $CI_SPLUNK_PASSWORD \
|
192
|
+
-p no:warnings -s
|
data/Gemfile.lock
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-k8s-metrics-agg (1.1.
|
5
|
-
fluentd (
|
4
|
+
fluent-plugin-k8s-metrics-agg (1.1.8)
|
5
|
+
fluentd (>= 1.9.1)
|
6
6
|
kubeclient (~> 4.6.0)
|
7
7
|
multi_json (~> 1.14.1)
|
8
8
|
oj (~> 3.10.2)
|
@@ -10,65 +10,67 @@ PATH
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
12
12
|
specs:
|
13
|
-
addressable (2.
|
14
|
-
public_suffix (>= 2.0.2, <
|
15
|
-
concurrent-ruby (1.1.
|
16
|
-
cool.io (1.
|
17
|
-
crack (0.4.
|
18
|
-
|
19
|
-
docile (1.
|
13
|
+
addressable (2.8.0)
|
14
|
+
public_suffix (>= 2.0.2, < 5.0)
|
15
|
+
concurrent-ruby (1.1.9)
|
16
|
+
cool.io (1.7.1)
|
17
|
+
crack (0.4.5)
|
18
|
+
rexml
|
19
|
+
docile (1.4.0)
|
20
20
|
domain_name (0.5.20190701)
|
21
21
|
unf (>= 0.0.5, < 1.0.0)
|
22
|
-
ffi (1.
|
22
|
+
ffi (1.15.4)
|
23
23
|
ffi-compiler (1.0.1)
|
24
24
|
ffi (>= 1.0.0)
|
25
25
|
rake
|
26
|
-
fluentd (1.
|
26
|
+
fluentd (1.14.2)
|
27
|
+
bundler
|
27
28
|
cool.io (>= 1.4.5, < 2.0.0)
|
28
|
-
http_parser.rb (>= 0.5.1, < 0.
|
29
|
+
http_parser.rb (>= 0.5.1, < 0.8.0)
|
29
30
|
msgpack (>= 1.3.1, < 2.0.0)
|
30
|
-
serverengine (>= 2.
|
31
|
+
serverengine (>= 2.2.2, < 3.0.0)
|
31
32
|
sigdump (~> 0.2.2)
|
32
|
-
strptime (>= 0.2.
|
33
|
+
strptime (>= 0.2.4, < 1.0.0)
|
33
34
|
tzinfo (>= 1.0, < 3.0)
|
34
35
|
tzinfo-data (~> 1.0)
|
36
|
+
webrick (>= 1.4.2, < 1.8.0)
|
35
37
|
yajl-ruby (~> 1.0)
|
36
|
-
hashdiff (0.
|
37
|
-
http (4.
|
38
|
+
hashdiff (1.0.1)
|
39
|
+
http (4.4.1)
|
38
40
|
addressable (~> 2.3)
|
39
41
|
http-cookie (~> 1.0)
|
40
42
|
http-form_data (~> 2.2)
|
41
43
|
http-parser (~> 1.2.0)
|
42
44
|
http-accept (1.7.0)
|
43
|
-
http-cookie (1.0.
|
45
|
+
http-cookie (1.0.4)
|
44
46
|
domain_name (~> 0.5)
|
45
|
-
http-form_data (2.
|
46
|
-
http-parser (1.2.
|
47
|
+
http-form_data (2.3.0)
|
48
|
+
http-parser (1.2.3)
|
47
49
|
ffi-compiler (>= 1.0, < 2.0)
|
48
|
-
http_parser.rb (0.
|
49
|
-
json (2.
|
50
|
+
http_parser.rb (0.7.0)
|
51
|
+
json (2.6.1)
|
50
52
|
kubeclient (4.6.0)
|
51
53
|
http (>= 3.0, < 5.0)
|
52
54
|
recursive-open-struct (~> 1.0, >= 1.0.4)
|
53
55
|
rest-client (~> 2.0)
|
54
56
|
mime-types (3.3.1)
|
55
57
|
mime-types-data (~> 3.2015)
|
56
|
-
mime-types-data (3.
|
57
|
-
msgpack (1.
|
58
|
+
mime-types-data (3.2021.0901)
|
59
|
+
msgpack (1.4.2)
|
58
60
|
multi_json (1.14.1)
|
59
61
|
netrc (0.11.0)
|
60
|
-
oj (3.10.
|
61
|
-
power_assert (
|
62
|
-
public_suffix (
|
63
|
-
rake (13.0.
|
64
|
-
recursive-open-struct (1.1.
|
62
|
+
oj (3.10.18)
|
63
|
+
power_assert (2.0.1)
|
64
|
+
public_suffix (4.0.6)
|
65
|
+
rake (13.0.6)
|
66
|
+
recursive-open-struct (1.1.3)
|
65
67
|
rest-client (2.1.0)
|
66
68
|
http-accept (>= 1.7.0, < 2.0)
|
67
69
|
http-cookie (>= 1.0.2, < 2.0)
|
68
70
|
mime-types (>= 1.16, < 4.0)
|
69
71
|
netrc (~> 0.8)
|
70
|
-
|
71
|
-
serverengine (2.2.
|
72
|
+
rexml (3.2.5)
|
73
|
+
serverengine (2.2.4)
|
72
74
|
sigdump (~> 0.2.2)
|
73
75
|
sigdump (0.2.4)
|
74
76
|
simplecov (0.16.1)
|
@@ -76,20 +78,21 @@ GEM
|
|
76
78
|
json (>= 1.8, < 3)
|
77
79
|
simplecov-html (~> 0.10.0)
|
78
80
|
simplecov-html (0.10.2)
|
79
|
-
strptime (0.2.
|
80
|
-
test-unit (3.3.
|
81
|
+
strptime (0.2.5)
|
82
|
+
test-unit (3.3.9)
|
81
83
|
power_assert
|
82
|
-
tzinfo (2.0.
|
84
|
+
tzinfo (2.0.4)
|
83
85
|
concurrent-ruby (~> 1.0)
|
84
|
-
tzinfo-data (1.
|
86
|
+
tzinfo-data (1.2021.5)
|
85
87
|
tzinfo (>= 1.0.0)
|
86
88
|
unf (0.1.4)
|
87
89
|
unf_ext
|
88
|
-
unf_ext (0.0.
|
90
|
+
unf_ext (0.0.8)
|
89
91
|
webmock (3.5.1)
|
90
92
|
addressable (>= 2.3.6)
|
91
93
|
crack (>= 0.3.2)
|
92
94
|
hashdiff
|
95
|
+
webrick (1.7.0)
|
93
96
|
yajl-ruby (1.4.1)
|
94
97
|
|
95
98
|
PLATFORMS
|
@@ -104,4 +107,4 @@ DEPENDENCIES
|
|
104
107
|
webmock (~> 3.5.1)
|
105
108
|
|
106
109
|
BUNDLED WITH
|
107
|
-
2.
|
110
|
+
2.2.33
|
data/Makefile
CHANGED
@@ -4,7 +4,7 @@ clean_pkg:
|
|
4
4
|
@rm -rf pkg/* docker/*.gem
|
5
5
|
|
6
6
|
clean_gems:
|
7
|
-
@rm -rf docker/gem
|
7
|
+
@rm -rf docker/gem/* docker/gems/*
|
8
8
|
|
9
9
|
clean: clean_pkg clean_gems
|
10
10
|
@rm -rf docker/licenses
|
@@ -12,11 +12,9 @@ clean: clean_pkg clean_gems
|
|
12
12
|
build: clean_pkg
|
13
13
|
@bundle exec rake build
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
@
|
18
|
-
@cp -rp LICENSE docker/licenses/
|
19
|
-
@docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/k8s-metrics-aggr:$(VERSION) ./docker
|
15
|
+
.PHONY: docker
|
16
|
+
docker:
|
17
|
+
@docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/k8s-metrics-aggr:$(VERSION) . -f docker/Dockerfile
|
20
18
|
|
21
19
|
unit-test:
|
22
20
|
@bundle exec rake test
|
@@ -28,6 +26,7 @@ install-deps:
|
|
28
26
|
|
29
27
|
unpack: build
|
30
28
|
@cp pkg/fluent-plugin-*.gem docker
|
31
|
-
@
|
29
|
+
@mkdir -p docker/gem
|
30
|
+
@rm -rf docker/gem
|
32
31
|
@gem unpack docker/fluent-plugin-*.gem --target docker/gem
|
33
32
|
@cd docker && bundle install
|
data/README.md
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/fluent-plugin-k8s-metrics-agg)
|
2
1
|
# Fluentd Plugin for Kubernetes Metrics - Aggregator
|
3
2
|
|
4
3
|
[Fluentd](https://fluentd.org/) input plugin collects kubernetes cluster metrics from the kubeapiserver API. The API is exposed by [KubeApiServer](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) on a kubernetes cluster.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.1.
|
1
|
+
1.1.8
|
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
set -e
|
3
|
+
|
4
|
+
#Make sure to check and clean previously failed deployment
|
5
|
+
echo "Checking if previous deployment exist..."
|
6
|
+
if [ "`helm ls --short`" == "" ]; then
|
7
|
+
echo "Nothing to clean, ready for deployment"
|
8
|
+
else
|
9
|
+
helm delete $(helm ls --short)
|
10
|
+
fi
|
11
|
+
|
12
|
+
# Clone splunk-connect-for-kubernetes repo
|
13
|
+
cd /opt
|
14
|
+
git clone https://github.com/splunk/splunk-connect-for-kubernetes.git
|
15
|
+
cd splunk-connect-for-kubernetes
|
16
|
+
|
17
|
+
minikube image load splunk/k8s-metrics-aggr:recent
|
18
|
+
|
19
|
+
echo "Deploying k8s-connect with latest changes"
|
20
|
+
helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \
|
21
|
+
--set global.splunk.hec.host=$CI_SPLUNK_HOST \
|
22
|
+
--set kubelet.serviceMonitor.https=true \
|
23
|
+
--set splunk-kubernetes-metrics.imageAgg.tag=recent \
|
24
|
+
--set splunk-kubernetes-metrics.imageAgg.pullPolicy=IfNotPresent \
|
25
|
+
-f ci_scripts/sck_values.yml helm-chart/splunk-connect-for-kubernetes
|
26
|
+
#wait for deployment to finish
|
27
|
+
until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do
|
28
|
+
sleep 1;
|
29
|
+
done
|
File without changes
|
@@ -0,0 +1,31 @@
|
|
1
|
+
apiVersion: v1
|
2
|
+
kind: Pod
|
3
|
+
metadata:
|
4
|
+
name: splunk
|
5
|
+
spec:
|
6
|
+
hostNetwork: true
|
7
|
+
securityContext:
|
8
|
+
runAsUser: 0
|
9
|
+
runAsGroup: 0
|
10
|
+
containers:
|
11
|
+
- name: splunk
|
12
|
+
image: splunk/splunk:latest
|
13
|
+
ports:
|
14
|
+
- containerPort: 8000
|
15
|
+
hostPort: 8000
|
16
|
+
protocol: TCP
|
17
|
+
- containerPort: 8088
|
18
|
+
hostPort: 8088
|
19
|
+
protocol: TCP
|
20
|
+
- containerPort: 8089
|
21
|
+
hostPort: 8089
|
22
|
+
protocol: TCP
|
23
|
+
env:
|
24
|
+
- name: SPLUNK_START_ARGS
|
25
|
+
value: --accept-license
|
26
|
+
- name: SPLUNK_USER
|
27
|
+
value: root
|
28
|
+
- name: SPLUNK_PASSWORD
|
29
|
+
value: helloworld
|
30
|
+
- name: SPLUNK_LAUNCH_CONF
|
31
|
+
value: OPTIMISTIC_ABOUT_FILE_LOCKING=1
|
data/docker/Dockerfile
CHANGED
@@ -1,4 +1,14 @@
|
|
1
|
-
FROM
|
1
|
+
FROM ruby:2.7.4-buster as builder
|
2
|
+
|
3
|
+
ADD ./ /app/
|
4
|
+
WORKDIR /app
|
5
|
+
RUN gem install bundler
|
6
|
+
RUN bundle update --bundler
|
7
|
+
RUN bundle install
|
8
|
+
|
9
|
+
RUN bundle exec rake build -t -v
|
10
|
+
|
11
|
+
FROM registry.access.redhat.com/ubi8/ruby-27
|
2
12
|
|
3
13
|
ARG VERSION
|
4
14
|
|
@@ -14,14 +24,20 @@ ENV VERSION=${VERSION}
|
|
14
24
|
ENV FLUENT_USER fluent
|
15
25
|
|
16
26
|
USER root
|
17
|
-
COPY licenses /licenses
|
18
27
|
|
19
|
-
COPY
|
28
|
+
COPY --from=builder /app/pkg/fluent-plugin-*.gem /tmp/
|
29
|
+
|
30
|
+
RUN mkdir /licenses
|
31
|
+
COPY --from=builder /app/LICENSE /licenses/LICENSE
|
32
|
+
|
33
|
+
COPY --from=builder /app/docker/Gemfile* ./
|
20
34
|
|
21
|
-
|
22
|
-
|
35
|
+
RUN yum update -y \
|
36
|
+
&& yum remove -y nodejs npm \
|
37
|
+
&& gem install bundler \
|
23
38
|
&& gem unpack /tmp/*.gem --target gem \
|
24
|
-
&& bundle install
|
39
|
+
&& bundle install \
|
40
|
+
&& rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* || true
|
25
41
|
|
26
42
|
RUN groupadd -r $FLUENT_USER && \
|
27
43
|
useradd -r -g $FLUENT_USER $FLUENT_USER && \
|
data/docker/Gemfile
CHANGED
@@ -3,15 +3,16 @@ source 'https://rubygems.org'
|
|
3
3
|
# This is separate gemfile for building docker image that has all plugins
|
4
4
|
# for kubernetes log collection agent
|
5
5
|
# List all required gems here and install via bundler to resolve dependencies
|
6
|
-
gem "fluentd", "
|
7
|
-
gem "fluent-plugin-prometheus", "
|
6
|
+
gem "fluentd", ">=1.14.2"
|
7
|
+
gem "fluent-plugin-prometheus", ">=2.0"
|
8
8
|
gem "fluent-plugin-record-modifier", "=2.1.0"
|
9
|
+
gem "fluent-plugin-kubernetes_metadata_filter", ">=2.9.2"
|
9
10
|
gem "kubeclient", "=4.6.0"
|
10
11
|
gem "oj", "=3.10.2"
|
11
12
|
gem 'multi_json', '=1.14.1'
|
12
13
|
gem 'http_parser.rb', '=0.5.3'
|
13
|
-
gem 'bigdecimal', '=
|
14
|
+
gem 'bigdecimal', '=3.0.0'
|
14
15
|
|
15
|
-
gem "fluent-plugin-splunk-hec", ">= 1.2.
|
16
|
+
gem "fluent-plugin-splunk-hec", ">= 1.2.7"
|
16
17
|
|
17
18
|
gem 'fluent-plugin-k8s-metrics-agg', path: 'gem/'
|
data/docker/Gemfile.lock
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: gem
|
3
3
|
specs:
|
4
|
-
fluent-plugin-k8s-metrics-agg (1.1.
|
5
|
-
fluentd (
|
4
|
+
fluent-plugin-k8s-metrics-agg (1.1.8)
|
5
|
+
fluentd (>= 1.9.1)
|
6
6
|
kubeclient (~> 4.6.0)
|
7
7
|
multi_json (~> 1.14.1)
|
8
8
|
oj (~> 3.10.2)
|
@@ -10,65 +10,72 @@ PATH
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
12
12
|
specs:
|
13
|
-
activemodel (
|
14
|
-
activesupport (=
|
15
|
-
activesupport (
|
13
|
+
activemodel (6.1.4.3)
|
14
|
+
activesupport (= 6.1.4.3)
|
15
|
+
activesupport (6.1.4.3)
|
16
16
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
17
|
-
i18n (>=
|
18
|
-
minitest (
|
19
|
-
tzinfo (~>
|
20
|
-
|
17
|
+
i18n (>= 1.6, < 2)
|
18
|
+
minitest (>= 5.1)
|
19
|
+
tzinfo (~> 2.0)
|
20
|
+
zeitwerk (~> 2.3)
|
21
|
+
addressable (2.8.0)
|
21
22
|
public_suffix (>= 2.0.2, < 5.0)
|
22
|
-
aes_key_wrap (1.0
|
23
|
+
aes_key_wrap (1.1.0)
|
23
24
|
attr_required (1.0.1)
|
24
|
-
bigdecimal (
|
25
|
-
bindata (2.4.
|
26
|
-
concurrent-ruby (1.1.
|
27
|
-
connection_pool (2.2.
|
28
|
-
cool.io (1.
|
25
|
+
bigdecimal (3.0.0)
|
26
|
+
bindata (2.4.10)
|
27
|
+
concurrent-ruby (1.1.9)
|
28
|
+
connection_pool (2.2.5)
|
29
|
+
cool.io (1.7.1)
|
29
30
|
domain_name (0.5.20190701)
|
30
31
|
unf (>= 0.0.5, < 1.0.0)
|
31
|
-
ffi (1.
|
32
|
+
ffi (1.15.4)
|
32
33
|
ffi-compiler (1.0.1)
|
33
34
|
ffi (>= 1.0.0)
|
34
35
|
rake
|
35
|
-
fluent-plugin-
|
36
|
-
fluentd (>= 0.14.
|
37
|
-
|
36
|
+
fluent-plugin-kubernetes_metadata_filter (2.9.3)
|
37
|
+
fluentd (>= 0.14.0, < 1.15)
|
38
|
+
kubeclient (>= 4.0.0, < 5.0.0)
|
39
|
+
lru_redux
|
40
|
+
fluent-plugin-prometheus (2.0.2)
|
41
|
+
fluentd (>= 1.9.1, < 2)
|
42
|
+
prometheus-client (>= 2.1.0)
|
38
43
|
fluent-plugin-record-modifier (2.1.0)
|
39
44
|
fluentd (>= 1.0, < 2)
|
40
|
-
fluent-plugin-splunk-hec (1.2.
|
45
|
+
fluent-plugin-splunk-hec (1.2.9)
|
41
46
|
fluentd (>= 1.4)
|
42
47
|
multi_json (~> 1.13)
|
43
48
|
net-http-persistent (~> 3.1)
|
44
49
|
openid_connect (~> 1.1.8)
|
45
|
-
prometheus-client (
|
46
|
-
fluentd (1.
|
50
|
+
prometheus-client (>= 2.1.0)
|
51
|
+
fluentd (1.14.3)
|
52
|
+
bundler
|
47
53
|
cool.io (>= 1.4.5, < 2.0.0)
|
48
|
-
http_parser.rb (>= 0.5.1, < 0.
|
54
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
49
55
|
msgpack (>= 1.3.1, < 2.0.0)
|
50
|
-
serverengine (>= 2.
|
56
|
+
serverengine (>= 2.2.2, < 3.0.0)
|
51
57
|
sigdump (~> 0.2.2)
|
52
|
-
strptime (>= 0.2.
|
58
|
+
strptime (>= 0.2.4, < 1.0.0)
|
53
59
|
tzinfo (>= 1.0, < 3.0)
|
54
60
|
tzinfo-data (~> 1.0)
|
61
|
+
webrick (>= 1.4.2, < 1.8.0)
|
55
62
|
yajl-ruby (~> 1.0)
|
56
|
-
http (4.
|
63
|
+
http (4.4.1)
|
57
64
|
addressable (~> 2.3)
|
58
65
|
http-cookie (~> 1.0)
|
59
66
|
http-form_data (~> 2.2)
|
60
67
|
http-parser (~> 1.2.0)
|
61
68
|
http-accept (1.7.0)
|
62
|
-
http-cookie (1.0.
|
69
|
+
http-cookie (1.0.4)
|
63
70
|
domain_name (~> 0.5)
|
64
|
-
http-form_data (2.
|
65
|
-
http-parser (1.2.
|
71
|
+
http-form_data (2.3.0)
|
72
|
+
http-parser (1.2.3)
|
66
73
|
ffi-compiler (>= 1.0, < 2.0)
|
67
74
|
http_parser.rb (0.5.3)
|
68
75
|
httpclient (2.8.3)
|
69
|
-
i18n (1.8.
|
76
|
+
i18n (1.8.11)
|
70
77
|
concurrent-ruby (~> 1.0)
|
71
|
-
json-jwt (1.
|
78
|
+
json-jwt (1.13.0)
|
72
79
|
activesupport (>= 4.2)
|
73
80
|
aes_key_wrap
|
74
81
|
bindata
|
@@ -76,14 +83,15 @@ GEM
|
|
76
83
|
http (>= 3.0, < 5.0)
|
77
84
|
recursive-open-struct (~> 1.0, >= 1.0.4)
|
78
85
|
rest-client (~> 2.0)
|
86
|
+
lru_redux (1.1.0)
|
79
87
|
mail (2.7.1)
|
80
88
|
mini_mime (>= 0.1.1)
|
81
|
-
mime-types (3.
|
89
|
+
mime-types (3.4.1)
|
82
90
|
mime-types-data (~> 3.2015)
|
83
|
-
mime-types-data (3.
|
84
|
-
mini_mime (1.
|
85
|
-
minitest (5.
|
86
|
-
msgpack (1.
|
91
|
+
mime-types-data (3.2021.1115)
|
92
|
+
mini_mime (1.1.2)
|
93
|
+
minitest (5.15.0)
|
94
|
+
msgpack (1.4.2)
|
87
95
|
multi_json (1.14.1)
|
88
96
|
net-http-persistent (3.1.0)
|
89
97
|
connection_pool (~> 2.2)
|
@@ -99,61 +107,61 @@ GEM
|
|
99
107
|
validate_email
|
100
108
|
validate_url
|
101
109
|
webfinger (>= 1.0.1)
|
102
|
-
prometheus-client (
|
103
|
-
|
104
|
-
public_suffix (4.0.3)
|
105
|
-
quantile (0.2.1)
|
110
|
+
prometheus-client (2.1.0)
|
111
|
+
public_suffix (4.0.6)
|
106
112
|
rack (2.2.3)
|
107
|
-
rack-oauth2 (1.
|
113
|
+
rack-oauth2 (1.19.0)
|
108
114
|
activesupport
|
109
115
|
attr_required
|
110
116
|
httpclient
|
111
117
|
json-jwt (>= 1.11.0)
|
112
118
|
rack (>= 2.1.0)
|
113
|
-
rake (13.0.
|
114
|
-
recursive-open-struct (1.1.
|
119
|
+
rake (13.0.6)
|
120
|
+
recursive-open-struct (1.1.3)
|
115
121
|
rest-client (2.1.0)
|
116
122
|
http-accept (>= 1.7.0, < 2.0)
|
117
123
|
http-cookie (>= 1.0.2, < 2.0)
|
118
124
|
mime-types (>= 1.16, < 4.0)
|
119
125
|
netrc (~> 0.8)
|
120
|
-
serverengine (2.2.
|
126
|
+
serverengine (2.2.4)
|
121
127
|
sigdump (~> 0.2.2)
|
122
128
|
sigdump (0.2.4)
|
123
|
-
strptime (0.2.
|
124
|
-
swd (1.
|
129
|
+
strptime (0.2.5)
|
130
|
+
swd (1.3.0)
|
125
131
|
activesupport (>= 3)
|
126
132
|
attr_required (>= 0.0.5)
|
127
133
|
httpclient (>= 2.4)
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
tzinfo-data (1.2019.3)
|
134
|
+
tzinfo (2.0.4)
|
135
|
+
concurrent-ruby (~> 1.0)
|
136
|
+
tzinfo-data (1.2021.5)
|
132
137
|
tzinfo (>= 1.0.0)
|
133
138
|
unf (0.1.4)
|
134
139
|
unf_ext
|
135
|
-
unf_ext (0.0.
|
140
|
+
unf_ext (0.0.8)
|
136
141
|
validate_email (0.1.6)
|
137
142
|
activemodel (>= 3.0)
|
138
143
|
mail (>= 2.2.5)
|
139
|
-
validate_url (1.0.
|
144
|
+
validate_url (1.0.13)
|
140
145
|
activemodel (>= 3.0.0)
|
141
146
|
public_suffix
|
142
|
-
webfinger (1.
|
147
|
+
webfinger (1.2.0)
|
143
148
|
activesupport
|
144
149
|
httpclient (>= 2.4)
|
150
|
+
webrick (1.7.0)
|
145
151
|
yajl-ruby (1.4.1)
|
152
|
+
zeitwerk (2.5.1)
|
146
153
|
|
147
154
|
PLATFORMS
|
148
155
|
ruby
|
149
156
|
|
150
157
|
DEPENDENCIES
|
151
|
-
bigdecimal (=
|
158
|
+
bigdecimal (= 3.0.0)
|
152
159
|
fluent-plugin-k8s-metrics-agg!
|
153
|
-
fluent-plugin-
|
160
|
+
fluent-plugin-kubernetes_metadata_filter (>= 2.9.2)
|
161
|
+
fluent-plugin-prometheus (>= 2.0)
|
154
162
|
fluent-plugin-record-modifier (= 2.1.0)
|
155
|
-
fluent-plugin-splunk-hec (>= 1.2.
|
156
|
-
fluentd (
|
163
|
+
fluent-plugin-splunk-hec (>= 1.2.7)
|
164
|
+
fluentd (>= 1.14.2)
|
157
165
|
http_parser.rb (= 0.5.3)
|
158
166
|
kubeclient (= 4.6.0)
|
159
167
|
multi_json (= 1.14.1)
|
@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_development_dependency 'test-unit', "~> 3.3.0"
|
26
26
|
spec.add_development_dependency "simplecov", '~> 0.16.1'
|
27
27
|
spec.add_development_dependency 'webmock', '~> 3.5.1'
|
28
|
-
spec.add_runtime_dependency 'fluentd', '
|
28
|
+
spec.add_runtime_dependency 'fluentd', '>= 1.9.1'
|
29
29
|
spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
|
30
30
|
spec.add_runtime_dependency 'multi_json', '~> 1.14.1'
|
31
31
|
spec.add_runtime_dependency 'oj', '~> 3.10.2'
|
@@ -51,11 +51,11 @@ module Fluent
|
|
51
51
|
# m cpu is assumed standard
|
52
52
|
@cpu_mult = 1
|
53
53
|
@cpu_mult = 1000 if cpu[-1] != 'm'
|
54
|
-
cpu.delete('^0-9').to_i
|
54
|
+
cpu.delete('^0-9.').to_i
|
55
55
|
end
|
56
56
|
|
57
57
|
def get_cpu_or_memory_value(resource)
|
58
|
-
resource = resource.tr('^0-9', '').to_i
|
58
|
+
resource = resource.tr('^0-9.', '').to_i
|
59
59
|
resource
|
60
60
|
end
|
61
61
|
|
@@ -66,29 +66,35 @@ module Fluent
|
|
66
66
|
end
|
67
67
|
|
68
68
|
# https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory
|
69
|
+
# 1 Ki = 1024 bytes
|
70
|
+
# 1 K = 1000 bytes = 1000/1024 Ki = 1000/1024*1024 Mi
|
69
71
|
def get_memory_mult(memory)
|
70
|
-
memory_mult = if memory[-2] == 'Ki'
|
71
|
-
0
|
72
|
-
elsif memory[-
|
73
|
-
|
74
|
-
elsif memory[-2] == 'Mi'
|
72
|
+
memory_mult = if memory[-2..] == 'Ki'
|
73
|
+
1.0/1024
|
74
|
+
elsif memory[-1] == 'K'
|
75
|
+
1e3 / 1024 ** 2
|
76
|
+
elsif memory[-2..] == 'Mi'
|
75
77
|
1
|
76
|
-
elsif memory[-
|
77
|
-
|
78
|
-
elsif memory[-2] == 'Gi'
|
79
|
-
1000
|
80
|
-
elsif memory[-2] == 'G'
|
78
|
+
elsif memory[-1] == 'M'
|
79
|
+
1e6 / 1024 ** 2
|
80
|
+
elsif memory[-2..] == 'Gi'
|
81
81
|
1024
|
82
|
-
elsif memory[-
|
83
|
-
|
84
|
-
elsif memory[-2] == '
|
85
|
-
|
86
|
-
elsif memory[-
|
87
|
-
|
82
|
+
elsif memory[-1] == 'G'
|
83
|
+
1e9 / 1024 ** 2
|
84
|
+
elsif memory[-2..] == 'Ti'
|
85
|
+
1024 ** 2
|
86
|
+
elsif memory[-1] == 'T'
|
87
|
+
1e12 / 1024 ** 2
|
88
|
+
elsif memory[-2..] == 'Pi'
|
89
|
+
1024**3
|
90
|
+
elsif memory[-2] == 'P'
|
91
|
+
1e15 / 1024 ** 2
|
92
|
+
elsif memory[-2..] == 'Ei'
|
93
|
+
1024**4
|
88
94
|
elsif memory[-2] == 'E'
|
89
|
-
|
95
|
+
1e18 / 1024 ** 2
|
90
96
|
else
|
91
|
-
|
97
|
+
1 / 1024 ** 2
|
92
98
|
end
|
93
99
|
memory_mult
|
94
100
|
end
|
@@ -278,43 +284,47 @@ module Fluent
|
|
278
284
|
end
|
279
285
|
|
280
286
|
def get_cpu_value(resource)
|
281
|
-
cpu_val = resource.tr('^0-9', '').to_i
|
287
|
+
cpu_val = resource.tr('^0-9.', '').to_i
|
282
288
|
mult = get_cpu_mult(resource)
|
283
|
-
cpu_val
|
289
|
+
cpu_val = cpu_val * mult
|
284
290
|
cpu_val
|
285
291
|
end
|
286
292
|
|
287
293
|
def get_memory_mult(memory)
|
288
|
-
memory_mult = if memory[-2] == 'Ki'
|
289
|
-
0
|
290
|
-
elsif memory[-
|
291
|
-
|
292
|
-
elsif memory[-2] == 'Mi'
|
293
|
-
1
|
294
|
-
elsif memory[-2] == 'M'
|
294
|
+
memory_mult = if memory[-2..] == 'Ki'
|
295
|
+
1.0/1024
|
296
|
+
elsif memory[-1] == 'K'
|
297
|
+
1e3 / 1024 ** 2
|
298
|
+
elsif memory[-2..] == 'Mi'
|
295
299
|
1
|
296
|
-
elsif memory[-
|
297
|
-
|
298
|
-
elsif memory[-2] == '
|
300
|
+
elsif memory[-1] == 'M'
|
301
|
+
1e6 / 1024 ** 2
|
302
|
+
elsif memory[-2..] == 'Gi'
|
299
303
|
1024
|
300
|
-
elsif memory[-
|
301
|
-
|
302
|
-
elsif memory[-2] == '
|
303
|
-
|
304
|
-
elsif memory[-
|
305
|
-
|
304
|
+
elsif memory[-1] == 'G'
|
305
|
+
1e9 / 1024 ** 2
|
306
|
+
elsif memory[-2..] == 'Ti'
|
307
|
+
1024 ** 2
|
308
|
+
elsif memory[-1] == 'T'
|
309
|
+
1e12 / 1024 ** 2
|
310
|
+
elsif memory[-2..] == 'Pi'
|
311
|
+
1024**3
|
312
|
+
elsif memory[-2] == 'P'
|
313
|
+
1e15 / 1024 ** 2
|
314
|
+
elsif memory[-2..] == 'Ei'
|
315
|
+
1024**4
|
306
316
|
elsif memory[-2] == 'E'
|
307
|
-
|
317
|
+
1e18 / 1024 ** 2
|
308
318
|
else
|
309
|
-
|
319
|
+
1 / 1024 ** 2
|
310
320
|
end
|
311
321
|
memory_mult
|
312
322
|
end
|
313
323
|
|
314
324
|
def get_memory_value(resource)
|
315
|
-
mem_val = resource.tr('^0-9', '').to_i
|
325
|
+
mem_val = resource.tr('^0-9.', '').to_i
|
316
326
|
mult = get_memory_mult(resource)
|
317
|
-
mem_val
|
327
|
+
mem_val = mem_val * mult
|
318
328
|
mem_val
|
319
329
|
end
|
320
330
|
|
@@ -513,7 +523,7 @@ module Fluent
|
|
513
523
|
router.emit generate_tag('node') << ('.cpu.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_utilization
|
514
524
|
node_cpu_reservation = node_req_lim.instance_variable_get(:@cpu_request).to_f / node_cpu_allocatable
|
515
525
|
router.emit generate_tag('node') << ('.cpu.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_reservation
|
516
|
-
node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f /
|
526
|
+
node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f / node_memory_allocatable # converting from bytes to megabytes
|
517
527
|
router.emit generate_tag('node') << ('.memory.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_utilization
|
518
528
|
node_memory_reservation = node_req_lim.instance_variable_get(:@memory_request).to_f / node_memory_allocatable
|
519
529
|
router.emit generate_tag('node') << ('.memory.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_reservation
|
@@ -573,25 +583,33 @@ module Fluent
|
|
573
583
|
end
|
574
584
|
end
|
575
585
|
|
576
|
-
|
586
|
+
begin
|
587
|
+
node_response = JSON.parse(node_rest_client.get(@client.headers))
|
588
|
+
rescue RestClient::ServiceUnavailable
|
589
|
+
log.warn("Couldn't scrap metric for node '#{node_name} as it is unavailable. Ignoring it.'")
|
590
|
+
next
|
591
|
+
end
|
592
|
+
|
577
593
|
Array(node_response['pods']).each do |pod_json|
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
594
|
+
unless pod_json['cpu'].nil? || pod_json['memory'].nil?
|
595
|
+
pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000
|
596
|
+
pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0) / 1024 ** 2 # Converting to Mi
|
597
|
+
pod_namespace = pod_json['podRef']['namespace']
|
598
|
+
pod_usage = ResourceUsageMetricsUnit.new
|
599
|
+
pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
|
600
|
+
if @@namespace_resource_usage_metrics_map[pod_namespace].nil?
|
601
|
+
namespace_usage_metrics = ResourceUsageMetricsUnit.new
|
602
|
+
@@namespace_resource_usage_metrics_map[pod_namespace] = pod_usage
|
603
|
+
else
|
604
|
+
@@namespace_resource_usage_metrics_map[pod_namespace].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
|
605
|
+
end
|
606
|
+
if @@node_resource_usage_metrics_map[node_name].nil?
|
607
|
+
node_name_usage_metrics = ResourceUsageMetricsUnit.new
|
608
|
+
@@node_resource_usage_metrics_map[node_name] = node_name_usage_metrics
|
609
|
+
end
|
610
|
+
@@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
|
611
|
+
pod_usage = nil
|
592
612
|
end
|
593
|
-
@@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
|
594
|
-
pod_usage = nil
|
595
613
|
end
|
596
614
|
end
|
597
615
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-k8s-metrics-agg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Splunk Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-12-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -84,14 +84,14 @@ dependencies:
|
|
84
84
|
name: fluentd
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: 1.9.1
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 1.9.1
|
97
97
|
- !ruby/object:Gem::Dependency
|
@@ -144,13 +144,11 @@ executables: []
|
|
144
144
|
extensions: []
|
145
145
|
extra_rdoc_files: []
|
146
146
|
files:
|
147
|
-
- ".circleci/config.yml"
|
148
|
-
- ".circleci/install_dep.sh"
|
149
|
-
- ".circleci/trigger_func_test.sh"
|
150
147
|
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
151
148
|
- ".github/ISSUE_TEMPLATE/enhancement_request.md"
|
152
149
|
- ".github/ISSUE_TEMPLATE/failing_test.md"
|
153
150
|
- ".github/PULL_REQUEST_TEMPLATE.md"
|
151
|
+
- ".github/workflows/ci_build_test.yaml"
|
154
152
|
- ".gitignore"
|
155
153
|
- CLA.md
|
156
154
|
- CONTRIBUTING.md
|
@@ -162,6 +160,9 @@ files:
|
|
162
160
|
- README.md
|
163
161
|
- Rakefile
|
164
162
|
- VERSION
|
163
|
+
- ci_scripts/deploy_connector.sh
|
164
|
+
- ci_scripts/install_dep.sh
|
165
|
+
- ci_scripts/k8s-splunk.yml
|
165
166
|
- docker/CONTRIBUTING.md
|
166
167
|
- docker/Dockerfile
|
167
168
|
- docker/Gemfile
|
@@ -200,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
200
201
|
- !ruby/object:Gem::Version
|
201
202
|
version: '0'
|
202
203
|
requirements: []
|
203
|
-
rubygems_version: 3.1.
|
204
|
+
rubygems_version: 3.1.4
|
204
205
|
signing_key:
|
205
206
|
specification_version: 4
|
206
207
|
summary: A fluentd input plugin that collects kubernetes cluster metrics.
|
data/.circleci/config.yml
DELETED
@@ -1,88 +0,0 @@
|
|
1
|
-
version: 2
|
2
|
-
jobs:
|
3
|
-
build:
|
4
|
-
docker:
|
5
|
-
- image: circleci/ruby:2.6.1-node-browsers
|
6
|
-
working_directory: ~/repo
|
7
|
-
steps:
|
8
|
-
- checkout
|
9
|
-
- setup_remote_docker:
|
10
|
-
reusable: true
|
11
|
-
- run:
|
12
|
-
name: Install dependencies
|
13
|
-
command: |
|
14
|
-
.circleci/install_dep.sh
|
15
|
-
- run:
|
16
|
-
name: Builder
|
17
|
-
command: |
|
18
|
-
rake build -t -v
|
19
|
-
cp -R pkg /tmp
|
20
|
-
- persist_to_workspace:
|
21
|
-
root: /tmp
|
22
|
-
paths:
|
23
|
-
- pkg
|
24
|
-
|
25
|
-
unit_test:
|
26
|
-
docker:
|
27
|
-
- image: circleci/ruby:2.6.1-node-browsers
|
28
|
-
working_directory: ~/repo
|
29
|
-
steps:
|
30
|
-
- attach_workspace:
|
31
|
-
at: /tmp
|
32
|
-
- checkout
|
33
|
-
- setup_remote_docker:
|
34
|
-
reusable: true
|
35
|
-
- run:
|
36
|
-
name: Install dependencies
|
37
|
-
command: |
|
38
|
-
.circleci/install_dep.sh
|
39
|
-
- run:
|
40
|
-
name: Run unit tests
|
41
|
-
command: |
|
42
|
-
bundle exec rake test -t -v
|
43
|
-
|
44
|
-
func_test_master:
|
45
|
-
docker:
|
46
|
-
- image: circleci/ruby:2.6.1-node-browsers
|
47
|
-
working_directory: ~/repo
|
48
|
-
steps:
|
49
|
-
- checkout
|
50
|
-
- setup_remote_docker:
|
51
|
-
reusable: true
|
52
|
-
- run:
|
53
|
-
name: trigger
|
54
|
-
command: |
|
55
|
-
.circleci/trigger_func_test.sh master
|
56
|
-
|
57
|
-
func_test_develop:
|
58
|
-
docker:
|
59
|
-
- image: circleci/ruby:2.6.1-node-browsers
|
60
|
-
working_directory: ~/repo
|
61
|
-
steps:
|
62
|
-
- checkout
|
63
|
-
- setup_remote_docker:
|
64
|
-
reusable: true
|
65
|
-
- run:
|
66
|
-
name: trigger
|
67
|
-
command: |
|
68
|
-
.circleci/trigger_func_test.sh develop
|
69
|
-
|
70
|
-
workflows:
|
71
|
-
version: 2
|
72
|
-
build_test_push:
|
73
|
-
jobs:
|
74
|
-
- build:
|
75
|
-
filters:
|
76
|
-
branches:
|
77
|
-
ignore:
|
78
|
-
- /^release\/.*/
|
79
|
-
- master
|
80
|
-
- unit_test:
|
81
|
-
requires:
|
82
|
-
- build
|
83
|
-
- func_test_master:
|
84
|
-
requires:
|
85
|
-
- unit_test
|
86
|
-
- func_test_develop:
|
87
|
-
requires:
|
88
|
-
- unit_test
|
@@ -1,42 +0,0 @@
|
|
1
|
-
#!/usr/bin/env bash
|
2
|
-
|
3
|
-
# trigger SCK 'master' branch to introduce this new image from this commit
|
4
|
-
# to working version of every other component.
|
5
|
-
ORGANIZATION=splunk
|
6
|
-
PROJECT=splunk-connect-for-kubernetes
|
7
|
-
BRANCH=$1
|
8
|
-
|
9
|
-
# Trigger functional test
|
10
|
-
curl -X POST --header "Content-Type: application/json" \
|
11
|
-
-d '{"build_parameters": {"CIRCLE_JOB":"build_test", "TRIG_BRANCH":"'"$CIRCLE_BRANCH"'", "TRIG_PROJECT":"'"$CIRCLE_PROJECT_REPONAME"'", "TRIG_REPO":"'"$CIRCLE_REPOSITORY_URL"'"}}' "https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/tree/$BRANCH?circle-token=$CIRCLE_TOKEN" > build.json
|
12
|
-
cat build.json
|
13
|
-
BUILD_NUM=$(jq -r .build_num build.json)
|
14
|
-
sleep 10
|
15
|
-
|
16
|
-
# Wait until finish or maximum 20 minutes
|
17
|
-
TIMEOUT=20
|
18
|
-
DONE="FALSE"
|
19
|
-
until [ "$TIMEOUT" -lt 0 ] || [ "$DONE" == "TRUE" ]; do
|
20
|
-
curl https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/$BUILD_NUM?circle-token=$CIRCLE_TOKEN > build_progress.json
|
21
|
-
STATUS=$(jq -r .status build_progress.json)
|
22
|
-
echo "STATUS = $STATUS"
|
23
|
-
if [ "$STATUS" != "running" ] && [ "$STATUS" != "queued" ]; then
|
24
|
-
DONE="TRUE"
|
25
|
-
else
|
26
|
-
let TIMEOUT--
|
27
|
-
sleep 60
|
28
|
-
fi
|
29
|
-
done
|
30
|
-
|
31
|
-
BUILD_URL=$(jq -r .build_url build_progress.json)
|
32
|
-
if [ "$DONE" == "FALSE" ]; then
|
33
|
-
# Cancel hanging job and fail
|
34
|
-
curl -X POST https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/$BUILD_NUM/cancel?circle-token=$CIRCLE_TOKEN
|
35
|
-
else
|
36
|
-
if [ "$STATUS" != "success" ] && [ "$STATUS" != "fixed" ]; then
|
37
|
-
echo "Functional test have failed please see:"
|
38
|
-
echo $BUILD_URL
|
39
|
-
exit 1
|
40
|
-
fi
|
41
|
-
exit 0
|
42
|
-
fi
|