fluent-plugin-http-cwm 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/publish.yml +1 -1
- data/Gemfile.lock +2 -2
- data/README.md +24 -2
- data/fluent-plugin-http-cwm.gemspec +1 -1
- data/lib/fluent/plugin/in_http_cwm.rb +62 -32
- data/test/plugin/test_in_http_cwm.rb +43 -11
- metadata +10 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ee799e3ac189ac3437e101b229bf0372b46b853312f45c36c89252f7928e3682
|
4
|
+
data.tar.gz: 2ec2f5a36742eb61f7365436d1e19d6ede930b20ad9c08f5cd2612a9af42ee1f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f98f46b4cd8e5864bbcd0ea2c1801bda9a18de1116904feaba035a3ece3f2c710cdc525d9f202bd15599c39b978c02e137e9a9c9b42f5b07c5bce529b7f8d17e
|
7
|
+
data.tar.gz: 90b110133ed9377e746b81f784fad358830c15660961f2f7cc142a6d5817da2e8d2f8d70018a1614b10e373a49c96c32c23f6f37145c51e9e61a076eeab573ae
|
@@ -11,7 +11,7 @@ on:
|
|
11
11
|
|
12
12
|
jobs:
|
13
13
|
publish:
|
14
|
-
if: github.event.workflow_run.conclusion == 'success'
|
14
|
+
if: github.event.workflow_run.conclusion == 'success' && startsWith(github.ref, 'refs/tags/v')
|
15
15
|
|
16
16
|
name: Publish Gem
|
17
17
|
runs-on: ubuntu-18.04
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-http-cwm (0.
|
4
|
+
fluent-plugin-http-cwm (0.3.0)
|
5
5
|
fluentd (>= 0.14.10, < 2)
|
6
6
|
json (~> 2.3, >= 2.3.0)
|
7
7
|
redis (~> 4.2, >= 4.2.5)
|
@@ -12,7 +12,7 @@ GEM
|
|
12
12
|
concurrent-ruby (1.1.8)
|
13
13
|
cool.io (1.7.1)
|
14
14
|
docile (1.3.5)
|
15
|
-
fluentd (1.12.
|
15
|
+
fluentd (1.12.3)
|
16
16
|
bundler
|
17
17
|
cool.io (>= 1.4.5, < 2.0.0)
|
18
18
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
data/README.md
CHANGED
@@ -10,6 +10,17 @@
|
|
10
10
|
![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/iamAzeem/fluent-plugin-http-cwm?style=flat-square)
|
11
11
|
![GitHub repo size](https://img.shields.io/github/repo-size/iamAzeem/fluent-plugin-http-cwm?style=flat-square)
|
12
12
|
|
13
|
+
- [Overview](#overview)
|
14
|
+
- [Installation](#installation)
|
15
|
+
- [RubyGems](#rubygems)
|
16
|
+
- [Bundler](#bundler)
|
17
|
+
- [Configuration](#configuration)
|
18
|
+
- [`<redis>` section (optional) (single)](#redis-section-optional-single)
|
19
|
+
- [Sample Configuration](#sample-configuration)
|
20
|
+
- [Contribute](#contribute)
|
21
|
+
- [Publish the gem](#publish-the-gem)
|
22
|
+
- [License](#license)
|
23
|
+
|
13
24
|
## Overview
|
14
25
|
|
15
26
|
[Fluentd](https://fluentd.org/) HTTP input plugin for
|
@@ -100,7 +111,7 @@ bundle
|
|
100
111
|
- Default value: `6379`.
|
101
112
|
- `db` (integer) (optional): The db to use.
|
102
113
|
- Default value: `0`.
|
103
|
-
- `grace_period` (time) (optional): The grace period for last update.
|
114
|
+
- `grace_period` (time) (optional): The grace period for last action update.
|
104
115
|
- Default value: `300s`.
|
105
116
|
- `flush_interval` (time) (optional): The flush interval to send metrics.
|
106
117
|
- Default value: `300s`.
|
@@ -164,10 +175,21 @@ Example:
|
|
164
175
|
- Check out the latest `main` branch.
|
165
176
|
- Create a feature or bugfix branch from `main`.
|
166
177
|
- Commit and push your changes.
|
167
|
-
- Make sure to add tests
|
178
|
+
- Make sure to add and run tests locally: `bundle exec rake test`.
|
168
179
|
- Run Rubocop locally and fix all the lint warnings.
|
180
|
+
- Make sure to update [Gemfile.lock](Gemfile.lock): `sudo bundle update`.
|
169
181
|
- Submit the PR.
|
170
182
|
|
183
|
+
## Publish the gem
|
184
|
+
|
185
|
+
The gem is published via the [publish.yml](.github/workflows/publish.yml)
|
186
|
+
Workflow on tagging. The tag must be of the format `v0.3.0`. This workflow
|
187
|
+
depends on the successful completion of the [ci.yml](.github/workflows/ci.yml)
|
188
|
+
workflow and then it looks for the tag. So, make sure that all the CI issues are
|
189
|
+
resolved before creating a new tag. If there are issues while publishing the gem
|
190
|
+
i.e. publish workflow doesn't work properly, you can delete and then recreate
|
191
|
+
the tag to retrigger this workflow.
|
192
|
+
|
171
193
|
## License
|
172
194
|
|
173
195
|
[Apache 2.0](./LICENSE)
|
@@ -53,7 +53,7 @@ module Fluent
|
|
53
53
|
desc 'The db to use.'
|
54
54
|
config_param :db, :integer, default: 0
|
55
55
|
|
56
|
-
desc 'The grace period for last update.'
|
56
|
+
desc 'The grace period for last action update.'
|
57
57
|
config_param :grace_period, :time, default: '300s'
|
58
58
|
|
59
59
|
desc 'The flush interval to send metrics.'
|
@@ -71,6 +71,9 @@ module Fluent
|
|
71
71
|
|
72
72
|
@redis = nil
|
73
73
|
@deployment_api_metrics = default_api_metrics_hash
|
74
|
+
|
75
|
+
@last_action_queue = Queue.new
|
76
|
+
@last_action_entry = []
|
74
77
|
end
|
75
78
|
|
76
79
|
def configure(conf)
|
@@ -82,18 +85,23 @@ module Fluent
|
|
82
85
|
def start
|
83
86
|
super
|
84
87
|
|
85
|
-
#
|
86
|
-
timer_execute(:
|
88
|
+
# start interval timer to flush api metrics
|
89
|
+
timer_execute(:api_metrics_flush_timer, @redis_config.flush_interval) do
|
87
90
|
flush_api_metrics
|
88
91
|
end
|
89
92
|
|
93
|
+
# start interval timer to flush last action entry
|
94
|
+
timer_execute(:last_action_flush_timer, '1s') do
|
95
|
+
flush_last_action
|
96
|
+
end
|
97
|
+
|
90
98
|
log.info("Starting HTTP server [#{@host}:#{@port}]...")
|
91
99
|
http_server_create_http_server(:http_server, addr: @host, port: @port, logger: log) do |server|
|
92
100
|
server.post("/#{tag}") do |req|
|
93
101
|
data = parse_data(req.body)
|
94
102
|
route(data) if update_deployment_metrics(data)
|
95
103
|
|
96
|
-
# return HTTP 200 OK response
|
104
|
+
# return HTTP 200 OK response with emtpy body
|
97
105
|
[200, { 'Content-Type' => 'text/plain' }, nil]
|
98
106
|
end
|
99
107
|
end
|
@@ -141,40 +149,62 @@ module Fluent
|
|
141
149
|
router.emit(@tag, time, record)
|
142
150
|
end
|
143
151
|
|
144
|
-
def
|
145
|
-
|
152
|
+
def flush_last_action
|
153
|
+
if @last_action_entry.empty?
|
154
|
+
@last_action_entry = @last_action_queue.deq.split('|')
|
155
|
+
log.debug("Dequed last action entry. #{@last_action_entry}")
|
156
|
+
else
|
157
|
+
deploymentid, last_action = @last_action_entry
|
158
|
+
@last_action_entry = [] if update_deployment_last_action(deploymentid, last_action)
|
159
|
+
end
|
146
160
|
end
|
147
161
|
|
148
|
-
def
|
149
|
-
|
162
|
+
def datetime_diff_in_secs(dt_begin, dt_end)
|
163
|
+
seconds = ((dt_end - dt_begin) * 24 * 60 * 60)
|
164
|
+
seconds.to_i
|
165
|
+
end
|
150
166
|
|
167
|
+
def update_deployment_last_action(deploymentid, last_action)
|
151
168
|
key = "#{@redis_config.last_update_prefix}:#{deploymentid}"
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
log.debug('last action entry not found. going to be set for the first time.')
|
169
|
+
log.debug("Checking existing last action entry [key: #{key}]")
|
170
|
+
lastval = @redis.get(key)
|
171
|
+
|
172
|
+
is_grace_period_expired = false
|
173
|
+
if lastval
|
174
|
+
curdt = DateTime.now
|
175
|
+
lastdt = DateTime.parse(lastval, FMT_DATETIME)
|
176
|
+
dt_diff_secs = datetime_diff_in_secs(lastdt, curdt)
|
177
|
+
log.debug("Current Data/Time: #{curdt}")
|
178
|
+
log.debug("Previous Date/Time: #{lastdt}")
|
179
|
+
log.debug("Date/Time diff (s): #{dt_diff_secs}")
|
180
|
+
|
181
|
+
if dt_diff_secs >= @redis_config.grace_period
|
182
|
+
is_grace_period_expired = true
|
183
|
+
log.debug("Grace period expired for last action update. [#{@redis_config.grace_period}]")
|
168
184
|
end
|
185
|
+
else
|
186
|
+
log.debug('Last action entry does not exist. It will be set for the first time.')
|
187
|
+
end
|
169
188
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
189
|
+
if lastdt.nil? || is_grace_period_expired
|
190
|
+
log.debug('Updating deployment last action')
|
191
|
+
last_action = DateTime.parse(last_action, FMT_DATETIME)
|
192
|
+
@redis.set(key, last_action)
|
193
|
+
log.debug("Updated last action entry [#{key} => #{last_action}]")
|
194
|
+
true
|
195
|
+
else
|
196
|
+
false
|
177
197
|
end
|
198
|
+
rescue StandardError => e
|
199
|
+
log.error("Unable to update last action! ERROR: '#{e}'.")
|
200
|
+
false
|
201
|
+
end
|
202
|
+
|
203
|
+
def enque_last_action_entry(deploymentid)
|
204
|
+
last_action = DateTime.now
|
205
|
+
entry = "#{deploymentid}|#{last_action}"
|
206
|
+
@last_action_queue.enq(entry)
|
207
|
+
log.debug("Enqued last action entry. [#{entry}]")
|
178
208
|
end
|
179
209
|
|
180
210
|
def validate_and_get_value(data_hash, key)
|
@@ -191,7 +221,7 @@ module Fluent
|
|
191
221
|
deploymentid = validate_and_get_value(data, 'deploymentid')
|
192
222
|
return false unless deploymentid
|
193
223
|
|
194
|
-
|
224
|
+
enque_last_action_entry(deploymentid)
|
195
225
|
|
196
226
|
api_data = validate_and_get_value(data, 'api')
|
197
227
|
return false unless api_data
|
@@ -8,9 +8,26 @@ class CwmHttpInputTest < Test::Unit::TestCase
|
|
8
8
|
setup do
|
9
9
|
Fluent::Test.setup
|
10
10
|
|
11
|
-
@default_conf =
|
12
|
-
|
13
|
-
|
11
|
+
@default_conf = config_element('ROOT', '', { 'tag' => 'test' })
|
12
|
+
@custom_conf = config_element('ROOT', '', { 'tag' => 'test' }, [
|
13
|
+
config_element('redis', '', {
|
14
|
+
'grace_period' => '1s',
|
15
|
+
'flush_interval' => '1s'
|
16
|
+
})
|
17
|
+
])
|
18
|
+
|
19
|
+
@expected_redis_output = {
|
20
|
+
'deploymentid:last_action:docker-compose-http' => '',
|
21
|
+
'deploymentid:last_action:docker-compose-https' => '',
|
22
|
+
'deploymentid:minio-metrics:docker-compose-http:bytes_in' => '61852',
|
23
|
+
'deploymentid:minio-metrics:docker-compose-https:bytes_in' => '14875',
|
24
|
+
'deploymentid:minio-metrics:docker-compose-http:bytes_out' => '112012',
|
25
|
+
'deploymentid:minio-metrics:docker-compose-https:bytes_out' => '2755',
|
26
|
+
'deploymentid:minio-metrics:docker-compose-http:num_requests_out' => '7',
|
27
|
+
'deploymentid:minio-metrics:docker-compose-http:num_requests_in' => '10',
|
28
|
+
'deploymentid:minio-metrics:docker-compose-https:num_requests_in' => '5',
|
29
|
+
'deploymentid:minio-metrics:docker-compose-http:num_requests_misc' => '5'
|
30
|
+
}.freeze
|
14
31
|
end
|
15
32
|
|
16
33
|
def create_driver(conf)
|
@@ -27,13 +44,7 @@ class CwmHttpInputTest < Test::Unit::TestCase
|
|
27
44
|
end
|
28
45
|
|
29
46
|
test 'redis default configuration test' do
|
30
|
-
|
31
|
-
tag test
|
32
|
-
<redis>
|
33
|
-
</redis>
|
34
|
-
)
|
35
|
-
|
36
|
-
driver = create_driver(conf)
|
47
|
+
driver = create_driver(@default_conf)
|
37
48
|
plugin = driver.instance
|
38
49
|
redis = plugin.redis_config
|
39
50
|
assert_equal 'localhost', redis.host
|
@@ -47,7 +58,9 @@ class CwmHttpInputTest < Test::Unit::TestCase
|
|
47
58
|
|
48
59
|
sub_test_case 'route#emit' do
|
49
60
|
test 'emit test' do
|
50
|
-
driver = create_driver(@
|
61
|
+
driver = create_driver(@custom_conf)
|
62
|
+
plugin = driver.instance
|
63
|
+
redis = plugin.redis_config
|
51
64
|
|
52
65
|
res_codes = []
|
53
66
|
lines = 0
|
@@ -63,6 +76,25 @@ class CwmHttpInputTest < Test::Unit::TestCase
|
|
63
76
|
assert_equal lines, res_codes.size
|
64
77
|
assert_equal '200', res_codes[0]
|
65
78
|
assert_equal 1, res_codes.uniq.size
|
79
|
+
|
80
|
+
# run and test private flushing methods
|
81
|
+
`redis-cli FLUSHALL`
|
82
|
+
sleep(redis.grace_period)
|
83
|
+
driver.events.each do
|
84
|
+
plugin.send(:flush_api_metrics)
|
85
|
+
plugin.send(:flush_last_action)
|
86
|
+
end
|
87
|
+
|
88
|
+
# verify from Redis server
|
89
|
+
@expected_redis_output.each do |key, expected_value|
|
90
|
+
if key.include? 'last_action'
|
91
|
+
exists = `redis-cli EXISTS #{key}`.chomp
|
92
|
+
assert_equal '1', exists
|
93
|
+
else
|
94
|
+
actual_value = `redis-cli GET #{key}`.chomp
|
95
|
+
assert_equal expected_value, actual_value
|
96
|
+
end
|
97
|
+
end
|
66
98
|
end
|
67
99
|
end
|
68
100
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-http-cwm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Azeem Sajid
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-11-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -108,22 +108,22 @@ dependencies:
|
|
108
108
|
name: json
|
109
109
|
requirement: !ruby/object:Gem::Requirement
|
110
110
|
requirements:
|
111
|
-
- - ">="
|
112
|
-
- !ruby/object:Gem::Version
|
113
|
-
version: 2.3.0
|
114
111
|
- - "~>"
|
115
112
|
- !ruby/object:Gem::Version
|
116
113
|
version: '2.3'
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 2.3.0
|
117
117
|
type: :runtime
|
118
118
|
prerelease: false
|
119
119
|
version_requirements: !ruby/object:Gem::Requirement
|
120
120
|
requirements:
|
121
|
-
- - ">="
|
122
|
-
- !ruby/object:Gem::Version
|
123
|
-
version: 2.3.0
|
124
121
|
- - "~>"
|
125
122
|
- !ruby/object:Gem::Version
|
126
123
|
version: '2.3'
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: 2.3.0
|
127
127
|
- !ruby/object:Gem::Dependency
|
128
128
|
name: redis
|
129
129
|
requirement: !ruby/object:Gem::Requirement
|
@@ -185,7 +185,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
185
185
|
- !ruby/object:Gem::Version
|
186
186
|
version: '0'
|
187
187
|
requirements: []
|
188
|
-
|
188
|
+
rubyforge_project:
|
189
|
+
rubygems_version: 2.7.6
|
189
190
|
signing_key:
|
190
191
|
specification_version: 4
|
191
192
|
summary: fluentd HTTP Input Plugin for CloudWebManage Logging Component
|