fluent-plugin-site24x7 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/LICENSE +10 -0
- data/README.md +69 -0
- data/Rakefile +13 -0
- data/fluent-plugin-site24x7.gemspec +32 -0
- data/lib/fluent/plugin/out_site24x7.rb +352 -0
- metadata +159 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 0f64dbce42d3ac7dc65c2e58daf74be07be8657c7cb9ec81089a0212cb15509b
|
4
|
+
data.tar.gz: b4732542ec17f5bb32d34dc218ae5c4e1b12064978d46ba3d0dec591d5be8115
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b5a1be07bed5a2f8aa0d4f5ea6572fd7ed4dc1cb635cca6b5f7329418d46aebc1bc6278a5f9823705f05e373a03fcffaec7bdc63a9040f2ba832e14bf33ef342
|
7
|
+
data.tar.gz: 8ef90d45f72943d151cbeed6121043c3c3c47a956f50a4c6a702bc91d79d30e627a61007985036c09cd7f5d6322c88a46557303cca7334ff147cf2734e956def
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
Copyright (c) 2021, ZOHO CORPORATION
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
5
|
+
|
6
|
+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
7
|
+
|
8
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
9
|
+
|
10
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
# Fluentd output plugin for Site24x7
|
2
|
+
|
3
|
+
This output plugin allows parse and sending logs directly from Fluentd to Site2x7 - so you don't have to use a separate log shipper.
|
4
|
+
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
To add the plugin to your fluentd agent, use the following command:
|
9
|
+
|
10
|
+
```
|
11
|
+
$ gem install fluent-plugin-site24x7
|
12
|
+
```
|
13
|
+
If you installed the td-agent instead
|
14
|
+
|
15
|
+
```
|
16
|
+
$ /usr/sbin/td-agent-gem install fluent-plugin-site24x7
|
17
|
+
```
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
**Configure the output plugin**
|
22
|
+
|
23
|
+
To match events and send them to Site24x7, simply add the following code to your configuration file.
|
24
|
+
|
25
|
+
```cfg
|
26
|
+
# Match events tagged with "site24x7.**" and send them to Site24x7
|
27
|
+
<match site24x7.**>
|
28
|
+
|
29
|
+
@type site24x7
|
30
|
+
@id site24x7_agent
|
31
|
+
log_type_config <your_log_type_config>
|
32
|
+
|
33
|
+
# Optional parameters
|
34
|
+
max_retry '3'
|
35
|
+
retry_interval '2'
|
36
|
+
http_idle_timeout '5'
|
37
|
+
http_read_timeout '30'
|
38
|
+
|
39
|
+
# Optional http proxy
|
40
|
+
http_proxy 'http://user:passs@mproxy.host:proxy.port'
|
41
|
+
|
42
|
+
<buffer>
|
43
|
+
@type memory
|
44
|
+
flush_thread_count 4
|
45
|
+
flush_interval 3s
|
46
|
+
chunk_limit_size 5m
|
47
|
+
chunk_limit_records 500
|
48
|
+
</buffer>
|
49
|
+
|
50
|
+
</match>
|
51
|
+
```
|
52
|
+
After a restart of FluentD, any events tagged with site24x7 are shipped to site24x7 platform.
|
53
|
+
|
54
|
+
## Parameters
|
55
|
+
As fluent-plugin-site24x7 is an output_buffer, you can set all output_buffer properties like it's describe in the [fluentd documentation](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
|
56
|
+
|
57
|
+
Property | Description | Default Value
|
58
|
+
------------ | -------------|------------
|
59
|
+
log_type_config | log_type_config for your configured log type in site24x7 | nil
|
60
|
+
max_retry | How many times to resend failed uploads | 3
|
61
|
+
retry_interval | How long to sleep initially between retries, exponential step-off | 2 seconds
|
62
|
+
http_idle_timeout | Timeout in seconds that the http persistent connection will stay open without traffic | 5 seconds
|
63
|
+
http_read_timeout | Timeout in seconds when the socket connects until the connection breaks | 30 secods
|
64
|
+
http_proxy | Your proxy uri | nil
|
65
|
+
|
66
|
+
## Release Notes
|
67
|
+
|
68
|
+
* 0.1.0 - Initial Release
|
69
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "bundler"
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
require "rake/testtask"
|
5
|
+
|
6
|
+
Rake::TestTask.new(:test) do |t|
|
7
|
+
t.libs.push("lib", "test")
|
8
|
+
t.test_files = FileList["test/**/test_*.rb"]
|
9
|
+
t.verbose = true
|
10
|
+
t.warning = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task default: [:test]
|
@@ -0,0 +1,32 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = "fluent-plugin-site24x7"
|
6
|
+
spec.version = "0.1.1"
|
7
|
+
spec.authors = ["Magesh Rajan"]
|
8
|
+
spec.email = ["magesh.rajan@zohocorp.com"]
|
9
|
+
|
10
|
+
spec.summary = %q{Site24x7 output plugin for Fluent event collector.}
|
11
|
+
spec.homepage = "https://github.com/site24x7/fluent-plugin-site24x7"
|
12
|
+
spec.license = ""
|
13
|
+
|
14
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
15
|
+
f.match(%r{^(test|spec|features)/})
|
16
|
+
end
|
17
|
+
spec.files = files
|
18
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
+
spec.test_files = test_files
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
|
22
|
+
spec.add_development_dependency "bundler", "~> 2.2.24"
|
23
|
+
spec.add_development_dependency "rake", "~> 13.0.3"
|
24
|
+
spec.add_development_dependency "test-unit", "~> 3.3.7"
|
25
|
+
spec.add_development_dependency "yajl-ruby", "~> 1.2"
|
26
|
+
spec.add_development_dependency 'webmock', '~> 3.5', '>= 3.5.0'
|
27
|
+
|
28
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
29
|
+
spec.add_runtime_dependency "net-http-persistent", '~> 3.1'
|
30
|
+
|
31
|
+
|
32
|
+
end
|
@@ -0,0 +1,352 @@
|
|
1
|
+
require "base64"
|
2
|
+
require "socket"
|
3
|
+
require "openssl"
|
4
|
+
require "yajl"
|
5
|
+
require "zlib"
|
6
|
+
require "date"
|
7
|
+
require "fluent/plugin/output"
|
8
|
+
|
9
|
+
class Fluent::Site24x7Output < Fluent::Plugin::Output
|
10
|
+
|
11
|
+
Fluent::Plugin.register_output('site24x7', self)
|
12
|
+
|
13
|
+
S247_MAX_RECORD_COUNT = 500
|
14
|
+
S247_MAX_RECORD_SIZE = 1000000
|
15
|
+
S247_MAX_BATCH_SIZE = 5000000
|
16
|
+
S247_LOG_UPLOAD_CHECK_INTERVAL = 600 #10 minutes
|
17
|
+
S247_TRUNCATION_SUFFIX = "##TRUNCATED###"
|
18
|
+
|
19
|
+
helpers :compat_parameters
|
20
|
+
|
21
|
+
config_param :log_type_config, :string
|
22
|
+
config_param :max_retry, :integer, :default => 3
|
23
|
+
config_param :retry_interval, :integer, :default => 2
|
24
|
+
config_param :http_idle_timeout, :integer, default: 5
|
25
|
+
config_param :http_read_timeout, :integer, default: 30
|
26
|
+
config_param :http_proxy, :string, :default => nil
|
27
|
+
|
28
|
+
config_section :buffer do
|
29
|
+
config_set_default :@type, "memory"
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
super
|
34
|
+
end
|
35
|
+
|
36
|
+
def configure(conf)
|
37
|
+
compat_parameters_convert(conf, :buffer)
|
38
|
+
super
|
39
|
+
if conf['http_proxy']
|
40
|
+
ENV['http_proxy'] = @http_proxy
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def multi_workers_ready?
|
45
|
+
true
|
46
|
+
end
|
47
|
+
|
48
|
+
def formatted_to_msgpack_binary?
|
49
|
+
true
|
50
|
+
end
|
51
|
+
|
52
|
+
def start
|
53
|
+
super
|
54
|
+
init_variables()
|
55
|
+
init_http_client(@logtype_config)
|
56
|
+
end
|
57
|
+
|
58
|
+
def shutdown
|
59
|
+
super
|
60
|
+
end
|
61
|
+
|
62
|
+
def terminate
|
63
|
+
super
|
64
|
+
@s247_http_client.shutdown if @s247_http_client
|
65
|
+
end
|
66
|
+
|
67
|
+
def init_variables()
|
68
|
+
@logtype_config = Yajl::Parser.parse(Base64.decode64(@log_type_config))
|
69
|
+
@s247_custom_regex = if @logtype_config.has_key? 'regex' then Regexp.compile(@logtype_config['regex'].gsub('?P<','?<')) else nil end
|
70
|
+
@s247_ignored_fields = if @logtype_config.has_key? 'ignored_fields' then @logtype_config['ignored_fields'] else [] end
|
71
|
+
@s247_tz = {'hrs': 0, 'mins': 0} #UTC
|
72
|
+
@log_source = Socket.gethostname
|
73
|
+
@valid_logtype = true
|
74
|
+
@log_upload_allowed = true
|
75
|
+
@log_upload_stopped_time = 0
|
76
|
+
@s247_datetime_format_string = @logtype_config['dateFormat']
|
77
|
+
@s247_datetime_format_string = @s247_datetime_format_string.sub('%f', '%N')
|
78
|
+
if !@s247_datetime_format_string.include? 'unix'
|
79
|
+
is_year_present = if @s247_datetime_format_string.include?('%y') || @s247_datetime_format_string.include?('%Y') then true else false end
|
80
|
+
if !is_year_present
|
81
|
+
@s247_datetime_format_string = @s247_datetime_format_string+ ' %Y'
|
82
|
+
end
|
83
|
+
is_timezone_present = if @s247_datetime_format_string.include? '%z' then true else false end
|
84
|
+
if !is_timezone_present && @logtype_config.has_key?('timezone')
|
85
|
+
tz_value = @logtype_config['timezone']
|
86
|
+
if tz_value.start_with?('+')
|
87
|
+
@s247_tz['hrs'] = Integer('-' + tz_value[1..4])
|
88
|
+
@s247_tz['mins'] = Integer('-' + tz_value[3..6])
|
89
|
+
elsif tz_value.start_with?('-')
|
90
|
+
@s247_tz['hrs'] = Integer('+' + tz_value[1..4])
|
91
|
+
@s247_tz['mins'] = Integer('+' + tz_value[3..6])
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def init_http_client(logtype_config)
|
98
|
+
require 'net/http/persistent'
|
99
|
+
upload_url = 'https://'+logtype_config['uploadDomain']+'/upload'
|
100
|
+
@uri = URI(upload_url)
|
101
|
+
log.info "Starting HTTP connection to #{upload_url}"
|
102
|
+
@s247_http_client = Net::HTTP::Persistent.new name: "fluent-plugin-site24x7", proxy: :ENV
|
103
|
+
@s247_http_client.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
104
|
+
@s247_http_client.idle_timeout = @http_idle_timeout
|
105
|
+
@s247_http_client.read_timeout = @http_read_timeout
|
106
|
+
@s247_http_client.override_headers["Content-Type"] = "application/json"
|
107
|
+
@s247_http_client.override_headers["Content-Encoding"] = "gzip"
|
108
|
+
@s247_http_client.override_headers["X-DeviceKey"] = logtype_config['apiKey']
|
109
|
+
@s247_http_client.override_headers["X-LogType"] = logtype_config['logType']
|
110
|
+
@s247_http_client.override_headers["X-StreamMode"] = 1
|
111
|
+
@s247_http_client.override_headers["User-Agent"] = 'Fluentd'
|
112
|
+
if !@s247_http_client.proxy_uri.nil?
|
113
|
+
log.info "Using HTTP proxy #{@s247_http_client.proxy_uri.scheme}://#{@s247_http_client.proxy_uri.host}:#{@s247_http_client.proxy_uri.port} username: #{@s247_http_client.proxy_uri.user ? "configured" : "not configured"}, password: #{@s247_http_client.proxy_uri.password ? "configured" : "not configured"}"
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def get_timestamp(datetime_string)
|
118
|
+
begin
|
119
|
+
# If the date value is in unix format the no need to process the date string
|
120
|
+
if @s247_datetime_format_string.include? 'unix'
|
121
|
+
return (if @s247_datetime_format_string == 'unix' then datetime_string+'000' else datetime_string end)
|
122
|
+
end
|
123
|
+
datetime_string += if !is_year_present then ' '+String(Time.new.year) else '' end
|
124
|
+
if !is_timezone_present
|
125
|
+
@s247_datetime_format_string += '%z'
|
126
|
+
time_zone = String(@s247_tz['hrs'])+':'+String(@s247_tz['mins'])
|
127
|
+
datetime_string += if time_zone.start_with?('-') then time_zone else '+'+time_zone end
|
128
|
+
end
|
129
|
+
datetime_data = DateTime.strptime(datetime_string, @s247_datetime_format_string)
|
130
|
+
return Integer(datetime_data.strftime('%Q'))
|
131
|
+
rescue
|
132
|
+
return 0
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
def parse_lines(lines)
|
137
|
+
parsed_lines = []
|
138
|
+
log_size = 0
|
139
|
+
lines.each do |line|
|
140
|
+
if !line.empty?
|
141
|
+
begin
|
142
|
+
if match = line.match(@s247_custom_regex)
|
143
|
+
log_size += line.bytesize
|
144
|
+
log_fields = match&.named_captures
|
145
|
+
removed_log_size=0
|
146
|
+
@s247_ignored_fields.each do |field_name|
|
147
|
+
removed_log_size += if log_fields.has_key?field_name then log_fields.delete(field_name).bytesize else 0 end
|
148
|
+
end
|
149
|
+
formatted_line = {'_zl_timestamp' => get_timestamp(log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source}
|
150
|
+
formatted_line.merge!(log_fields)
|
151
|
+
parsed_lines.push(formatted_line)
|
152
|
+
log_size -= removed_log_size
|
153
|
+
end
|
154
|
+
rescue Exception => e
|
155
|
+
log.error "Exception in parse_line #{e.backtrace}"
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
return parsed_lines, log_size
|
160
|
+
end
|
161
|
+
|
162
|
+
def is_filters_matched(formatted_line)
|
163
|
+
if @logtype_config.has_key?'filterConfig'
|
164
|
+
@logtype_config['filterConfig'].each do |config|
|
165
|
+
if formatted_line.has_key?config && (filter_config[config]['match'] ^ (filter_config[config]['values'].include?formatted_line[config]))
|
166
|
+
return false
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
return true
|
171
|
+
end
|
172
|
+
|
173
|
+
def get_json_value(obj, key, datatype=nil)
|
174
|
+
if obj != nil && (obj.has_key?key)
|
175
|
+
if datatype and datatype == 'json-object'
|
176
|
+
arr_json = []
|
177
|
+
child_obj = obj[key]
|
178
|
+
if child_obj.class == String
|
179
|
+
child_obj = Yajl::Parser.parse(child_obj.gsub('\\','\\\\'))
|
180
|
+
end
|
181
|
+
child_obj.each do |key, value|
|
182
|
+
arr_json.push({'key' => key, 'value' => String(value)})
|
183
|
+
end
|
184
|
+
return arr_json
|
185
|
+
else
|
186
|
+
return (if obj.has_key?key then obj[key] else obj[key.downcase] end)
|
187
|
+
end
|
188
|
+
elsif key.include?'.'
|
189
|
+
parent_key = key[0..key.index('.')-1]
|
190
|
+
child_key = key[key.index('.')+1..-1]
|
191
|
+
child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
|
192
|
+
if child_obj.class == String
|
193
|
+
child_obj = Yajl::Parser.parse(child_obj.replace('\\','\\\\'))
|
194
|
+
end
|
195
|
+
return get_json_value(child_obj, child_key)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def json_log_parser(lines_read)
|
200
|
+
log_size = 0
|
201
|
+
parsed_lines = []
|
202
|
+
lines_read.each do |line|
|
203
|
+
if !line.empty?
|
204
|
+
current_log_size = 0
|
205
|
+
formatted_line = {}
|
206
|
+
event_obj = Yajl::Parser.parse(line)
|
207
|
+
@logtype_config['jsonPath'].each do |path_obj|
|
208
|
+
value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
|
209
|
+
if value
|
210
|
+
formatted_line[path_obj['name']] = value
|
211
|
+
current_log_size+= String(value).size - (if value.class == Array then value.size*20 else 0 end)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
if is_filters_matched(formatted_line)
|
215
|
+
formatted_line['_zl_timestamp'] = get_timestamp(formatted_line[@logtype_config['dateField']])
|
216
|
+
formatted_line['s247agentuid'] = @log_source
|
217
|
+
parsed_lines.push(formatted_line)
|
218
|
+
log_size+=current_log_size
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
222
|
+
return parsed_lines, log_size
|
223
|
+
end
|
224
|
+
|
225
|
+
def format(tag, time, record)
|
226
|
+
if @valid_logtype && (@log_upload_allowed || (time.to_i - @log_upload_stopped_time > S247_LOG_UPLOAD_CHECK_INTERVAL))
|
227
|
+
[record['message']].to_msgpack
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
def write(chunk)
|
232
|
+
begin
|
233
|
+
events = Array.new
|
234
|
+
chunk.msgpack_each do |record|
|
235
|
+
next if record.empty?
|
236
|
+
events.push record[0]
|
237
|
+
end
|
238
|
+
process_http_events(events)
|
239
|
+
rescue Exception => e
|
240
|
+
log.error "Exception #{e.backtrace}"
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def process_http_events(events)
|
245
|
+
batches = batch_http_events(events)
|
246
|
+
batches.each do |batched_event|
|
247
|
+
formatted_events, log_size = format_http_event_batch(batched_event)
|
248
|
+
formatted_events = gzip_compress(formatted_events)
|
249
|
+
send_logs_to_s247(formatted_events, log_size)
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
def batch_http_events(encoded_events)
|
254
|
+
batches = []
|
255
|
+
current_batch = []
|
256
|
+
current_batch_size = 0
|
257
|
+
encoded_events.each_with_index do |encoded_event, i|
|
258
|
+
current_event_size = encoded_event.bytesize
|
259
|
+
if current_event_size > S247_MAX_RECORD_SIZE
|
260
|
+
encoded_event = encoded_event[0..(S247_MAX_RECORD_SIZE-DD_TRUNCATION_SUFFIX.length)]+DD_TRUNCATION_SUFFIX
|
261
|
+
current_event_size = encoded_event.bytesize
|
262
|
+
end
|
263
|
+
|
264
|
+
if (i > 0 and i % S247_MAX_RECORD_COUNT == 0) or (current_batch_size + current_event_size > S247_MAX_BATCH_SIZE)
|
265
|
+
batches << current_batch
|
266
|
+
current_batch = []
|
267
|
+
current_batch_size = 0
|
268
|
+
end
|
269
|
+
|
270
|
+
current_batch_size += current_event_size
|
271
|
+
current_batch << encoded_event
|
272
|
+
end
|
273
|
+
batches << current_batch
|
274
|
+
batches
|
275
|
+
end
|
276
|
+
|
277
|
+
def format_http_event_batch(events)
|
278
|
+
parsed_lines = []
|
279
|
+
log_size = 0
|
280
|
+
if @logtype_config.has_key?'jsonPath'
|
281
|
+
parsed_lines, log_size = json_log_parser(events)
|
282
|
+
else
|
283
|
+
parsed_lines, log_size = parse_lines(events)
|
284
|
+
end
|
285
|
+
return Yajl.dump(parsed_lines), log_size
|
286
|
+
end
|
287
|
+
|
288
|
+
def gzip_compress(payload)
|
289
|
+
gz = StringIO.new
|
290
|
+
gz.set_encoding("BINARY")
|
291
|
+
z = Zlib::GzipWriter.new(gz, 9)
|
292
|
+
begin
|
293
|
+
z.write(payload)
|
294
|
+
ensure
|
295
|
+
z.close
|
296
|
+
end
|
297
|
+
gz.string
|
298
|
+
end
|
299
|
+
|
300
|
+
def send_logs_to_s247(gzipped_parsed_lines, log_size)
|
301
|
+
request = Net::HTTP::Post.new @uri.request_uri
|
302
|
+
request.body = gzipped_parsed_lines
|
303
|
+
@s247_http_client.override_headers["Log-Size"] = log_size
|
304
|
+
sleep_interval = @retry_interval
|
305
|
+
begin
|
306
|
+
@max_retry.times do |counter|
|
307
|
+
need_retry = false
|
308
|
+
begin
|
309
|
+
response = @s247_http_client.request @uri, request
|
310
|
+
resp_headers = response.each_header.to_h
|
311
|
+
|
312
|
+
if response.code == '200'
|
313
|
+
if resp_headers.has_key?'LOG_LICENSE_EXCEEDS' && resp_headers['LOG_LICENSE_EXCEEDS'] == 'True'
|
314
|
+
log.error "Log license limit exceeds so not able to send logs"
|
315
|
+
@log_upload_allowed = false
|
316
|
+
@log_upload_stopped_time =Time.now.to_i
|
317
|
+
elsif resp_headers.has_key?'BLOCKED_LOGTYPE' && resp_headers['BLOCKED_LOGTYPE'] == 'True'
|
318
|
+
log.error "Max upload limit reached for log type"
|
319
|
+
@log_upload_allowed = false
|
320
|
+
@log_upload_stopped_time =Time.now.to_i
|
321
|
+
elsif resp_headers.has_key?'INVALID_LOGTYPE' && resp_headers['INVALID_LOGTYPE'] == 'True'
|
322
|
+
log.error "Log type not present in this account so stopping log collection"
|
323
|
+
@valid_logtype = false
|
324
|
+
else
|
325
|
+
@log_upload_allowed = true
|
326
|
+
log.debug "Successfully sent logs with size #{gzipped_parsed_lines.size} / #{log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
|
327
|
+
end
|
328
|
+
else
|
329
|
+
log.error "Response Code #{resp_headers} from Site24x7, so retrying (#{counter + 1}/#{@max_retry})"
|
330
|
+
need_retry = true
|
331
|
+
end
|
332
|
+
rescue StandardError => e
|
333
|
+
log.error "Error connecting to Site24x7. exception: #{e} (#{counter + 1}/#{@max_retry})"
|
334
|
+
end
|
335
|
+
|
336
|
+
if need_retry
|
337
|
+
if counter == @max_retry - 1
|
338
|
+
log.error "Could not send your logs after #{max_retry} tries"
|
339
|
+
break
|
340
|
+
end
|
341
|
+
sleep(sleep_interval)
|
342
|
+
sleep_interval *= 2
|
343
|
+
else
|
344
|
+
return
|
345
|
+
end
|
346
|
+
end
|
347
|
+
rescue Exception => e
|
348
|
+
log.error "Exception occurred in sendig logs : #{e}"
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
end
|
metadata
ADDED
@@ -0,0 +1,159 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-site24x7
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Magesh Rajan
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-08-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 2.2.24
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 2.2.24
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 13.0.3
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 13.0.3
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 3.3.7
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 3.3.7
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: yajl-ruby
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.2'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.2'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: webmock
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.5'
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: 3.5.0
|
79
|
+
type: :development
|
80
|
+
prerelease: false
|
81
|
+
version_requirements: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - "~>"
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '3.5'
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 3.5.0
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: fluentd
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: 0.14.10
|
96
|
+
- - "<"
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '2'
|
99
|
+
type: :runtime
|
100
|
+
prerelease: false
|
101
|
+
version_requirements: !ruby/object:Gem::Requirement
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: 0.14.10
|
106
|
+
- - "<"
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '2'
|
109
|
+
- !ruby/object:Gem::Dependency
|
110
|
+
name: net-http-persistent
|
111
|
+
requirement: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - "~>"
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '3.1'
|
116
|
+
type: :runtime
|
117
|
+
prerelease: false
|
118
|
+
version_requirements: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - "~>"
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '3.1'
|
123
|
+
description:
|
124
|
+
email:
|
125
|
+
- magesh.rajan@zohocorp.com
|
126
|
+
executables: []
|
127
|
+
extensions: []
|
128
|
+
extra_rdoc_files: []
|
129
|
+
files:
|
130
|
+
- Gemfile
|
131
|
+
- LICENSE
|
132
|
+
- README.md
|
133
|
+
- Rakefile
|
134
|
+
- fluent-plugin-site24x7.gemspec
|
135
|
+
- lib/fluent/plugin/out_site24x7.rb
|
136
|
+
homepage: https://github.com/site24x7/fluent-plugin-site24x7
|
137
|
+
licenses:
|
138
|
+
- ''
|
139
|
+
metadata: {}
|
140
|
+
post_install_message:
|
141
|
+
rdoc_options: []
|
142
|
+
require_paths:
|
143
|
+
- lib
|
144
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
145
|
+
requirements:
|
146
|
+
- - ">="
|
147
|
+
- !ruby/object:Gem::Version
|
148
|
+
version: '0'
|
149
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: '0'
|
154
|
+
requirements: []
|
155
|
+
rubygems_version: 3.2.24
|
156
|
+
signing_key:
|
157
|
+
specification_version: 4
|
158
|
+
summary: Site24x7 output plugin for Fluent event collector.
|
159
|
+
test_files: []
|