hot_catch 0.1.5.1 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/generators/hot_catch/install_generator.rb +2 -1
- data/lib/hot_catch/get_nginx_logs.rb +57 -11
- data/lib/hot_catch/get_system_metrics.rb +27 -0
- data/lib/hot_catch/nginx_worker.rb +1 -1
- data/lib/hot_catch/system_worker.rb +11 -0
- data/lib/hot_catch/version.rb +1 -1
- data/lib/hot_catch.rb +14 -1
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2ffd4e525003d19c6df28482eecd732735d665ba
|
4
|
+
data.tar.gz: 1a7851f99b056f4ebffbf7ef3bc5b434e7a87a22
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5d558f2d46414bd9f35718a1f16c9159bacdfb5f0289662ae3f59474d5d2dc96c38fc8fba63a086f4ea5ad3da564460add40cb03c07fb02f9bbff762d94ae1d7
|
7
|
+
data.tar.gz: 53bc4cf7ab9cabd097871c2838411af0cce13f001de6c5264a0a28b8799df64712baae6db6ef0fc1803c931b85ceee13f9a057631fa46cba315ba1aa34767ebb
|
@@ -36,7 +36,8 @@ module HotCatch
|
|
36
36
|
create_file "config/initializers/hot_catch_sender_logs.rb" do
|
37
37
|
"require 'sidekiq-cron'\n" +
|
38
38
|
"Rails.application.config.sender_logs = HotCatch::MakeHttpsRequest.new\n" +
|
39
|
-
"Sidekiq::Cron::Job.create(name: 'NginxWorker - every 5min', cron: '*/5 * * * *', class: 'NginxWorker')
|
39
|
+
"Sidekiq::Cron::Job.create(name: 'NginxWorker - every 5min', cron: '*/5 * * * *', class: 'NginxWorker')\n" +
|
40
|
+
"Sidekiq::Cron::Job.create(name: 'NginxWorker - every 5min', cron: '*/1 * * * *', class: 'SystemWorker')\n"
|
40
41
|
end
|
41
42
|
end
|
42
43
|
|
@@ -1,18 +1,64 @@
|
|
1
1
|
require 'time'
|
2
|
+
=begin
|
3
|
+
Принцип получения логов (сделано так, чтобы не потерять логи за секунду между интервалами выборок):
|
4
|
+
1. Время последнего получения логов хранится в файле tmp/hot_catch_from_time.txt
|
5
|
+
2. Сами логи хранятся в файле log/nginx.access.log
|
6
|
+
3. Логи берутся от времени последнего получения логов минус одна секунда и до лога
|
7
|
+
время которого меньше текущего минус одна секунда
|
8
|
+
(интервал смещён для точной передачи всех данных за время t - 1 секунда)
|
9
|
+
4. Если нет файла tmp/hot_catch_from_time.txt или он пустой, берутся все логи
|
10
|
+
от начала до последнего лога, время которого меньше текущего минус одна секунда.
|
11
|
+
=end
|
2
12
|
|
3
|
-
def
|
4
|
-
|
5
|
-
|
6
|
-
|
13
|
+
def get_ind_log_from_now_minus_one_second(logs, now)
|
14
|
+
logs.reverse.each_with_index do |elem, index|
|
15
|
+
t = Time.strptime(logs[0 - index - 1].match(/\d{2}\/\w+\/\d{4}:\d{2}:\d{2}:\d{2}/)[0], "%d/%b/%Y:%H:%M:%S")
|
16
|
+
if t < (now - 1)
|
17
|
+
return (logs.size - 1 - index)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
nil # все логи удовлетворяют выборке
|
21
|
+
end
|
7
22
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
23
|
+
def get_nginx_logs(from_time_file = 'tmp/hot_catch_from_time.txt', input_file = 'log/nginx.access.log')
|
24
|
+
from_time = ""
|
25
|
+
if File.exist?(from_time_file) && File.open(from_time_file, 'r'){|file| from_time = file.read } && !from_time.empty?
|
26
|
+
from_time = Time.parse(from_time) - 1
|
27
|
+
logs = ""
|
28
|
+
if File.exist?(input_file) && File.open(input_file, 'r'){|file| logs = file.readlines} && !logs.empty?
|
29
|
+
now = Time.now
|
30
|
+
from_ind = nil
|
31
|
+
for i in 0 ... logs.size
|
32
|
+
t = Time.strptime(logs[0 - i - 1].match(/\d{2}\/\w+\/\d{4}:\d{2}:\d{2}:\d{2}/)[0], "%d/%b/%Y:%H:%M:%S")
|
33
|
+
if from_time <= t
|
34
|
+
from_ind = 0 - i - 1
|
35
|
+
else
|
36
|
+
break
|
37
|
+
end
|
38
|
+
end
|
39
|
+
logs = from_ind ? logs[from_ind..-1] : ""
|
40
|
+
if !logs.empty?
|
41
|
+
to_ind = get_ind_log_from_now_minus_one_second(logs, now)
|
42
|
+
logs = to_ind ? logs[0..to_ind] : ""
|
14
43
|
end
|
44
|
+
|
45
|
+
File.open(from_time_file, 'w'){|file| file.write now}
|
46
|
+
logs.empty? ? logs : logs.join("")
|
47
|
+
end
|
48
|
+
else
|
49
|
+
logs = ""
|
50
|
+
if File.exist?(input_file) && File.open(input_file, 'r'){|file| logs = file.readlines}
|
15
51
|
end
|
16
|
-
|
52
|
+
now = Time.now
|
53
|
+
if !logs.empty?
|
54
|
+
ind = get_ind_log_from_now_minus_one_second(logs, now)
|
55
|
+
puts ind
|
56
|
+
logs = ind ? logs[0..ind] : ""
|
57
|
+
File.open(from_time_file, 'w'){|file| file.write now}
|
58
|
+
logs.join("")
|
59
|
+
end
|
60
|
+
logs
|
17
61
|
end
|
18
62
|
end
|
63
|
+
|
64
|
+
puts get_nginx_logs
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'server_metrics'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
def parse_descriptors_info
|
5
|
+
descriptors = `cat /proc/sys/fs/file-nr`
|
6
|
+
descriptors = descriptors.strip.split("\t").map(&:to_i)
|
7
|
+
{current: descriptors[0], unused: descriptors[1], max: descriptors[2]}
|
8
|
+
end
|
9
|
+
|
10
|
+
def get_system_metrics
|
11
|
+
metrics = [ServerMetrics::Network, ServerMetrics::Cpu, ServerMetrics::Disk, ServerMetrics::Memory]
|
12
|
+
metrics_human_name = ["network", "cpu", "disk", "memory", "system_info", "descriptors", "time"]
|
13
|
+
result = {}
|
14
|
+
|
15
|
+
metrics.each_with_index do |m, index|
|
16
|
+
metric = m.new
|
17
|
+
metric.run
|
18
|
+
sleep 1 if m == ServerMetrics::Network || m == ServerMetrics::Cpu
|
19
|
+
metric.build_report
|
20
|
+
result[metrics_human_name[index]] = metric.data
|
21
|
+
end
|
22
|
+
result[metrics_human_name[-3]] = ServerMetrics::SystemInfo.to_h
|
23
|
+
result[metrics_human_name[-2]] = parse_descriptors_info
|
24
|
+
result[metrics_human_name[-1]] = Time.now
|
25
|
+
|
26
|
+
result
|
27
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'sidekiq'
|
2
|
+
# Отвечает за сбор и отправку логов системы
|
3
|
+
class SystemWorker
|
4
|
+
include Sidekiq::Worker
|
5
|
+
sidekiq_options :retry => false
|
6
|
+
def perform
|
7
|
+
logs = get_system_metrics()
|
8
|
+
sender = HotCatch::MakeHttpsRequest.new
|
9
|
+
sender.system_g_log(logs)
|
10
|
+
end
|
11
|
+
end
|
data/lib/hot_catch/version.rb
CHANGED
data/lib/hot_catch.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
module HotCatch
|
2
2
|
require 'hot_catch/hard_worker'
|
3
3
|
require 'hot_catch/nginx_worker'
|
4
|
+
require 'hot_catch/system_worker'
|
4
5
|
require 'hot_catch/custom_log_subscribers.rb'
|
5
6
|
require 'hot_catch/get_nginx_logs.rb'
|
7
|
+
require 'hot_catch/get_system_metrics'
|
6
8
|
|
7
9
|
require 'uri'
|
8
10
|
require 'net/https'
|
@@ -58,11 +60,22 @@ module HotCatch
|
|
58
60
|
HardWorker.perform_async(body_log, @url, @try_count)
|
59
61
|
end
|
60
62
|
|
63
|
+
def system_g_log(log_data)
|
64
|
+
body_log = {main_hot_catch_log:
|
65
|
+
{
|
66
|
+
"log_data":log_data,
|
67
|
+
"name_app":Rails.application.class.parent_name,
|
68
|
+
"from_log":"System"
|
69
|
+
}
|
70
|
+
}
|
71
|
+
HardWorker.perform_async(body_log, @url, @try_count)
|
72
|
+
end
|
73
|
+
|
61
74
|
def send_log(body_log)
|
62
75
|
response = call(@url, body_log)
|
63
76
|
# если на запрос пришёл ответ, то пришло уведомление об ошибке, которое логируется
|
64
77
|
# Кроме того данное сообщение помещается в очередь и отправляется через какое-то время
|
65
|
-
unless response.empty?
|
78
|
+
unless response.to_s.empty?
|
66
79
|
str = "\n" + (?-*20) + "\n#{Time.now}\nlogs:\n#{body_log.to_s}\nresponse\n:#{response}\n" + (?-*20) + "\n"
|
67
80
|
File.open("log/hot_catch_log_response_errors", 'a'){ |file| file.write str.encode('UTF-8', {
|
68
81
|
:invalid => :replace,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hot_catch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Davhot
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-10-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 0.6.3
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: server_metrics
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
description: Description of Testgem.
|
42
56
|
email:
|
43
57
|
- david.home@mail.ru
|
@@ -53,9 +67,11 @@ files:
|
|
53
67
|
- lib/hot_catch.rb
|
54
68
|
- lib/hot_catch/custom_log_subscribers.rb
|
55
69
|
- lib/hot_catch/get_nginx_logs.rb
|
70
|
+
- lib/hot_catch/get_system_metrics.rb
|
56
71
|
- lib/hot_catch/hard_worker.rb
|
57
72
|
- lib/hot_catch/hot_catch_logger.rb
|
58
73
|
- lib/hot_catch/nginx_worker.rb
|
74
|
+
- lib/hot_catch/system_worker.rb
|
59
75
|
- lib/hot_catch/version.rb
|
60
76
|
- lib/tasks/hot_catch_tasks.rake
|
61
77
|
homepage: https://myhomepage.com
|