greenhat 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +64 -0
- data/bin/greenhat +12 -0
- data/lib/greenhat.rb +80 -0
- data/lib/greenhat/accessors/disk.rb +27 -0
- data/lib/greenhat/accessors/logs/production.rb +41 -0
- data/lib/greenhat/accessors/logs/sidekiq.rb +41 -0
- data/lib/greenhat/accessors/memory.rb +46 -0
- data/lib/greenhat/accessors/network.rb +8 -0
- data/lib/greenhat/accessors/process.rb +8 -0
- data/lib/greenhat/archive.rb +108 -0
- data/lib/greenhat/cli.rb +448 -0
- data/lib/greenhat/host.rb +182 -0
- data/lib/greenhat/logbot.rb +86 -0
- data/lib/greenhat/pry_helpers.rb +51 -0
- data/lib/greenhat/settings.rb +51 -0
- data/lib/greenhat/shell.rb +92 -0
- data/lib/greenhat/shell/cat.rb +125 -0
- data/lib/greenhat/shell/disk.rb +68 -0
- data/lib/greenhat/shell/faststats.rb +195 -0
- data/lib/greenhat/shell/gitlab.rb +45 -0
- data/lib/greenhat/shell/help.rb +15 -0
- data/lib/greenhat/shell/helper.rb +514 -0
- data/lib/greenhat/shell/log.rb +344 -0
- data/lib/greenhat/shell/memory.rb +31 -0
- data/lib/greenhat/shell/network.rb +12 -0
- data/lib/greenhat/shell/process.rb +12 -0
- data/lib/greenhat/shell/report.rb +319 -0
- data/lib/greenhat/thing.rb +121 -0
- data/lib/greenhat/thing/file_types.rb +705 -0
- data/lib/greenhat/thing/formatters/api_json.rb +34 -0
- data/lib/greenhat/thing/formatters/bracket_log.rb +44 -0
- data/lib/greenhat/thing/formatters/clean_raw.rb +23 -0
- data/lib/greenhat/thing/formatters/colon_split_strip.rb +12 -0
- data/lib/greenhat/thing/formatters/dotenv.rb +10 -0
- data/lib/greenhat/thing/formatters/format.rb +12 -0
- data/lib/greenhat/thing/formatters/free_m.rb +29 -0
- data/lib/greenhat/thing/formatters/gitlab_ctl_tail.rb +51 -0
- data/lib/greenhat/thing/formatters/gitlab_status.rb +26 -0
- data/lib/greenhat/thing/formatters/json.rb +63 -0
- data/lib/greenhat/thing/formatters/json_shellwords.rb +44 -0
- data/lib/greenhat/thing/formatters/multiline_json.rb +10 -0
- data/lib/greenhat/thing/formatters/raw.rb +18 -0
- data/lib/greenhat/thing/formatters/shellwords.rb +23 -0
- data/lib/greenhat/thing/formatters/table.rb +26 -0
- data/lib/greenhat/thing/formatters/time_json.rb +21 -0
- data/lib/greenhat/thing/formatters/time_shellwords.rb +28 -0
- data/lib/greenhat/thing/formatters/time_space.rb +36 -0
- data/lib/greenhat/thing/helpers.rb +71 -0
- data/lib/greenhat/thing/history.rb +51 -0
- data/lib/greenhat/thing/info_format.rb +193 -0
- data/lib/greenhat/thing/kind.rb +97 -0
- data/lib/greenhat/thing/spinner.rb +52 -0
- data/lib/greenhat/thing/super_log.rb +102 -0
- data/lib/greenhat/tty/custom_line.rb +29 -0
- data/lib/greenhat/tty/line.rb +326 -0
- data/lib/greenhat/tty/reader.rb +110 -0
- data/lib/greenhat/version.rb +3 -0
- data/lib/greenhat/views/css.slim +126 -0
- data/lib/greenhat/views/disk_free.slim +18 -0
- data/lib/greenhat/views/index.slim +51 -0
- data/lib/greenhat/views/info.slim +39 -0
- data/lib/greenhat/views/manifest.slim +22 -0
- data/lib/greenhat/views/memory.slim +18 -0
- data/lib/greenhat/views/netstat.slim +20 -0
- data/lib/greenhat/views/process.slim +21 -0
- data/lib/greenhat/views/systemctl.slim +40 -0
- data/lib/greenhat/views/ulimit.slim +15 -0
- data/lib/greenhat/web.rb +46 -0
- metadata +476 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: fbb51c051ab366c7bb7a172c4a917d7646bd5bdbc00c894217403655130bb2f3
|
4
|
+
data.tar.gz: b4e997388954091b257df23a1f67093490b8c1996635c860b3e462fa3f44213c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 965baad78deb383ed1c1cb569a219439daa4c73adf7fd8b3f2852ee1b9703e758e91b0784ac1c1ac999045dd8a0d871927680d2718829ebcc3740dcc4c334a00
|
7
|
+
data.tar.gz: 17111df14082319b62bca88224ccf4b0b433ac9bcda5d1fe9c922e08477632a6c0831f61dc8615f633f0d8c4b70c58d54fe765c63172c4f4573b6812fc61aee1
|
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# GreenHat
|
2
|
+
|
3
|
+
Experimental SOS and Log Parser for GitLab
|
4
|
+
|
5
|
+
12.x and above only (due to structured logging)
|
6
|
+
|
7
|
+
# Requirements
|
8
|
+
|
9
|
+
`bsdtar` - Used to extract archives
|
10
|
+
|
11
|
+
### Ubuntu
|
12
|
+
|
13
|
+
`apt-get install libarchive-tools`
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
|
17
|
+
While potentially so in the future, this is currently not a published gem. To install:
|
18
|
+
|
19
|
+
```
|
20
|
+
git clone https://gitlab.com/gitlab-com/support/toolbox/greenhat
|
21
|
+
cd greenhat
|
22
|
+
bundle install
|
23
|
+
bundle exec rake install
|
24
|
+
```
|
25
|
+
|
26
|
+
## Usage
|
27
|
+
|
28
|
+
```
|
29
|
+
greenhat sos-archive.tar.gz
|
30
|
+
greenhat production_log.json
|
31
|
+
# launches a console
|
32
|
+
>> help # the program is self-documented through the builtin help command
|
33
|
+
```
|
34
|
+
|
35
|
+
## Testing
|
36
|
+
|
37
|
+
```
|
38
|
+
bundle exec rake
|
39
|
+
```
|
40
|
+
|
41
|
+
## Release Process
|
42
|
+
|
43
|
+
- Update CHANGELOG
|
44
|
+
- Increment `version.rb`
|
45
|
+
- Create MR
|
46
|
+
- Merge to Master
|
47
|
+
|
48
|
+
```
|
49
|
+
# Build and Push
|
50
|
+
bundle exec rake build
|
51
|
+
gem push pkg/greenhat-1.x.x.gem
|
52
|
+
```
|
53
|
+
|
54
|
+
## Contributing
|
55
|
+
|
56
|
+
Bug reports and merge requests are welcome on GitLab at https://gitlab.com/gitlab-com/support/toolbox/greenhat. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
57
|
+
|
58
|
+
## Limitations
|
59
|
+
|
60
|
+
- Shell Helpers cannot use Camel Case
|
61
|
+
|
62
|
+
## License
|
63
|
+
|
64
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/bin/greenhat
ADDED
data/lib/greenhat.rb
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'action_view'
|
4
|
+
require 'benchmark'
|
5
|
+
require 'find'
|
6
|
+
require 'tty-prompt'
|
7
|
+
require 'tty-spinner'
|
8
|
+
require 'tty-table'
|
9
|
+
require 'tty-pager'
|
10
|
+
require 'tty-cursor'
|
11
|
+
require 'tty-reader'
|
12
|
+
require 'hash_dot'
|
13
|
+
require 'oj'
|
14
|
+
require 'slim'
|
15
|
+
require 'active_support'
|
16
|
+
require 'active_support/core_ext'
|
17
|
+
require 'pry'
|
18
|
+
require 'amazing_print'
|
19
|
+
require 'colorize'
|
20
|
+
require 'semantic'
|
21
|
+
require 'tty-progressbar'
|
22
|
+
require 'require_all'
|
23
|
+
require 'warning'
|
24
|
+
require 'dotenv'
|
25
|
+
|
26
|
+
# Custom Gem
|
27
|
+
require 'teron'
|
28
|
+
|
29
|
+
# Oj Settings
|
30
|
+
Oj.default_options = { symbol_keys: true, mode: :compat }
|
31
|
+
|
32
|
+
# Amazing Print
|
33
|
+
AmazingPrint.defaults = {
|
34
|
+
indent: -2,
|
35
|
+
ruby19_syntax: true,
|
36
|
+
index: false,
|
37
|
+
sort_keys: true,
|
38
|
+
sort_vars: true
|
39
|
+
}
|
40
|
+
|
41
|
+
# HashDot Instead of Struct
|
42
|
+
Hash.use_dot_syntax = true
|
43
|
+
Hash.hash_dot_use_default = true
|
44
|
+
|
45
|
+
# Load Required Files
|
46
|
+
require 'greenhat/version'
|
47
|
+
require 'greenhat/cli'
|
48
|
+
require 'greenhat/archive'
|
49
|
+
require 'greenhat/host'
|
50
|
+
require 'greenhat/logbot'
|
51
|
+
require 'greenhat/settings'
|
52
|
+
|
53
|
+
# Formatters - Loads Required Files Automatically
|
54
|
+
require 'greenhat/thing/super_log'
|
55
|
+
require 'greenhat/thing/file_types'
|
56
|
+
require 'greenhat/thing/kind'
|
57
|
+
require 'greenhat/thing/history'
|
58
|
+
|
59
|
+
# Thing
|
60
|
+
require 'greenhat/thing/helpers'
|
61
|
+
require 'greenhat/thing/spinner'
|
62
|
+
require 'greenhat/thing'
|
63
|
+
|
64
|
+
# Shell - Loads Required Files Automatically
|
65
|
+
require 'greenhat/shell'
|
66
|
+
|
67
|
+
# TODO: Confirm
|
68
|
+
# require 'greenhat/thing/log_format'
|
69
|
+
# require 'greenhat/host'
|
70
|
+
# require 'greenhat/web'
|
71
|
+
|
72
|
+
require 'greenhat/pry_helpers'
|
73
|
+
|
74
|
+
## TTY Shims
|
75
|
+
|
76
|
+
require 'greenhat/tty/line'
|
77
|
+
require 'greenhat/tty/reader'
|
78
|
+
require 'greenhat/tty/custom_line'
|
79
|
+
|
80
|
+
Warning.ignore(/The table size exceeds the currently set width/)
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module GreenHat
|
2
|
+
# Sidekiq Log Helpers
|
3
|
+
module Disk
|
4
|
+
def self.data
|
5
|
+
df.map(&:data).flatten.compact
|
6
|
+
end
|
7
|
+
|
8
|
+
# Get Max Size
|
9
|
+
def self.max_padding(disks, key = :mounted_on)
|
10
|
+
list = disks.map do |x|
|
11
|
+
x.slice(key)
|
12
|
+
end
|
13
|
+
|
14
|
+
list.map(&:values).flatten.max_by(&:size).size + 4
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.padding(disks, keys = %i[mounted_on size used avail])
|
18
|
+
keys.map do |key|
|
19
|
+
max_padding(disks, key)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.df
|
24
|
+
Thing.where(name: 'df_h')
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module GreenHat
|
2
|
+
# Sidekiq Log Helpers
|
3
|
+
module Production
|
4
|
+
def self.fast_stats
|
5
|
+
things.each do |thing|
|
6
|
+
puts `fast-stats #{thing.file}`
|
7
|
+
end
|
8
|
+
|
9
|
+
:ok!
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.logs
|
13
|
+
@logs ||= things.map(&:data).flatten.compact
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.raw
|
17
|
+
@raw ||= things.map(&:raw).flatten.compact
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.things
|
21
|
+
Thing.where(name: 'gitlab_rails_production_json_log')
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.errors
|
25
|
+
logs.select { |x| x.severity == 'ERROR' }
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.pages
|
29
|
+
show logs
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.queue_duration(data = nil)
|
33
|
+
data ||= logs
|
34
|
+
data.select { |x| x.key? :enqueued_at }.each do |row|
|
35
|
+
next if row.key? :queue_duration
|
36
|
+
|
37
|
+
row[:queue_duration] = row.enqueued_at - row.created_at
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module GreenHat
|
2
|
+
# Sidekiq Log Helpers
|
3
|
+
module Sidekiq
|
4
|
+
def self.fast_stats
|
5
|
+
things.each do |thing|
|
6
|
+
puts `fast-stats #{thing.file}`
|
7
|
+
end
|
8
|
+
|
9
|
+
:ok!
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.logs
|
13
|
+
@logs ||= things.map(&:data).flatten.compact
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.raw
|
17
|
+
@raw ||= things.map(&:raw).flatten.compact
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.things
|
21
|
+
Thing.where(name: 'sidekiq_current')
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.errors
|
25
|
+
logs.select { |x| x.severity == 'ERROR' }
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.pages
|
29
|
+
show logs
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.queue_duration(data = nil)
|
33
|
+
data ||= logs
|
34
|
+
data.select { |x| x.key? :enqueued_at }.each do |row|
|
35
|
+
next if row.key? :queue_duration
|
36
|
+
|
37
|
+
row[:queue_duration] = row.enqueued_at - row.created_at
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module GreenHat
|
2
|
+
# Sidekiq Log Helpers
|
3
|
+
module Memory
|
4
|
+
def self.free
|
5
|
+
Thing.where(name: 'free_m')
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.percentage(used, total)
|
9
|
+
return 0 if used.to_i.zero?
|
10
|
+
|
11
|
+
# Show at least one bar if below 1%
|
12
|
+
[1, ((used.to_i / total.to_f).round(1) * 100).round].max
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.memory_row(mem)
|
16
|
+
total = mem.total.to_i
|
17
|
+
total_name = number_to_human_size(total.megabytes)
|
18
|
+
used_name = number_to_human_size(mem.used.to_i.megabytes)
|
19
|
+
|
20
|
+
bar = [
|
21
|
+
'|'.colorize(:green) * percentage(mem.used, total),
|
22
|
+
'|'.colorize(:blue) * percentage(mem.shared, total),
|
23
|
+
'|'.colorize(:teal) * percentage(mem.buffcache, total),
|
24
|
+
' ' * percentage(mem.free, total)
|
25
|
+
].join
|
26
|
+
|
27
|
+
# Make Even
|
28
|
+
padding = 125 - bar.uncolorize.size
|
29
|
+
bar += ' ' * padding if padding.positive?
|
30
|
+
|
31
|
+
[
|
32
|
+
mem.kind.ljust(4).colorize(:cyan),
|
33
|
+
' ['.colorize(:light_black),
|
34
|
+
bar.ljust(120),
|
35
|
+
used_name.colorize(:magenta),
|
36
|
+
' / '.colorize(:light_black),
|
37
|
+
total_name.colorize(:blue),
|
38
|
+
']'.colorize(:light_black)
|
39
|
+
].join
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.number_to_human_size(num)
|
43
|
+
ActiveSupport::NumberHelper.number_to_human_size num
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
# Top level namespace
|
2
|
+
module GreenHat
|
3
|
+
# Archive Manipulator
|
4
|
+
module ArchiveLoader
|
5
|
+
def self.load(archive_path)
|
6
|
+
path = "#{$TMP}/#{SecureRandom.uuid}"
|
7
|
+
Dir.mkdir path
|
8
|
+
|
9
|
+
# Initially Copy file into tmpdir
|
10
|
+
FileUtils.cp(archive_path, "#{path}/")
|
11
|
+
|
12
|
+
# Extract Everything
|
13
|
+
loop do
|
14
|
+
archive_list = Find.find(path).reject { |x| File.directory? x }.select { |y| archive? y }
|
15
|
+
break if archive_list.empty?
|
16
|
+
|
17
|
+
archive_list.each do |archive|
|
18
|
+
# Different File Type Handler
|
19
|
+
unpack(archive, path)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Ignore Directories
|
24
|
+
list = Find.find(path).reject { |x| File.directory? x }
|
25
|
+
|
26
|
+
# Ignore Empty Files
|
27
|
+
list.reject! { |x| File.size(x).zero? }
|
28
|
+
|
29
|
+
archive = Archive.new(name: archive_path, path: path)
|
30
|
+
archive.save
|
31
|
+
# file = list[2]
|
32
|
+
# thing = archive.things_create(file: file)
|
33
|
+
# thing.setup
|
34
|
+
|
35
|
+
list.each do |file|
|
36
|
+
# Ensure Valid Content
|
37
|
+
next if missing_command(file)
|
38
|
+
|
39
|
+
# Thread.new do
|
40
|
+
thing = archive.things_create(file: file)
|
41
|
+
thing.setup
|
42
|
+
# end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# Ignore No Commands
|
47
|
+
# rubocop:disable Style/SymbolProc
|
48
|
+
def self.missing_command(file)
|
49
|
+
first_line = File.open(file) { |f| f.readline }
|
50
|
+
['command not found', ': not found'].any? { |x| first_line.include? x }
|
51
|
+
end
|
52
|
+
# rubocop:enable Style/SymbolProc
|
53
|
+
|
54
|
+
# Handle Different Types of Archives
|
55
|
+
def self.unpack(archive_path, path)
|
56
|
+
case File.extname archive_path
|
57
|
+
when '.tar'
|
58
|
+
`bsdtar -xf "#{archive_path}" -C #{path}`
|
59
|
+
FileUtils.rm(archive_path)
|
60
|
+
when '.gz'
|
61
|
+
`gzip -d "#{archive_path}"`
|
62
|
+
when '.s'
|
63
|
+
# Find Original Directory, Split Path, Rename to .gz
|
64
|
+
base_path = archive_path.gsub(File.basename(archive_path), '')
|
65
|
+
FileUtils.mv(archive_path, "#{base_path}/#{File.basename(archive_path, '.s')}.gz")
|
66
|
+
else
|
67
|
+
FileUtils.cp(archive_path, "#{path}/#{archive_path}")
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.archive?(file_name)
|
72
|
+
archive_types.any? do |type|
|
73
|
+
file_name.match?(/(\.#{type})$/)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.archive_types
|
78
|
+
%w[s gz tar]
|
79
|
+
end
|
80
|
+
# -- Archive Loader ----------------------------------------------
|
81
|
+
end
|
82
|
+
# Archive End
|
83
|
+
end
|
84
|
+
|
85
|
+
# Archive Parent
|
86
|
+
class Archive < Teron
|
87
|
+
has_one :host
|
88
|
+
has_many :things
|
89
|
+
|
90
|
+
field :path
|
91
|
+
field :name
|
92
|
+
|
93
|
+
# TODO: Fix from Number of Files / Needed?
|
94
|
+
def friendly_name
|
95
|
+
# Difficult with multiple Archives
|
96
|
+
# File.basename(name, '.tar.gz').gsub('gitlabsos.', '').split('_', 2).first
|
97
|
+
|
98
|
+
File.basename(name, File.extname(name)).gsub('gitlabsos.', '')
|
99
|
+
end
|
100
|
+
|
101
|
+
def inspect
|
102
|
+
"#<Archive name: '#{name}'>"
|
103
|
+
end
|
104
|
+
|
105
|
+
def report
|
106
|
+
GreenHat::Report.new(self)
|
107
|
+
end
|
108
|
+
end
|