greenhat 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/greenhat/accessors/disk.rb +13 -13
- data/lib/greenhat/accessors/gitlab.rb +75 -0
- data/lib/greenhat/accessors/memory.rb +10 -10
- data/lib/greenhat/accessors/process.rb +4 -0
- data/lib/greenhat/cli.rb +127 -52
- data/lib/greenhat/color.rb +27 -0
- data/lib/greenhat/logbot.rb +9 -9
- data/lib/greenhat/settings.rb +27 -7
- data/lib/greenhat/shell/args.rb +146 -0
- data/lib/greenhat/shell/cat.rb +25 -73
- data/lib/greenhat/shell/color_string.rb +8 -8
- data/lib/greenhat/shell/disk.rb +31 -4
- data/lib/greenhat/shell/faststats.rb +79 -59
- data/lib/greenhat/shell/{filter.rb → filter_help.rb} +48 -33
- data/lib/greenhat/shell/gitlab.rb +61 -2
- data/lib/greenhat/shell/help.rb +96 -15
- data/lib/greenhat/shell/list.rb +46 -0
- data/lib/greenhat/shell/log.rb +77 -104
- data/lib/greenhat/shell/page.rb +11 -5
- data/lib/greenhat/shell/process.rb +29 -17
- data/lib/greenhat/shell/report.rb +27 -43
- data/lib/greenhat/shell/{helper.rb → shell_helper.rb} +204 -187
- data/lib/greenhat/shell.rb +23 -9
- data/lib/greenhat/thing/file_types.rb +20 -1
- data/lib/greenhat/thing/formatters/json_shellwords.rb +0 -3
- data/lib/greenhat/thing/helpers.rb +4 -4
- data/lib/greenhat/thing/kind.rb +9 -2
- data/lib/greenhat/thing/spinner.rb +3 -3
- data/lib/greenhat/thing.rb +3 -3
- data/lib/greenhat/tty/columns.rb +4 -0
- data/lib/greenhat/version.rb +1 -1
- data/lib/greenhat.rb +14 -14
- metadata +22 -18
data/lib/greenhat/shell/help.rb
CHANGED
@@ -1,15 +1,96 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
|
4
|
-
#
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
1
|
+
module GreenHat
|
2
|
+
# Root Level Shell / Splitting Help into its own file
|
3
|
+
module Shell
|
4
|
+
# rubocop:disable Layout/LineLength
|
5
|
+
def self.help
|
6
|
+
puts 'Quickstart'.pastel(:blue)
|
7
|
+
puts " Commands are organized by submodule: #{'log'.pastel(:blue)}, #{'cat'.pastel(:blue)}, #{'faststats'.pastel(:blue)}"
|
8
|
+
puts " Use #{'help'.pastel(:bright_blue)} for available commands in each module"
|
9
|
+
puts
|
10
|
+
|
11
|
+
puts ' Example Commands'
|
12
|
+
puts ' log filter sidekiq/current'.pastel(:yellow)
|
13
|
+
puts ' disk free'.pastel(:yellow)
|
14
|
+
puts
|
15
|
+
|
16
|
+
puts 'Top Level Commands'.pastel(:blue)
|
17
|
+
puts ' report'.pastel(:green)
|
18
|
+
puts ' Show summary report of SOS Report. OS, CPU, Memory, Disk, and etc'
|
19
|
+
puts
|
20
|
+
|
21
|
+
puts ' ps,df,netstat,free,uptime,uname'.pastel(:green)
|
22
|
+
puts ' Show common files from archives / Emulate terminal commands'
|
23
|
+
puts
|
24
|
+
|
25
|
+
puts ' Noisy Output'.pastel(:green)
|
26
|
+
puts " Use #{'quiet'.pastel(:blue)} or to #{'debug'.pastel(:blue)} to toggle greenhat logging"
|
27
|
+
puts
|
28
|
+
|
29
|
+
cli_shortcuts
|
30
|
+
|
31
|
+
puts "See #{'about'.pastel(:bright_blue)} for more details about GreenHat"
|
32
|
+
end
|
33
|
+
# rubocop:enable Layout/LineLength
|
34
|
+
|
35
|
+
def self.cli_shortcuts
|
36
|
+
puts "\u2500".pastel(:cyan) * 25
|
37
|
+
puts 'Nav / Keyboard Shortcuts'.pastel(:blue)
|
38
|
+
puts "\u2500".pastel(:cyan) * 25
|
39
|
+
puts <<~BLOCK
|
40
|
+
| Hotkey | Description |
|
41
|
+
| ------------------- | ----------------------- |
|
42
|
+
| Ctrl + U | Clear Input |
|
43
|
+
| Ctrl + A | Go to beginning |
|
44
|
+
| Ctrl + E | Go to End |
|
45
|
+
| Ctrl + Left/Right | Move left/right by word |
|
46
|
+
| Ctrl + D, Ctrl + Z | Exit |
|
47
|
+
| Ctrl + C, Shift Tab | Up one module |
|
48
|
+
BLOCK
|
49
|
+
puts
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.about
|
53
|
+
puts "\u2500".pastel(:cyan) * 20
|
54
|
+
puts "About GreenHat #{GreenHat::VERSION}".pastel(:yellow)
|
55
|
+
puts "\u2500".pastel(:cyan) * 20
|
56
|
+
|
57
|
+
puts 'TLDR; Put in SOS reports, run commands, and find stuffs'.pastel(:green)
|
58
|
+
puts
|
59
|
+
|
60
|
+
puts <<~BLOCK
|
61
|
+
General overview (OS, Memory, Disk, GitLab)
|
62
|
+
#{'report'.pastel(:bright_cyan)}
|
63
|
+
|
64
|
+
Log Searching
|
65
|
+
#{'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'.pastel(:bright_cyan)}
|
66
|
+
|
67
|
+
Read File(s) across SOS archives
|
68
|
+
#{'cat uptime'.pastel(:bright_cyan)} or #{'cat mount etc/fstab'.pastel(:bright_cyan)}
|
69
|
+
|
70
|
+
BLOCK
|
71
|
+
|
72
|
+
puts 'What it does / How it works'.pastel(:blue)
|
73
|
+
puts
|
74
|
+
puts <<~BLOCK
|
75
|
+
GreenHat is a support utility to enhance troubleshooting with GitLabSOS Reports and log files. Make it easy to find stuff
|
76
|
+
|
77
|
+
Supplied input files are staged, unpacked, identified, and normalized.
|
78
|
+
This enables other utilities to automatically find and present data. (Faststats, report, and etc)
|
79
|
+
|
80
|
+
BLOCK
|
81
|
+
|
82
|
+
puts 'Commands and Submodules'.pastel(:blue)
|
83
|
+
puts
|
84
|
+
puts <<~BLOCK
|
85
|
+
Greenhat is broken down into different "modules". Each module has its own commands. For example: log, cat, and faststats.
|
86
|
+
You can "cd" into or execute commands directly against with their names.
|
87
|
+
|
88
|
+
- Direct: #{'log filter sidekiq/current'.pastel(:cyan)}
|
89
|
+
- Or within: First #{'log'.pastel(:cyan)}, then #{'filter sidekiq/current'.pastel(:cyan)}
|
90
|
+
|
91
|
+
You can find the list of commands and submodules of each with #{'help'.pastel(:yellow)}
|
92
|
+
|
93
|
+
BLOCK
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module GreenHat
|
2
|
+
module ShellHelper
|
3
|
+
# Helper to handle listing of files
|
4
|
+
module List
|
5
|
+
# List Files Helpers
|
6
|
+
def self.list(raw = [], files)
|
7
|
+
filter, flags, _args = Args.parse(raw)
|
8
|
+
|
9
|
+
# Sort
|
10
|
+
files.sort_by!(&:name)
|
11
|
+
|
12
|
+
# Simplified vs Full. Full file name/path / or just file kinds
|
13
|
+
all = flags.key?(:all) || flags.key?(:a)
|
14
|
+
|
15
|
+
# Short & Uniq
|
16
|
+
files.uniq!(&:name) unless all
|
17
|
+
|
18
|
+
# Filter / Pattern
|
19
|
+
files.select! { |f| filter.any? { |x| f.name.include? x } } unless filter.empty?
|
20
|
+
|
21
|
+
# Print
|
22
|
+
files.each do |log|
|
23
|
+
if all
|
24
|
+
puts "- #{log.friendly_name}"
|
25
|
+
else
|
26
|
+
puts "- #{log.name.pastel(:yellow)}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# Unified Help
|
32
|
+
def self.help
|
33
|
+
puts ' ls'.pastel(:green)
|
34
|
+
puts ' List available files'
|
35
|
+
puts ' Options'.pastel(:cyan)
|
36
|
+
puts ' -a, --all, show full file name/path including source'
|
37
|
+
puts ' <string> filter available'
|
38
|
+
puts ' Examples'.pastel(:cyan)
|
39
|
+
puts ' ls -a rails'
|
40
|
+
puts ' ls sys'
|
41
|
+
puts
|
42
|
+
end
|
43
|
+
# ----
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/greenhat/shell/log.rb
CHANGED
@@ -4,171 +4,144 @@ module GreenHat
|
|
4
4
|
# Logs
|
5
5
|
module Log
|
6
6
|
def self.help
|
7
|
-
puts "\u2500".
|
8
|
-
puts "#{'Logs'.
|
9
|
-
puts "\u2500".
|
7
|
+
puts "\u2500".pastel(:cyan) * 20
|
8
|
+
puts "#{'Logs'.pastel(:yellow)} find stuff"
|
9
|
+
puts "\u2500".pastel(:cyan) * 20
|
10
|
+
|
11
|
+
puts 'Command Summary'.pastel(:blue)
|
12
|
+
puts ' filter'.pastel(:green)
|
13
|
+
puts " Primary way for log searching within greenhat. See #{'filter_help'.pastel(:blue)}"
|
14
|
+
puts ' Time, round, slice/except, and/or, stats, uniq, sort'
|
15
|
+
puts
|
10
16
|
|
11
|
-
puts '
|
12
|
-
puts ' show'.colorize(:green)
|
17
|
+
puts ' show'.pastel(:green)
|
13
18
|
puts ' Just print selected logs'
|
14
|
-
puts ' filter'.colorize(:green)
|
15
|
-
puts ' Key/Field Filtering'
|
16
|
-
puts ' - See `filter_help`'
|
17
|
-
puts ' search'.colorize(:green)
|
18
|
-
puts ' General text by entry searching'
|
19
|
-
puts ' - See `search_help`'
|
20
|
-
puts ' ls'.colorize(:green)
|
21
|
-
puts ' List available files'
|
22
19
|
puts
|
23
|
-
end
|
24
20
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
# List Files Helpers
|
30
|
-
def self.list(args = [])
|
31
|
-
all = false
|
32
|
-
all = true if args.include?('-a') || args.include?('--all')
|
33
|
-
|
34
|
-
files = ShellHelper::Log.list
|
21
|
+
puts ' search'.pastel(:green)
|
22
|
+
puts " General full text by file searching. See #{'search_help'.pastel(:blue)}"
|
23
|
+
puts
|
35
24
|
|
36
|
-
|
37
|
-
files.sort_by!(&:name)
|
25
|
+
puts ShellHelper::List.help
|
38
26
|
|
39
|
-
#
|
40
|
-
|
27
|
+
puts "See #{'examples'.pastel(:bright_blue)} for query examples"
|
28
|
+
end
|
41
29
|
|
42
|
-
|
43
|
-
|
44
|
-
if all
|
45
|
-
puts "- #{log.friendly_name}"
|
46
|
-
else
|
47
|
-
puts "- #{log.name.colorize(:yellow)}"
|
48
|
-
end
|
49
|
-
end
|
30
|
+
def self.filter_help
|
31
|
+
ShellHelper::Filter.help
|
50
32
|
end
|
51
33
|
|
52
34
|
def self.ls(args = [])
|
53
|
-
list(args)
|
35
|
+
ShellHelper::List.list(args, ShellHelper::Log.list)
|
54
36
|
end
|
55
37
|
|
56
|
-
def self.show(
|
57
|
-
#
|
58
|
-
|
59
|
-
|
60
|
-
# Convert to Things
|
61
|
-
logs = ShellHelper.find_things(log_list)
|
38
|
+
def self.show(raw = {})
|
39
|
+
# Extract Args
|
40
|
+
files_list, flags, _args = Args.parse(raw)
|
62
41
|
|
63
|
-
|
42
|
+
# Collect Files
|
43
|
+
files = ShellHelper.files(files_list, Thing.all, flags)
|
64
44
|
|
65
|
-
ShellHelper.show
|
45
|
+
ShellHelper.show files.map(&:data).flatten
|
66
46
|
end
|
67
47
|
|
68
48
|
# ========================================================================
|
69
|
-
# Filter
|
49
|
+
# Filter (See Filter Help)
|
70
50
|
# ========================================================================
|
71
|
-
# Supported Params
|
72
|
-
# --or (Filter OR instead of AND)
|
73
|
-
# --total Total Count Entries Only
|
74
|
-
# --project=thingy --exclude_this!=asdf *
|
75
|
-
# --slice: only grab specific fields --slice=path (slice multiple with comma)
|
76
|
-
# --slice=time,path (E.g. log filter --path='mirror/pull' --slice=path,time )
|
77
|
-
# --except: Exclude specific fields (except multiple with comma)
|
78
|
-
# Example: log filter --path='mirror/pull' --except=params
|
79
51
|
def self.default(raw_list)
|
80
52
|
filter(raw_list)
|
81
53
|
end
|
82
54
|
|
83
|
-
def self.filter(
|
55
|
+
def self.filter(raw)
|
84
56
|
# Print Helper
|
85
|
-
if
|
57
|
+
if raw == ['help']
|
86
58
|
filter_help
|
87
59
|
return true
|
88
60
|
end
|
89
61
|
|
90
|
-
#
|
91
|
-
|
62
|
+
# Argument Parsing
|
63
|
+
files, flags, args = Args.parse(raw)
|
92
64
|
|
93
65
|
# Prepare Log List
|
94
|
-
|
95
|
-
|
96
|
-
# AND / OR Filtering
|
97
|
-
filter_type = args.or ? :any? : :all?
|
66
|
+
files = ShellHelper.prepare_list(files, ShellHelper::Log.list, flags)
|
98
67
|
|
99
|
-
results = ShellHelper.filter_start(
|
68
|
+
results = ShellHelper.filter_start(files, flags, args)
|
100
69
|
|
101
|
-
#
|
102
|
-
if
|
70
|
+
# Skip and Print Total if set
|
71
|
+
if flags[:total]
|
103
72
|
ShellHelper.total_count(results)
|
104
73
|
return true
|
105
74
|
end
|
106
75
|
|
107
76
|
# Check Search Results
|
108
77
|
if results.instance_of?(Hash) && results.values.flatten.empty?
|
109
|
-
puts 'No results'.
|
78
|
+
puts 'No results'.pastel(:red)
|
110
79
|
else
|
111
80
|
# This causes the key 'colorized' output to also be included
|
112
|
-
ShellHelper.show(results.to_a.compact.flatten,
|
81
|
+
ShellHelper.show(results.to_a.compact.flatten, flags)
|
113
82
|
end
|
114
83
|
|
115
84
|
# log filter --path='cloud/gitlab-automation' --path='/pull' --all
|
116
85
|
# log filter --project=thingy --other_filter=asdf *
|
117
86
|
rescue StandardError => e
|
118
|
-
LogBot.fatal('Filter', message: e.message
|
87
|
+
LogBot.fatal('Filter', message: e.message)
|
88
|
+
ap e.backtrace
|
119
89
|
end
|
120
90
|
# ========================================================================
|
121
91
|
|
122
92
|
# rubocop:disable Layout/LineLength
|
123
93
|
# TODO: Add a lot more examples
|
124
94
|
def self.examples
|
125
|
-
puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.
|
95
|
+
puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.pastel(:bright_green)
|
126
96
|
puts 'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'
|
127
97
|
puts
|
128
|
-
|
98
|
+
|
99
|
+
puts 'Find 500s only show exceptions'.pastel(:bright_green)
|
129
100
|
puts 'log filter --status=500 --slice=exception.message gitlab-rails/production_json.log'
|
130
101
|
puts
|
102
|
+
|
103
|
+
puts 'Show unique sidekiq queue namespaces. Exclude Specifics'.pastel(:bright_green)
|
104
|
+
puts 'filter sidekiq/current --slice=queue_namespace --uniq=queue_namespace --queue_namespace!=jira_connect --queue_namespace!=hashed_storage'
|
105
|
+
puts
|
106
|
+
|
107
|
+
puts 'Show user,ip from API logs where `meta.user` field is present '.pastel(:bright_green)
|
108
|
+
puts 'gitlab-rails/api_json.log --slice=meta.user,meta.remote_ip --exists=meta.user'
|
109
|
+
puts
|
110
|
+
|
111
|
+
puts 'Count/% occurences for both user and remote ip fields'.pastel(:bright_green)
|
112
|
+
puts 'gitlab-rails/api_json.log --stats=meta.user,meta.remote_ip --exists=meta.user'
|
113
|
+
puts
|
131
114
|
end
|
132
115
|
|
133
116
|
# rubocop:enable Layout/LineLength
|
134
117
|
# ========================================================================
|
135
118
|
# Search (Full Text / String Search)
|
136
119
|
# ========================================================================
|
137
|
-
|
138
|
-
# --text='asdf'
|
139
|
-
# --text!='asdf'
|
140
|
-
# --regex='' # TODO?
|
141
|
-
# --slice=time,path (E.g. log filter --path='mirror/pull' --slice=path,time )
|
142
|
-
# --except: Exclude specific fields (except multiple with comma)
|
143
|
-
|
144
|
-
# --total Total Count Entries Only
|
145
|
-
def self.search(initial_param)
|
120
|
+
def self.search(raw)
|
146
121
|
# Extract Args
|
147
|
-
|
122
|
+
files_list, flags, args = Args.parse(raw)
|
148
123
|
|
149
124
|
# Prepare Log List
|
150
|
-
|
151
|
-
|
152
|
-
# AND / OR Filtering
|
153
|
-
filter_type = args.or ? :any? : :all?
|
125
|
+
files = ShellHelper.prepare_list(files_list)
|
154
126
|
|
155
|
-
results = ShellHelper.search_start(
|
127
|
+
results = ShellHelper.search_start(files, flags, args)
|
156
128
|
|
157
|
-
#
|
158
|
-
if
|
129
|
+
# Skip and Print Total if set
|
130
|
+
if flags[:total]
|
159
131
|
ShellHelper.total_count(results)
|
160
132
|
return true
|
161
133
|
end
|
162
134
|
|
163
135
|
# Check Search Results
|
164
136
|
if results.values.flatten.empty?
|
165
|
-
puts 'No results'.
|
137
|
+
puts 'No results'.pastel(:red)
|
166
138
|
else
|
167
139
|
# This causes the key 'colorized' output to also be included
|
168
|
-
ShellHelper.show(results.to_a.compact.flatten,
|
140
|
+
ShellHelper.show(results.to_a.compact.flatten, flags)
|
169
141
|
end
|
170
142
|
rescue StandardError => e
|
171
|
-
LogBot.fatal('
|
143
|
+
LogBot.fatal('Search', message: e.message)
|
144
|
+
ap e.backtrace
|
172
145
|
end
|
173
146
|
# ========================================================================
|
174
147
|
|
@@ -180,48 +153,48 @@ module GreenHat
|
|
180
153
|
|
181
154
|
# rubocop:disable Metrics/MethodLength
|
182
155
|
def self.search_help
|
183
|
-
puts "\u2500".
|
184
|
-
puts 'Log Search'.
|
185
|
-
puts "\u2500".
|
156
|
+
puts "\u2500".pastel(:cyan) * 20
|
157
|
+
puts 'Log Search'.pastel(:yellow)
|
158
|
+
puts "\u2500".pastel(:cyan) * 20
|
186
159
|
|
187
160
|
puts 'Search will do a full line include or exclude text search'
|
188
161
|
|
189
|
-
puts 'Options'.
|
190
|
-
puts '--text'.
|
162
|
+
puts 'Options'.pastel(:blue)
|
163
|
+
puts '--text'.pastel(:green)
|
191
164
|
puts ' Primary parameter for searching. Include or ! to exclude'
|
192
165
|
puts ' Ex: --text=BuildHooksWorker --text!=start sidekiq/current'
|
193
166
|
puts
|
194
167
|
|
195
|
-
puts '--total'.
|
168
|
+
puts '--total'.pastel(:green)
|
196
169
|
puts ' Print only total count of matching entries'
|
197
170
|
puts
|
198
171
|
|
199
|
-
puts '--slice'.
|
172
|
+
puts '--slice'.pastel(:green)
|
200
173
|
puts ' Extract specific fields from entries (slice multiple with comma)'
|
201
174
|
puts ' Ex: --slice=path or --slice=path,params'
|
202
175
|
puts
|
203
176
|
|
204
|
-
puts '--except'.
|
177
|
+
puts '--except'.pastel(:green)
|
205
178
|
puts ' Exclude specific fields (except multiple with comma)'
|
206
179
|
puts ' Ex: --except=params --except=params,path'
|
207
180
|
puts
|
208
181
|
|
209
|
-
puts '--archive'.
|
210
|
-
puts ' Limit to specific
|
182
|
+
puts '--archive'.pastel(:green)
|
183
|
+
puts ' Limit to specific archive name (inclusive). Matching SOS tar.gz name'
|
211
184
|
puts ' Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
|
212
185
|
puts
|
213
186
|
|
214
|
-
puts '--limit'.
|
187
|
+
puts '--limit'.pastel(:green)
|
215
188
|
puts ' Limit total number of results. Default to half total screen size'
|
216
189
|
puts ' Ex: --limit; --limit=10'
|
217
190
|
puts
|
218
191
|
|
219
|
-
puts 'Search specific logs'.
|
192
|
+
puts 'Search specific logs'.pastel(:blue)
|
220
193
|
puts ' Any non dash parameters will be the log list to search from'
|
221
|
-
puts " Ex: log filter --path=api sidekiq/current (hint: use `#{'ls'.
|
194
|
+
puts " Ex: log filter --path=api sidekiq/current (hint: use `#{'ls'.pastel(:yellow)}` for log names)"
|
222
195
|
puts
|
223
196
|
|
224
|
-
puts 'Example Queries'.
|
197
|
+
puts 'Example Queries'.pastel(:blue)
|
225
198
|
puts 'log search --text=BuildHooksWorker --text!=start sidekiq/current --total'
|
226
199
|
puts 'log search --text=BuildHooksWorker --text!=start --slice=enqueued_at sidekiq/current'
|
227
200
|
puts
|