greenhat 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,15 +1,96 @@
1
- # module GreenHat
2
- # module Shell
3
- # # CLI Helper
4
- # module Help
5
- # def self.help
6
- # puts 'Available Commands: '
7
- # puts '=> help'
8
-
9
- # ShellCommand.descendants_s.each do |item|
10
- # puts "-> #{item.colorize(:yellow)}"
11
- # end
12
- # end
13
- # end
14
- # end
15
- # end
1
+ module GreenHat
2
+ # Root Level Shell / Splitting Help into its own file
3
+ module Shell
4
+ # rubocop:disable Layout/LineLength
5
+ def self.help
6
+ puts 'Quickstart'.pastel(:blue)
7
+ puts " Commands are organized by submodule: #{'log'.pastel(:blue)}, #{'cat'.pastel(:blue)}, #{'faststats'.pastel(:blue)}"
8
+ puts " Use #{'help'.pastel(:bright_blue)} for available commands in each module"
9
+ puts
10
+
11
+ puts ' Example Commands'
12
+ puts ' log filter sidekiq/current'.pastel(:yellow)
13
+ puts ' disk free'.pastel(:yellow)
14
+ puts
15
+
16
+ puts 'Top Level Commands'.pastel(:blue)
17
+ puts ' report'.pastel(:green)
18
+ puts ' Show summary report of SOS Report. OS, CPU, Memory, Disk, and etc'
19
+ puts
20
+
21
+ puts ' ps,df,netstat,free,uptime,uname'.pastel(:green)
22
+ puts ' Show common files from archives / Emulate terminal commands'
23
+ puts
24
+
25
+ puts ' Noisy Output'.pastel(:green)
26
+ puts " Use #{'quiet'.pastel(:blue)} or to #{'debug'.pastel(:blue)} to toggle greenhat logging"
27
+ puts
28
+
29
+ cli_shortcuts
30
+
31
+ puts "See #{'about'.pastel(:bright_blue)} for more details about GreenHat"
32
+ end
33
+ # rubocop:enable Layout/LineLength
34
+
35
+ def self.cli_shortcuts
36
+ puts "\u2500".pastel(:cyan) * 25
37
+ puts 'Nav / Keyboard Shortcuts'.pastel(:blue)
38
+ puts "\u2500".pastel(:cyan) * 25
39
+ puts <<~BLOCK
40
+ | Hotkey | Description |
41
+ | ------------------- | ----------------------- |
42
+ | Ctrl + U | Clear Input |
43
+ | Ctrl + A | Go to beginning |
44
+ | Ctrl + E | Go to End |
45
+ | Ctrl + Left/Right | Move left/right by word |
46
+ | Ctrl + D, Ctrl + Z | Exit |
47
+ | Ctrl + C, Shift Tab | Up one module |
48
+ BLOCK
49
+ puts
50
+ end
51
+
52
+ def self.about
53
+ puts "\u2500".pastel(:cyan) * 20
54
+ puts "About GreenHat #{GreenHat::VERSION}".pastel(:yellow)
55
+ puts "\u2500".pastel(:cyan) * 20
56
+
57
+ puts 'TLDR; Put in SOS reports, run commands, and find stuffs'.pastel(:green)
58
+ puts
59
+
60
+ puts <<~BLOCK
61
+ General overview (OS, Memory, Disk, GitLab)
62
+ #{'report'.pastel(:bright_cyan)}
63
+
64
+ Log Searching
65
+ #{'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'.pastel(:bright_cyan)}
66
+
67
+ Read File(s) across SOS archives
68
+ #{'cat uptime'.pastel(:bright_cyan)} or #{'cat mount etc/fstab'.pastel(:bright_cyan)}
69
+
70
+ BLOCK
71
+
72
+ puts 'What it does / How it works'.pastel(:blue)
73
+ puts
74
+ puts <<~BLOCK
75
+ GreenHat is a support utility to enhance troubleshooting with GitLabSOS Reports and log files. Make it easy to find stuff
76
+
77
+ Supplied input files are staged, unpacked, identified, and normalized.
78
+ This enables other utilities to automatically find and present data. (Faststats, report, and etc)
79
+
80
+ BLOCK
81
+
82
+ puts 'Commands and Submodules'.pastel(:blue)
83
+ puts
84
+ puts <<~BLOCK
85
+ Greenhat is broken down into different "modules". Each module has its own commands. For example: log, cat, and faststats.
86
+ You can "cd" into or execute commands directly against with their names.
87
+
88
+ - Direct: #{'log filter sidekiq/current'.pastel(:cyan)}
89
+ - Or within: First #{'log'.pastel(:cyan)}, then #{'filter sidekiq/current'.pastel(:cyan)}
90
+
91
+ You can find the list of commands and submodules of each with #{'help'.pastel(:yellow)}
92
+
93
+ BLOCK
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,46 @@
1
+ module GreenHat
2
+ module ShellHelper
3
+ # Helper to handle listing of files
4
+ module List
5
+ # List Files Helpers
6
+ def self.list(raw = [], files)
7
+ filter, flags, _args = Args.parse(raw)
8
+
9
+ # Sort
10
+ files.sort_by!(&:name)
11
+
12
+ # Simplified vs Full. Full file name/path / or just file kinds
13
+ all = flags.key?(:all) || flags.key?(:a)
14
+
15
+ # Short & Uniq
16
+ files.uniq!(&:name) unless all
17
+
18
+ # Filter / Pattern
19
+ files.select! { |f| filter.any? { |x| f.name.include? x } } unless filter.empty?
20
+
21
+ # Print
22
+ files.each do |log|
23
+ if all
24
+ puts "- #{log.friendly_name}"
25
+ else
26
+ puts "- #{log.name.pastel(:yellow)}"
27
+ end
28
+ end
29
+ end
30
+
31
+ # Unified Help
32
+ def self.help
33
+ puts ' ls'.pastel(:green)
34
+ puts ' List available files'
35
+ puts ' Options'.pastel(:cyan)
36
+ puts ' -a, --all, show full file name/path including source'
37
+ puts ' <string> filter available'
38
+ puts ' Examples'.pastel(:cyan)
39
+ puts ' ls -a rails'
40
+ puts ' ls sys'
41
+ puts
42
+ end
43
+ # ----
44
+ end
45
+ end
46
+ end
@@ -4,171 +4,144 @@ module GreenHat
4
4
  # Logs
5
5
  module Log
6
6
  def self.help
7
- puts "\u2500".colorize(:cyan) * 20
8
- puts "#{'Logs'.colorize(:yellow)} find stuff"
9
- puts "\u2500".colorize(:cyan) * 20
7
+ puts "\u2500".pastel(:cyan) * 20
8
+ puts "#{'Logs'.pastel(:yellow)} find stuff"
9
+ puts "\u2500".pastel(:cyan) * 20
10
+
11
+ puts 'Command Summary'.pastel(:blue)
12
+ puts ' filter'.pastel(:green)
13
+ puts " Primary way for log searching within greenhat. See #{'filter_help'.pastel(:blue)}"
14
+ puts ' Time, round, slice/except, and/or, stats, uniq, sort'
15
+ puts
10
16
 
11
- puts 'Command Summary'.colorize(:blue)
12
- puts ' show'.colorize(:green)
17
+ puts ' show'.pastel(:green)
13
18
  puts ' Just print selected logs'
14
- puts ' filter'.colorize(:green)
15
- puts ' Key/Field Filtering'
16
- puts ' - See `filter_help`'
17
- puts ' search'.colorize(:green)
18
- puts ' General text by entry searching'
19
- puts ' - See `search_help`'
20
- puts ' ls'.colorize(:green)
21
- puts ' List available files'
22
19
  puts
23
- end
24
20
 
25
- def self.filter_help
26
- ShellHelper::Filter.help
27
- end
28
-
29
- # List Files Helpers
30
- def self.list(args = [])
31
- all = false
32
- all = true if args.include?('-a') || args.include?('--all')
33
-
34
- files = ShellHelper::Log.list
21
+ puts ' search'.pastel(:green)
22
+ puts " General full text by file searching. See #{'search_help'.pastel(:blue)}"
23
+ puts
35
24
 
36
- # Sort
37
- files.sort_by!(&:name)
25
+ puts ShellHelper::List.help
38
26
 
39
- # Short & Uniq
40
- files.uniq!(&:name) unless all
27
+ puts "See #{'examples'.pastel(:bright_blue)} for query examples"
28
+ end
41
29
 
42
- # Print
43
- files.each do |log|
44
- if all
45
- puts "- #{log.friendly_name}"
46
- else
47
- puts "- #{log.name.colorize(:yellow)}"
48
- end
49
- end
30
+ def self.filter_help
31
+ ShellHelper::Filter.help
50
32
  end
51
33
 
52
34
  def self.ls(args = [])
53
- list(args)
35
+ ShellHelper::List.list(args, ShellHelper::Log.list)
54
36
  end
55
37
 
56
- def self.show(log_list)
57
- # Prepare Log List
58
- log_list = ShellHelper.prepare_list(log_list)
59
-
60
- # Convert to Things
61
- logs = ShellHelper.find_things(log_list)
38
+ def self.show(raw = {})
39
+ # Extract Args
40
+ files_list, flags, _args = Args.parse(raw)
62
41
 
63
- logs.map!(&:data)
42
+ # Collect Files
43
+ files = ShellHelper.files(files_list, Thing.all, flags)
64
44
 
65
- ShellHelper.show logs.flatten
45
+ ShellHelper.show files.map(&:data).flatten
66
46
  end
67
47
 
68
48
  # ========================================================================
69
- # Filter
49
+ # Filter (See Filter Help)
70
50
  # ========================================================================
71
- # Supported Params
72
- # --or (Filter OR instead of AND)
73
- # --total Total Count Entries Only
74
- # --project=thingy --exclude_this!=asdf *
75
- # --slice: only grab specific fields --slice=path (slice multiple with comma)
76
- # --slice=time,path (E.g. log filter --path='mirror/pull' --slice=path,time )
77
- # --except: Exclude specific fields (except multiple with comma)
78
- # Example: log filter --path='mirror/pull' --except=params
79
51
  def self.default(raw_list)
80
52
  filter(raw_list)
81
53
  end
82
54
 
83
- def self.filter(raw_list)
55
+ def self.filter(raw)
84
56
  # Print Helper
85
- if raw_list == ['help']
57
+ if raw == ['help']
86
58
  filter_help
87
59
  return true
88
60
  end
89
61
 
90
- # Extract Args
91
- log_list, opts, args = ShellHelper.param_parse(raw_list)
62
+ # Argument Parsing
63
+ files, flags, args = Args.parse(raw)
92
64
 
93
65
  # Prepare Log List
94
- log_list = ShellHelper.prepare_list(log_list)
95
-
96
- # AND / OR Filtering
97
- filter_type = args.or ? :any? : :all?
66
+ files = ShellHelper.prepare_list(files, ShellHelper::Log.list, flags)
98
67
 
99
- results = ShellHelper.filter_start(log_list, filter_type, args, opts)
68
+ results = ShellHelper.filter_start(files, flags, args)
100
69
 
101
- # Skipo and Print Total if set
102
- if args.total
70
+ # Skip and Print Total if set
71
+ if flags[:total]
103
72
  ShellHelper.total_count(results)
104
73
  return true
105
74
  end
106
75
 
107
76
  # Check Search Results
108
77
  if results.instance_of?(Hash) && results.values.flatten.empty?
109
- puts 'No results'.colorize(:red)
78
+ puts 'No results'.pastel(:red)
110
79
  else
111
80
  # This causes the key 'colorized' output to also be included
112
- ShellHelper.show(results.to_a.compact.flatten, args)
81
+ ShellHelper.show(results.to_a.compact.flatten, flags)
113
82
  end
114
83
 
115
84
  # log filter --path='cloud/gitlab-automation' --path='/pull' --all
116
85
  # log filter --project=thingy --other_filter=asdf *
117
86
  rescue StandardError => e
118
- LogBot.fatal('Filter', message: e.message, backtrace: e.backtrace.first)
87
+ LogBot.fatal('Filter', message: e.message)
88
+ ap e.backtrace
119
89
  end
120
90
  # ========================================================================
121
91
 
122
92
  # rubocop:disable Layout/LineLength
123
93
  # TODO: Add a lot more examples
124
94
  def self.examples
125
- puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.colorize(:light_green)
95
+ puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.pastel(:bright_green)
126
96
  puts 'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'
127
97
  puts
128
- puts 'Find 500s only show exceptions'.colorize(:light_green)
98
+
99
+ puts 'Find 500s only show exceptions'.pastel(:bright_green)
129
100
  puts 'log filter --status=500 --slice=exception.message gitlab-rails/production_json.log'
130
101
  puts
102
+
103
+ puts 'Show unique sidekiq queue namespaces. Exclude Specifics'.pastel(:bright_green)
104
+ puts 'filter sidekiq/current --slice=queue_namespace --uniq=queue_namespace --queue_namespace!=jira_connect --queue_namespace!=hashed_storage'
105
+ puts
106
+
107
+ puts 'Show user,ip from API logs where `meta.user` field is present '.pastel(:bright_green)
108
+ puts 'gitlab-rails/api_json.log --slice=meta.user,meta.remote_ip --exists=meta.user'
109
+ puts
110
+
111
+ puts 'Count/% occurences for both user and remote ip fields'.pastel(:bright_green)
112
+ puts 'gitlab-rails/api_json.log --stats=meta.user,meta.remote_ip --exists=meta.user'
113
+ puts
131
114
  end
132
115
 
133
116
  # rubocop:enable Layout/LineLength
134
117
  # ========================================================================
135
118
  # Search (Full Text / String Search)
136
119
  # ========================================================================
137
- # Supported Params
138
- # --text='asdf'
139
- # --text!='asdf'
140
- # --regex='' # TODO?
141
- # --slice=time,path (E.g. log filter --path='mirror/pull' --slice=path,time )
142
- # --except: Exclude specific fields (except multiple with comma)
143
-
144
- # --total Total Count Entries Only
145
- def self.search(initial_param)
120
+ def self.search(raw)
146
121
  # Extract Args
147
- log_list, opts, args = ShellHelper.param_parse(initial_param)
122
+ files_list, flags, args = Args.parse(raw)
148
123
 
149
124
  # Prepare Log List
150
- log_list = ShellHelper.prepare_list(log_list)
151
-
152
- # AND / OR Filtering
153
- filter_type = args.or ? :any? : :all?
125
+ files = ShellHelper.prepare_list(files_list)
154
126
 
155
- results = ShellHelper.search_start(log_list, filter_type, args, opts)
127
+ results = ShellHelper.search_start(files, flags, args)
156
128
 
157
- # Skipo and Print Total if set
158
- if args.total
129
+ # Skip and Print Total if set
130
+ if flags[:total]
159
131
  ShellHelper.total_count(results)
160
132
  return true
161
133
  end
162
134
 
163
135
  # Check Search Results
164
136
  if results.values.flatten.empty?
165
- puts 'No results'.colorize(:red)
137
+ puts 'No results'.pastel(:red)
166
138
  else
167
139
  # This causes the key 'colorized' output to also be included
168
- ShellHelper.show(results.to_a.compact.flatten, args)
140
+ ShellHelper.show(results.to_a.compact.flatten, flags)
169
141
  end
170
142
  rescue StandardError => e
171
- LogBot.fatal('Filter', message: e.message, backtrace: e.backtrace.first)
143
+ LogBot.fatal('Search', message: e.message)
144
+ ap e.backtrace
172
145
  end
173
146
  # ========================================================================
174
147
 
@@ -180,48 +153,48 @@ module GreenHat
180
153
 
181
154
  # rubocop:disable Metrics/MethodLength
182
155
  def self.search_help
183
- puts "\u2500".colorize(:cyan) * 20
184
- puts 'Log Search'.colorize(:yellow)
185
- puts "\u2500".colorize(:cyan) * 20
156
+ puts "\u2500".pastel(:cyan) * 20
157
+ puts 'Log Search'.pastel(:yellow)
158
+ puts "\u2500".pastel(:cyan) * 20
186
159
 
187
160
  puts 'Search will do a full line include or exclude text search'
188
161
 
189
- puts 'Options'.colorize(:blue)
190
- puts '--text'.colorize(:green)
162
+ puts 'Options'.pastel(:blue)
163
+ puts '--text'.pastel(:green)
191
164
  puts ' Primary parameter for searching. Include or ! to exclude'
192
165
  puts ' Ex: --text=BuildHooksWorker --text!=start sidekiq/current'
193
166
  puts
194
167
 
195
- puts '--total'.colorize(:green)
168
+ puts '--total'.pastel(:green)
196
169
  puts ' Print only total count of matching entries'
197
170
  puts
198
171
 
199
- puts '--slice'.colorize(:green)
172
+ puts '--slice'.pastel(:green)
200
173
  puts ' Extract specific fields from entries (slice multiple with comma)'
201
174
  puts ' Ex: --slice=path or --slice=path,params'
202
175
  puts
203
176
 
204
- puts '--except'.colorize(:green)
177
+ puts '--except'.pastel(:green)
205
178
  puts ' Exclude specific fields (except multiple with comma)'
206
179
  puts ' Ex: --except=params --except=params,path'
207
180
  puts
208
181
 
209
- puts '--archive'.colorize(:green)
210
- puts ' Limit to specific archvie name (inclusive). Matching SOS tar.gz name'
182
+ puts '--archive'.pastel(:green)
183
+ puts ' Limit to specific archive name (inclusive). Matching SOS tar.gz name'
211
184
  puts ' Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
212
185
  puts
213
186
 
214
- puts '--limit'.colorize(:green)
187
+ puts '--limit'.pastel(:green)
215
188
  puts ' Limit total number of results. Default to half total screen size'
216
189
  puts ' Ex: --limit; --limit=10'
217
190
  puts
218
191
 
219
- puts 'Search specific logs'.colorize(:blue)
192
+ puts 'Search specific logs'.pastel(:blue)
220
193
  puts ' Any non dash parameters will be the log list to search from'
221
- puts " Ex: log filter --path=api sidekiq/current (hint: use `#{'ls'.colorize(:yellow)}` for log names)"
194
+ puts " Ex: log filter --path=api sidekiq/current (hint: use `#{'ls'.pastel(:yellow)}` for log names)"
222
195
  puts
223
196
 
224
- puts 'Example Queries'.colorize(:blue)
197
+ puts 'Example Queries'.pastel(:blue)
225
198
  puts 'log search --text=BuildHooksWorker --text!=start sidekiq/current --total'
226
199
  puts 'log search --text=BuildHooksWorker --text!=start --slice=enqueued_at sidekiq/current'
227
200
  puts