greenhat 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +64 -0
- data/bin/greenhat +12 -0
- data/lib/greenhat.rb +80 -0
- data/lib/greenhat/accessors/disk.rb +27 -0
- data/lib/greenhat/accessors/logs/production.rb +41 -0
- data/lib/greenhat/accessors/logs/sidekiq.rb +41 -0
- data/lib/greenhat/accessors/memory.rb +46 -0
- data/lib/greenhat/accessors/network.rb +8 -0
- data/lib/greenhat/accessors/process.rb +8 -0
- data/lib/greenhat/archive.rb +108 -0
- data/lib/greenhat/cli.rb +448 -0
- data/lib/greenhat/host.rb +182 -0
- data/lib/greenhat/logbot.rb +86 -0
- data/lib/greenhat/pry_helpers.rb +51 -0
- data/lib/greenhat/settings.rb +51 -0
- data/lib/greenhat/shell.rb +92 -0
- data/lib/greenhat/shell/cat.rb +125 -0
- data/lib/greenhat/shell/disk.rb +68 -0
- data/lib/greenhat/shell/faststats.rb +195 -0
- data/lib/greenhat/shell/gitlab.rb +45 -0
- data/lib/greenhat/shell/help.rb +15 -0
- data/lib/greenhat/shell/helper.rb +514 -0
- data/lib/greenhat/shell/log.rb +344 -0
- data/lib/greenhat/shell/memory.rb +31 -0
- data/lib/greenhat/shell/network.rb +12 -0
- data/lib/greenhat/shell/process.rb +12 -0
- data/lib/greenhat/shell/report.rb +319 -0
- data/lib/greenhat/thing.rb +121 -0
- data/lib/greenhat/thing/file_types.rb +705 -0
- data/lib/greenhat/thing/formatters/api_json.rb +34 -0
- data/lib/greenhat/thing/formatters/bracket_log.rb +44 -0
- data/lib/greenhat/thing/formatters/clean_raw.rb +23 -0
- data/lib/greenhat/thing/formatters/colon_split_strip.rb +12 -0
- data/lib/greenhat/thing/formatters/dotenv.rb +10 -0
- data/lib/greenhat/thing/formatters/format.rb +12 -0
- data/lib/greenhat/thing/formatters/free_m.rb +29 -0
- data/lib/greenhat/thing/formatters/gitlab_ctl_tail.rb +51 -0
- data/lib/greenhat/thing/formatters/gitlab_status.rb +26 -0
- data/lib/greenhat/thing/formatters/json.rb +63 -0
- data/lib/greenhat/thing/formatters/json_shellwords.rb +44 -0
- data/lib/greenhat/thing/formatters/multiline_json.rb +10 -0
- data/lib/greenhat/thing/formatters/raw.rb +18 -0
- data/lib/greenhat/thing/formatters/shellwords.rb +23 -0
- data/lib/greenhat/thing/formatters/table.rb +26 -0
- data/lib/greenhat/thing/formatters/time_json.rb +21 -0
- data/lib/greenhat/thing/formatters/time_shellwords.rb +28 -0
- data/lib/greenhat/thing/formatters/time_space.rb +36 -0
- data/lib/greenhat/thing/helpers.rb +71 -0
- data/lib/greenhat/thing/history.rb +51 -0
- data/lib/greenhat/thing/info_format.rb +193 -0
- data/lib/greenhat/thing/kind.rb +97 -0
- data/lib/greenhat/thing/spinner.rb +52 -0
- data/lib/greenhat/thing/super_log.rb +102 -0
- data/lib/greenhat/tty/custom_line.rb +29 -0
- data/lib/greenhat/tty/line.rb +326 -0
- data/lib/greenhat/tty/reader.rb +110 -0
- data/lib/greenhat/version.rb +3 -0
- data/lib/greenhat/views/css.slim +126 -0
- data/lib/greenhat/views/disk_free.slim +18 -0
- data/lib/greenhat/views/index.slim +51 -0
- data/lib/greenhat/views/info.slim +39 -0
- data/lib/greenhat/views/manifest.slim +22 -0
- data/lib/greenhat/views/memory.slim +18 -0
- data/lib/greenhat/views/netstat.slim +20 -0
- data/lib/greenhat/views/process.slim +21 -0
- data/lib/greenhat/views/systemctl.slim +40 -0
- data/lib/greenhat/views/ulimit.slim +15 -0
- data/lib/greenhat/web.rb +46 -0
- metadata +476 -0
@@ -0,0 +1,195 @@
|
|
1
|
+
module GreenHat
|
2
|
+
# CLI Helper
|
3
|
+
module Shell
|
4
|
+
# Logs
|
5
|
+
module Faststats
|
6
|
+
# rubocop:disable Metrics/MethodLength)
|
7
|
+
def self.help
|
8
|
+
puts "\u2500".colorize(:cyan) * 25
|
9
|
+
puts "Gimme #{'Performance Stats'.colorize(:yellow)}"
|
10
|
+
puts "\u2500".colorize(:cyan) * 25
|
11
|
+
|
12
|
+
puts 'General'.colorize(:blue)
|
13
|
+
puts " Any double dash arguments (e.g. #{'--color-output'.colorize(:green)}) are passed directly to fast-stats"
|
14
|
+
puts " See #{'`fast-stats --help`'.colorize(:light_black)} for all available options"
|
15
|
+
puts
|
16
|
+
|
17
|
+
puts 'Common Options'.colorize(:blue)
|
18
|
+
puts ' --raw'.colorize(:green)
|
19
|
+
puts ' Do not use less/paging'
|
20
|
+
puts ' --display'.colorize(:green)
|
21
|
+
puts ' Show percentage of totals and/or actual durations (value, perc, both, default: both)'
|
22
|
+
puts ' --sort'.colorize(:green)
|
23
|
+
puts ' count,fail,max,median,min,p95,p99,rps,score'
|
24
|
+
puts ' --limit'.colorize(:green)
|
25
|
+
puts ' The number of rows to print'
|
26
|
+
puts ' --verbose'.colorize(:green)
|
27
|
+
puts ' Prints the component details of the maximum, P99, P95, and median events by total duration for each'
|
28
|
+
puts
|
29
|
+
|
30
|
+
puts 'Commands'.colorize(:blue)
|
31
|
+
puts 'ls'.colorize(:green)
|
32
|
+
puts ' List available files'
|
33
|
+
puts ' Options'
|
34
|
+
puts ' -a, --all, show all files including source'
|
35
|
+
puts
|
36
|
+
|
37
|
+
puts '<file names+>'.colorize(:green)
|
38
|
+
puts ' Print any file names'
|
39
|
+
puts ' Ex: `gitaly/current`'
|
40
|
+
puts ' Ex: `gitlab-rails/api_json.log gitlab-rails/production_json.log --raw`'
|
41
|
+
puts
|
42
|
+
|
43
|
+
puts 'top'.colorize(:green)
|
44
|
+
puts ' Print a summary of top items in a category'
|
45
|
+
puts
|
46
|
+
|
47
|
+
puts 'errors'.colorize(:green)
|
48
|
+
puts ' Show summary of errors captured in log'
|
49
|
+
end
|
50
|
+
# rubocop:enable Metrics/MethodLength)
|
51
|
+
|
52
|
+
# List Files Helpers
|
53
|
+
def self.list(args = [])
|
54
|
+
all = false
|
55
|
+
all = true if args.include?('-a') || args.include?('--all')
|
56
|
+
|
57
|
+
files = ShellHelper::Faststats.things
|
58
|
+
|
59
|
+
# Sort
|
60
|
+
files.sort_by!(&:name)
|
61
|
+
|
62
|
+
# Short & Uniq
|
63
|
+
files.uniq!(&:name) unless all
|
64
|
+
|
65
|
+
# Print
|
66
|
+
files.each do |log|
|
67
|
+
if all
|
68
|
+
puts "- #{log.friendly_name}"
|
69
|
+
else
|
70
|
+
puts "- #{log.name.colorize(:yellow)}"
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.ls(args = [])
|
76
|
+
list(args)
|
77
|
+
end
|
78
|
+
|
79
|
+
# Vanilla Fast Stats
|
80
|
+
def self.default(file_list, _other = nil)
|
81
|
+
# Extract Args
|
82
|
+
file_list, opts, args = ShellHelper.param_parse(file_list)
|
83
|
+
|
84
|
+
cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
|
85
|
+
cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
|
86
|
+
|
87
|
+
# Prepare List
|
88
|
+
file_list = ShellHelper.prepare_list(file_list, ShellHelper::Faststats.things)
|
89
|
+
|
90
|
+
# Convert to Things
|
91
|
+
files = ShellHelper.find_things(file_list)
|
92
|
+
|
93
|
+
results = ShellHelper.file_process(files) do |file|
|
94
|
+
[
|
95
|
+
file.friendly_name,
|
96
|
+
`fast-stats #{cmd} #{file.file}`.split("\n"),
|
97
|
+
"\n"
|
98
|
+
]
|
99
|
+
end
|
100
|
+
|
101
|
+
ShellHelper.show(results.flatten, args)
|
102
|
+
end
|
103
|
+
|
104
|
+
# ========================================================================
|
105
|
+
# ===== [ Fast Stats - Top ] ====================
|
106
|
+
# -------------------------------------------------
|
107
|
+
# Options:
|
108
|
+
# --display, value,perc,both
|
109
|
+
# Show percentage of totals, actual durations, or both. Defaults to both.
|
110
|
+
# --limit=X
|
111
|
+
# --sort=count,fail,max,median,min,p95,p99,rps,score
|
112
|
+
# ========================================================================
|
113
|
+
def self.top(log_list)
|
114
|
+
# Extract Args
|
115
|
+
log_list, opts, args = ShellHelper.param_parse(log_list)
|
116
|
+
|
117
|
+
# Default to color output
|
118
|
+
args['color-output'] ||= true
|
119
|
+
|
120
|
+
cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
|
121
|
+
cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
|
122
|
+
|
123
|
+
# Prepare Log List
|
124
|
+
log_list = ShellHelper.prepare_list(log_list, ShellHelper::Faststats.things)
|
125
|
+
|
126
|
+
# Convert to Things
|
127
|
+
files = ShellHelper.find_things(log_list)
|
128
|
+
|
129
|
+
results = ShellHelper.file_process(files) do |file|
|
130
|
+
[
|
131
|
+
file.friendly_name,
|
132
|
+
`fast-stats top #{cmd} #{file.file}`.split("\n"),
|
133
|
+
"\n"
|
134
|
+
]
|
135
|
+
end
|
136
|
+
|
137
|
+
ShellHelper.show(results.flatten, args)
|
138
|
+
end
|
139
|
+
# ========================================================================
|
140
|
+
|
141
|
+
# ========================================================================
|
142
|
+
# ===== [ Fast Stats - Errors ] ====================
|
143
|
+
# ========================================================================
|
144
|
+
def self.errors(log_list)
|
145
|
+
# Extract Args
|
146
|
+
log_list, opts, args = ShellHelper.param_parse(log_list)
|
147
|
+
|
148
|
+
# Default to color output
|
149
|
+
args['color-output'] ||= true
|
150
|
+
|
151
|
+
cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
|
152
|
+
cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
|
153
|
+
|
154
|
+
# Prepare Log List
|
155
|
+
log_list = ShellHelper.prepare_list(log_list, ShellHelper::Faststats.things)
|
156
|
+
|
157
|
+
# Convert to Things
|
158
|
+
files = ShellHelper.find_things(log_list)
|
159
|
+
|
160
|
+
results = ShellHelper.file_process(files) do |file|
|
161
|
+
[
|
162
|
+
file.friendly_name,
|
163
|
+
`fast-stats errors #{cmd} #{file.file}`.split("\n"),
|
164
|
+
"\n"
|
165
|
+
]
|
166
|
+
end
|
167
|
+
|
168
|
+
ShellHelper.show(results.flatten, args)
|
169
|
+
end
|
170
|
+
# ========================================================================
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
module GreenHat
|
176
|
+
module ShellHelper
|
177
|
+
# Hide from Commands
|
178
|
+
module Faststats
|
179
|
+
def self.files
|
180
|
+
%w[
|
181
|
+
production_json
|
182
|
+
api_json
|
183
|
+
gitaly/current
|
184
|
+
sidekiq/current
|
185
|
+
]
|
186
|
+
end
|
187
|
+
|
188
|
+
def self.things
|
189
|
+
Thing.all.select do |thing|
|
190
|
+
files.any? { |x| thing.name.include? x }
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module GreenHat
|
2
|
+
# CLI Helper
|
3
|
+
module Shell
|
4
|
+
# Logs
|
5
|
+
module Gitlab
|
6
|
+
def self.version
|
7
|
+
Thing.where(type: 'gitlab/version-manifest.json').each do |file|
|
8
|
+
next unless file.data
|
9
|
+
|
10
|
+
puts file.friendly_name
|
11
|
+
puts Semantic::Version.new(file.data.build_version).to_s.colorize(:yellow)
|
12
|
+
puts
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.status
|
17
|
+
Thing.where(type: 'gitlab_status').each do |file|
|
18
|
+
next unless file.data
|
19
|
+
|
20
|
+
puts file.friendly_name
|
21
|
+
|
22
|
+
file.data.each_value do |services|
|
23
|
+
output = services.each_with_index.flat_map do |service, i|
|
24
|
+
color = service.status == 'run' ? :green : :red
|
25
|
+
|
26
|
+
pad = i.zero? ? 18 : 0
|
27
|
+
|
28
|
+
[
|
29
|
+
"#{service.status}:",
|
30
|
+
service.name.ljust(pad).colorize(color),
|
31
|
+
"#{service.pid_uptime};".ljust(pad)
|
32
|
+
|
33
|
+
]
|
34
|
+
end
|
35
|
+
|
36
|
+
puts output.join(' ')
|
37
|
+
end
|
38
|
+
|
39
|
+
puts
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
# -----
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# module GreenHat
|
2
|
+
# module Shell
|
3
|
+
# # CLI Helper
|
4
|
+
# module Help
|
5
|
+
# def self.help
|
6
|
+
# puts 'Available Commands: '
|
7
|
+
# puts '=> help'
|
8
|
+
|
9
|
+
# ShellCommand.descendants_s.each do |item|
|
10
|
+
# puts "-> #{item.colorize(:yellow)}"
|
11
|
+
# end
|
12
|
+
# end
|
13
|
+
# end
|
14
|
+
# end
|
15
|
+
# end
|
@@ -0,0 +1,514 @@
|
|
1
|
+
module GreenHat
|
2
|
+
# Common Helpers
|
3
|
+
# rubocop:disable Metrics/ModuleLength
|
4
|
+
module ShellHelper
|
5
|
+
# Generic Parameter Parsing
|
6
|
+
def self.param_parse(params)
|
7
|
+
# Turn Params into Hash
|
8
|
+
opts = params.flat_map { |param| param_opt_scan(param) }
|
9
|
+
|
10
|
+
# Collect and Remove Args
|
11
|
+
args = params.each_with_object({}) { |param, obj| param_arg_scan(param, obj) }
|
12
|
+
|
13
|
+
# Move Special Arguments
|
14
|
+
opts.reject! do |opt|
|
15
|
+
if param_special_opts.include? opt.field
|
16
|
+
args[opt.field] = opt.value.split(',').map(&:to_sym)
|
17
|
+
true
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
# Remove Opts and Args
|
22
|
+
params.reject! do |param|
|
23
|
+
opt_field_remove?(opts, param) || arg_field_remove?(args, param)
|
24
|
+
end
|
25
|
+
|
26
|
+
[params, opts, args]
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.opt_field_remove?(opts, param)
|
30
|
+
opts.any? do |opt|
|
31
|
+
param.include? "--#{opt.field}#{opt.bang ? '!' : nil}=#{opt.value}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.arg_field_remove?(args, param)
|
36
|
+
args.keys.any? { |field| param.include? "--#{field}" }
|
37
|
+
end
|
38
|
+
|
39
|
+
# Def Param Scan (Split -- values into keys)
|
40
|
+
def self.param_opt_scan(param)
|
41
|
+
param.scan(/--([^=]+)=(.*)/).map do |field, value|
|
42
|
+
bang = false
|
43
|
+
if field.include? '!'
|
44
|
+
field.delete!('!')
|
45
|
+
bang = true
|
46
|
+
end
|
47
|
+
|
48
|
+
{ field: field.to_sym, value: value, bang: bang }
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Params that should be set as Args
|
53
|
+
# Comma Delimited
|
54
|
+
# --slice=thing,thing2
|
55
|
+
# --except=time,params
|
56
|
+
# --round=2
|
57
|
+
|
58
|
+
def self.param_special_opts
|
59
|
+
%i[slice except stats uniq pluck round archive start end sort]
|
60
|
+
end
|
61
|
+
|
62
|
+
# Parameter Extraction
|
63
|
+
# Special Opts: --slice, --except
|
64
|
+
def self.param_arg_scan(param, obj)
|
65
|
+
# TODO: Why is capture group doing two arrays
|
66
|
+
param.scan(/--([^=]+)$/).flatten.each do |field|
|
67
|
+
obj[field.to_sym] = param_arg_defaults(field.to_sym)
|
68
|
+
end
|
69
|
+
|
70
|
+
obj
|
71
|
+
end
|
72
|
+
|
73
|
+
# Arg Defaults
|
74
|
+
def self.param_arg_defaults(field)
|
75
|
+
case field
|
76
|
+
when *param_special_opts then []
|
77
|
+
when :round then 2
|
78
|
+
else
|
79
|
+
true
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# Use File Process for Output
|
84
|
+
def self.file_output(files)
|
85
|
+
results = file_process(files) do |file|
|
86
|
+
[
|
87
|
+
file.friendly_name,
|
88
|
+
file.output(false),
|
89
|
+
"\n"
|
90
|
+
]
|
91
|
+
end
|
92
|
+
|
93
|
+
ShellHelper.show(results.flatten)
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.file_process(files, &block)
|
97
|
+
files.map do |file|
|
98
|
+
next if file.output(false).empty?
|
99
|
+
|
100
|
+
block.call(file)
|
101
|
+
end.flatten
|
102
|
+
end
|
103
|
+
|
104
|
+
# Pagination Helper
|
105
|
+
def self.page(data)
|
106
|
+
TTY::Pager.page do |pager|
|
107
|
+
data.flatten.each do |output|
|
108
|
+
pager.write("\n#{output}") # write line to the pager
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
# Show Data / Auto Paginate Helper
|
114
|
+
def self.show(data, args = {})
|
115
|
+
# If Block of String
|
116
|
+
if data.instance_of?(String)
|
117
|
+
TTY::Pager.page data
|
118
|
+
return true
|
119
|
+
end
|
120
|
+
|
121
|
+
# If raw just print out
|
122
|
+
if args.raw
|
123
|
+
puts data.join("\n")
|
124
|
+
return true
|
125
|
+
end
|
126
|
+
|
127
|
+
# Check if content needs to paged, or if auto_height is off
|
128
|
+
if !args.page && (data.sum(&:size) < TTY::Screen.height)
|
129
|
+
puts data.map { |x| entry_show(x, args) }.compact.join("\n")
|
130
|
+
return true
|
131
|
+
end
|
132
|
+
|
133
|
+
# Default Pager
|
134
|
+
TTY::Pager.page do |pager|
|
135
|
+
data.each do |entry|
|
136
|
+
output = entry_show(entry, args)
|
137
|
+
|
138
|
+
next if output.blank?
|
139
|
+
|
140
|
+
pager.write("\n#{output}") # write line to the pager
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
# Entry Shower
|
146
|
+
def self.entry_show(entry, args)
|
147
|
+
case entry
|
148
|
+
when Hash then render_table(entry, args)
|
149
|
+
when String, Float, Integer then entry
|
150
|
+
when Array
|
151
|
+
entry.map { |x| x.ai(multiline: false) }.join("\n")
|
152
|
+
else
|
153
|
+
LogBot.warn('Shell Show', "Unknown #{entry.class}")
|
154
|
+
nil
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
# Print the Table in a Nice way
|
159
|
+
def self.render_table(entry, args)
|
160
|
+
table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
|
161
|
+
|
162
|
+
table.render(:unicode, padding: [0, 1, 0, 1]) do |renderer|
|
163
|
+
renderer.border.style = :cyan
|
164
|
+
|
165
|
+
renderer.filter = lambda do |val, _row_index, col_index|
|
166
|
+
if col_index == 1
|
167
|
+
if val.numeric?
|
168
|
+
# TODO: Better Casting?
|
169
|
+
val = val.to_f.round(args.round.first.to_s.to_i) if args.round
|
170
|
+
val.to_s.colorize(:blue)
|
171
|
+
else
|
172
|
+
val.to_s
|
173
|
+
end
|
174
|
+
else
|
175
|
+
val.to_s
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
# Fall Back to Amazing Inspect
|
181
|
+
rescue StandardError => e
|
182
|
+
LogBot.warn('Table', message: e.message, backtrace: e.backtrace.first) if ENV['DEBUG']
|
183
|
+
|
184
|
+
[
|
185
|
+
entry.ai,
|
186
|
+
('_' * (TTY::Screen.width / 3)).colorize(:cyan),
|
187
|
+
"\n"
|
188
|
+
].join("\n")
|
189
|
+
end
|
190
|
+
|
191
|
+
# Main Entry Point for Filtering
|
192
|
+
def self.filter_start(log_list, filter_type, args, opts)
|
193
|
+
# Convert to Things
|
194
|
+
logs = ShellHelper.find_things(log_list, args).select(&:processed?)
|
195
|
+
|
196
|
+
# Ignore Archive/Host Dividers
|
197
|
+
if args[:combine]
|
198
|
+
results = logs.reject(&:blank?).map(&:data).flatten.compact
|
199
|
+
ShellHelper.filter(results, filter_type, args, opts)
|
200
|
+
else
|
201
|
+
# Iterate and Preserve Archive/Host Index
|
202
|
+
logs.each_with_object({}) do |log, obj|
|
203
|
+
# Ignore Empty Results / No Thing
|
204
|
+
next if log&.blank?
|
205
|
+
|
206
|
+
obj[log.friendly_name] = ShellHelper.filter(log.data, filter_type, args, opts)
|
207
|
+
|
208
|
+
obj
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# Filter Logic
|
214
|
+
# TODO: Simplify
|
215
|
+
# rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
216
|
+
def self.filter(data, type = :all?, args = {}, opts = {})
|
217
|
+
results = data.clone.flatten.compact
|
218
|
+
results.select! do |row|
|
219
|
+
opts.send(type) do |opt|
|
220
|
+
filter_row_key(row, opt)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
# Time Filtering
|
225
|
+
results = filter_time(results, args) if args[:start] || args[:end]
|
226
|
+
|
227
|
+
# Strip Results if Slice is defined
|
228
|
+
results = filter_slice(results, args[:slice]) if args[:slice]
|
229
|
+
|
230
|
+
# Strip Results if Except is defined
|
231
|
+
results = filter_except(results, args[:except]) if args[:except]
|
232
|
+
|
233
|
+
# Remove Blank from either slice or except
|
234
|
+
results.reject!(&:empty?)
|
235
|
+
|
236
|
+
# Sort
|
237
|
+
results.sort_by! { |x| x.slice(*args[:sort]).values } if args[:sort]
|
238
|
+
|
239
|
+
# JSON Formatting
|
240
|
+
results = results.map { |x| Oj.dump(x) } if args[:json]
|
241
|
+
|
242
|
+
# Show Unique Only
|
243
|
+
results = filter_uniq(results, args[:uniq]) if args.key?(:uniq)
|
244
|
+
|
245
|
+
# Reverse
|
246
|
+
results.reverse! if args[:reverse]
|
247
|
+
|
248
|
+
# Count occurrences / Skip Results
|
249
|
+
return filter_stats(results, args[:stats]) if args[:stats]
|
250
|
+
|
251
|
+
# Pluck
|
252
|
+
results = filter_pluck(results, args[:pluck]) if args.key?(:pluck)
|
253
|
+
|
254
|
+
results
|
255
|
+
end
|
256
|
+
# rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
257
|
+
|
258
|
+
# Filter Start and End Times
|
259
|
+
# rubocop:disable Metrics/MethodLength
|
260
|
+
# TODO: This is a bit icky, simplify/dry
|
261
|
+
def self.filter_time(results, args)
|
262
|
+
if args.start
|
263
|
+
begin
|
264
|
+
time_start = Time.parse(args.start.first.to_s)
|
265
|
+
|
266
|
+
results.select! do |x|
|
267
|
+
if x.time
|
268
|
+
time_start < x.time
|
269
|
+
else
|
270
|
+
true
|
271
|
+
end
|
272
|
+
end
|
273
|
+
rescue StandardError
|
274
|
+
puts 'Unable to Process Start Time Filter'.colorize(:red)
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
if args.end
|
279
|
+
begin
|
280
|
+
time_start = Time.parse(args.end.first.to_s)
|
281
|
+
|
282
|
+
results.select! do |x|
|
283
|
+
if x.time
|
284
|
+
time_start > x.time
|
285
|
+
else
|
286
|
+
true
|
287
|
+
end
|
288
|
+
end
|
289
|
+
rescue StandardError
|
290
|
+
puts 'Unable to Process End Time Filter'.colorize(:red)
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
results
|
295
|
+
end
|
296
|
+
# rubocop:enable Metrics/MethodLength
|
297
|
+
|
298
|
+
def self.filter_except(results, except)
|
299
|
+
# Avoid Empty Results
|
300
|
+
if slice.empty?
|
301
|
+
filter_empty_arg('slice')
|
302
|
+
return results
|
303
|
+
end
|
304
|
+
|
305
|
+
results.map { |row| row.except(*except) }
|
306
|
+
end
|
307
|
+
|
308
|
+
def self.filter_slice(results, slice)
|
309
|
+
# Avoid Empty Results
|
310
|
+
if slice.empty?
|
311
|
+
filter_empty_arg('slice')
|
312
|
+
return results
|
313
|
+
end
|
314
|
+
|
315
|
+
results.map { |row| row.slice(*slice) }
|
316
|
+
end
|
317
|
+
|
318
|
+
def self.filter_pluck(results, pluck)
|
319
|
+
# Avoid Empty Results
|
320
|
+
if pluck.empty?
|
321
|
+
filter_empty_arg('pluck')
|
322
|
+
return results
|
323
|
+
end
|
324
|
+
|
325
|
+
results.map { |x| x.slice(*pluck).values }.flatten
|
326
|
+
end
|
327
|
+
|
328
|
+
def self.filter_uniq(results, unique)
|
329
|
+
# Avoid Empty Results
|
330
|
+
if unique.empty?
|
331
|
+
filter_empty_arg('uniq')
|
332
|
+
return results
|
333
|
+
end
|
334
|
+
|
335
|
+
unique.map do |field|
|
336
|
+
results.uniq { |x| x[field] }
|
337
|
+
end.flatten
|
338
|
+
end
|
339
|
+
|
340
|
+
def self.filter_stats(results, stats)
|
341
|
+
# Avoid Empty Results
|
342
|
+
if stats.empty?
|
343
|
+
filter_empty_arg('stats')
|
344
|
+
return results
|
345
|
+
end
|
346
|
+
|
347
|
+
stats.map do |stat|
|
348
|
+
occurrences = filter_count_occurrences(results, stat)
|
349
|
+
occurrences.sort_by(&:last).to_h
|
350
|
+
end
|
351
|
+
end
|
352
|
+
|
353
|
+
# Helper to Count occurrences
|
354
|
+
def self.filter_count_occurrences(results, stat)
|
355
|
+
results.each_with_object(Hash.new(0)) do |entry, counts|
|
356
|
+
next unless entry.key? stat
|
357
|
+
|
358
|
+
counts[entry[stat]] += 1
|
359
|
+
|
360
|
+
counts
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|
364
|
+
def self.filter_empty_arg(arg)
|
365
|
+
puts [
|
366
|
+
'Ignoring'.colorize(:light_yellow),
|
367
|
+
"--#{arg}".colorize(:cyan),
|
368
|
+
'it requires an argument'.colorize(:red)
|
369
|
+
].join(' ')
|
370
|
+
end
|
371
|
+
|
372
|
+
# Break out filter row logic into separate method
|
373
|
+
def self.filter_row_key(row, param)
|
374
|
+
# Ignore Other Logic if Field isn't even included
|
375
|
+
return false unless row.key? param.field
|
376
|
+
|
377
|
+
# Not Included Param
|
378
|
+
included = row[param.field].to_s.include? param.value.to_s
|
379
|
+
|
380
|
+
if param.bang
|
381
|
+
!included
|
382
|
+
else
|
383
|
+
included
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
# Total Count Helper
|
388
|
+
def self.total_count(results)
|
389
|
+
results.each do |k, v|
|
390
|
+
puts k
|
391
|
+
puts "Total: #{v.count.to_s.colorize(:blue)}"
|
392
|
+
puts
|
393
|
+
end
|
394
|
+
end
|
395
|
+
|
396
|
+
# Total Log List Manipulator
|
397
|
+
def self.prepare_list(log_list, base_list = nil)
|
398
|
+
base_list ||= GreenHat::ShellHelper::Log.list
|
399
|
+
|
400
|
+
# Assume all
|
401
|
+
log_list.push '*' if log_list.empty?
|
402
|
+
|
403
|
+
# Map for All
|
404
|
+
log_list = base_list.map(&:name) if log_list == ['*']
|
405
|
+
|
406
|
+
log_list
|
407
|
+
end
|
408
|
+
|
409
|
+
# Shortcut find things
|
410
|
+
def self.find_things(log_list, args = {})
|
411
|
+
things = log_list.uniq.flat_map do |log|
|
412
|
+
Thing.where name: log
|
413
|
+
end
|
414
|
+
|
415
|
+
# Host / Archive
|
416
|
+
things.select! { |x| x.archive? args.archive } if args.archive
|
417
|
+
|
418
|
+
things
|
419
|
+
end
|
420
|
+
|
421
|
+
# Main Entry Point for Searching
|
422
|
+
def self.search_start(log_list, filter_type, args, opts)
|
423
|
+
# Convert to Things
|
424
|
+
logs = ShellHelper.find_things(log_list, args)
|
425
|
+
|
426
|
+
logs.each_with_object({}) do |log, obj|
|
427
|
+
# Ignore Empty Results / No Thing
|
428
|
+
next if log&.data.blank?
|
429
|
+
|
430
|
+
obj[log.friendly_name] = ShellHelper.search(log.data, filter_type, args, opts)
|
431
|
+
|
432
|
+
obj
|
433
|
+
end
|
434
|
+
end
|
435
|
+
|
436
|
+
# Generic Search Helper / String/Regex
|
437
|
+
def self.search(data, type = :all?, args = {}, opts = {})
|
438
|
+
results = data.clone.flatten.compact
|
439
|
+
|
440
|
+
results.select! do |row|
|
441
|
+
opts.send(type) do |opt|
|
442
|
+
search_row(row, opt)
|
443
|
+
end
|
444
|
+
end
|
445
|
+
|
446
|
+
# Strip Results if Slice is defined
|
447
|
+
results.map! { |row| row.slice(*args[:slice]) } if args[:slice]
|
448
|
+
|
449
|
+
# Strip Results if Except is defined
|
450
|
+
results.map! { |row| row.except(*args[:except]) } if args[:except]
|
451
|
+
|
452
|
+
# Remove Blank from either slice or except
|
453
|
+
results.reject!(&:empty?)
|
454
|
+
|
455
|
+
results
|
456
|
+
end
|
457
|
+
|
458
|
+
# Break out filter row logic into separate method
|
459
|
+
def self.search_row(row, param)
|
460
|
+
# Not Included Param
|
461
|
+
included = row.to_s.include? param.value
|
462
|
+
|
463
|
+
if param.bang
|
464
|
+
!included
|
465
|
+
else
|
466
|
+
included
|
467
|
+
end
|
468
|
+
end
|
469
|
+
|
470
|
+
# Color Reader Helper
|
471
|
+
def self.pastel
|
472
|
+
@pastel ||= Pastel.new
|
473
|
+
end
|
474
|
+
|
475
|
+
# Number Helper
|
476
|
+
# https://gitlab.com/zedtux/human_size_to_number/-/blob/master/lib/human_size_to_number/helper.rb
|
477
|
+
def self.human_size_to_number(string)
|
478
|
+
size, unit = string.scan(/(\d*\.?\d+)\s?(Bytes?|KB|MB|GB|TB)/i).first
|
479
|
+
number = size.to_f
|
480
|
+
|
481
|
+
number = case unit.downcase
|
482
|
+
when 'byte', 'bytes'
|
483
|
+
number
|
484
|
+
when 'kb'
|
485
|
+
number * 1024
|
486
|
+
when 'mb'
|
487
|
+
number * 1024 * 1024
|
488
|
+
when 'gb'
|
489
|
+
number * 1024 * 1024 * 1024
|
490
|
+
when 'tb'
|
491
|
+
number * 1024 * 1024 * 1024 * 1024
|
492
|
+
end
|
493
|
+
number.round
|
494
|
+
end
|
495
|
+
|
496
|
+
# TODO: Needed?
|
497
|
+
def self.filter_and(data, params = {})
|
498
|
+
result = data.clone.flatten.compact
|
499
|
+
params.each do |k, v|
|
500
|
+
result.select! do |row|
|
501
|
+
if row.key? k.to_sym
|
502
|
+
row[k.to_sym].include? v
|
503
|
+
else
|
504
|
+
false
|
505
|
+
end
|
506
|
+
end
|
507
|
+
next
|
508
|
+
end
|
509
|
+
|
510
|
+
result
|
511
|
+
end
|
512
|
+
end
|
513
|
+
# rubocop:enable Metrics/ModuleLength
|
514
|
+
end
|