greenhat 0.5.0 → 0.6.2

data/lib/greenhat/shell/faststats.rb

@@ -100,6 +100,13 @@ module GreenHat
             puts "- #{log.name.pastel(:yellow)}"
           end
         end
+
+        return unless all
+
+        puts "\n#{'Other / Unknown'.pastel(:red)}"
+        (Thing.all - files).each do |file|
+          puts "- #{file.name.pastel(:yellow)}"
+        end
       end
 
       def self.ls(args = [])
@@ -117,15 +124,23 @@ module GreenHat
 
         LogBot.debug('FastStats CMD', cmd) if ENV['DEBUG']
 
+        # binding.pry
+
+        # Ignore Unknown Errors
         results = ShellHelper.file_process(files) do |file|
+          output = `fast-stats #{cmd} #{file.file} 2>&1`
+          result = $CHILD_STATUS.success?
+
+          next unless result
+
           [
             file.friendly_name,
-            `fast-stats #{cmd} #{file.file}`.split("\n"),
+            output.split("\n"),
             "\n"
           ]
         end
 
-        ShellHelper.show(results.flatten, flags)
+        ShellHelper.show(results.compact.flatten, flags)
       end
 
       # ========================================================================
@@ -219,11 +234,15 @@ module GreenHat
           end
         end.join(' ')
 
-        # Prepare Log List
-        file_list = ShellHelper.prepare_list(file_list, ShellHelper::Faststats.things, flags)
+        # Prepare Log List / Allow attempting of parsing everything
+        file_list = if file_list == ['*']
+                      Thing.all.map(&:name)
+                    else
+                      ShellHelper.prepare_list(file_list, ShellHelper::Faststats.things)
+                    end
 
         # Convert to Things
-        files = ShellHelper.find_things(file_list)
+        files = ShellHelper.find_things(file_list, fuzzy_file_match: true)
 
         [files, flags, cmd]
       end

data/lib/greenhat/shell/field_helper.rb

@@ -34,7 +34,7 @@ module GreenHat
       %w[
         archive case combine end exact except exists json limit or page pluck
         raw reverse round slice sort start stats table_style text time_zone
-        total truncate uniq
+        total truncate uniq percentile transform
       ]
     end
 

data/lib/greenhat/shell/filter_help.rb

@@ -2,198 +2,22 @@ module GreenHat
   # CLI Helper
   module ShellHelper
     # Unify Filter / Filter Help
-    # rubocop:disable Metrics/MethodLength,Layout/LineLength
+    # rubocop:disable Metrics/MethodLength,Layout/LineLength,Metrics/ModuleLength
     module Filter
       def self.help(args = '')
-        if args.blank?
-          help_index.values.map { |x| x.push '' }
-        else
-          list = help_index.select do |k, _v|
-            k.to_s.include? args
-          end
+        output = if args.blank?
+                   help_index.values.map { |x| x.push '' }
+                 else
+                   list = help_index.select do |k, _v|
+                     k.to_s.include? args
+                   end
 
-          list.values
-        end
+                   list.values
+                 end
 
-        # .map { |x| x.join("\n") }.join("\n\n")
+        output.flatten(2)
       end
 
-      # TODO: Confirm Remove
-      # def self.legacy
-      #   puts "\u2500".pastel(:cyan) * 20
-      #   puts 'Filter'.pastel(:yellow)
-      #   puts "\u2500".pastel(:cyan) * 20
-
-      #   puts 'Options'.pastel(:blue)
-      #   puts '--raw'.pastel(:green)
-      #   puts '  Disable formatting and page/less'
-      #   puts
-
-      #   puts '--page'.pastel(:green)
-      #   puts '  Specifically enable or disable paging'
-      #   puts '  E.g. --page (default to true), --page=true, --page=false'
-      #   puts
-
-      #   puts '--round'.pastel(:green)
-      #   puts '  Attempt to round all integers. Default: 2.'
-      #   puts '  E.g. --round, --round=3, --round=0'
-      #   puts
-
-      #   puts '--limit'.pastel(:green)
-      #   puts '  Limit total output lines. Disabled by default. Default without value is based on screen height'
-      #   puts '  E.g. --limit, --limit=5'
-      #   puts
-
-      #   puts '--json'.pastel(:green)
-      #   puts '  Print output back into JSON'
-      #   puts
-
-      #   puts '--or'.pastel(:green)
-      #   puts '  Filters will use OR instead of AND (all match vs any match)'
-      #   puts
-
-      #   puts '--total'.pastel(:green)
-      #   puts '  Show total count, duration, start/end time for matching entries'
-      #   puts
-
-      #   puts '--fields'.pastel(:green)
-      #   puts '  Print only Available fields for selected files'
-      #   puts
-
-      #   puts '--slice'.pastel(:green)
-      #   puts '  Extract specific fields from entries (slice multiple with comma)'
-      #   puts '  Ex: --slice=path or --slice=path,params'
-      #   puts
-
-      #   puts '--except'.pastel(:green)
-      #   puts '  Exclude specific fields (except multiple with comma)'
-      #   puts '  Ex: --except=params --except=params,path'
-      #   puts
-
-      #   puts '--exists'.pastel(:green)
-      #   puts '  Ensure field exists regardless of contents'
-      #   puts '  Ex: --exists=params --exists=params,path'
-      #   puts
-
-      #   puts '--stats'.pastel(:green)
-      #   puts '  Order/Count occurrances by field. Combine with `truncate` for key names'
-      #   puts '  Ex: --stats=params --except=params,path'
-      #   puts
-
-      #   puts '--uniq'.pastel(:green)
-      #   puts '  Show unique values only'
-      #   puts '  Ex: --uniq=params --uniq=params,path'
-      #   puts
-
-      #   puts '--pluck'.pastel(:green)
-      #   puts '  Extract values from entries'
-      #   puts '  Ex: --pluck=params --pluck=params,path'
-      #   puts
-
-      #   puts '--archive'.pastel(:green)
-      #   puts '  Limit to specific archvie name (partial matching /inclusive). Matching SOS tar.gz name'
-      #   puts '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
-      #   puts
-
-      #   puts '--sort'.pastel(:green)
-      #   puts '  Sort by multiple fields'
-      #   puts '  Ex: --sort=duration_s,db_duration_s'
-      #   puts
-
-      #   puts '--reverse'.pastel(:green)
-      #   puts '  Reverse all results'
-      #   puts '  Ex: --reverse'
-      #   puts
-
-      #   puts '--combine'.pastel(:green)
-      #   puts '  Omit archive identifier dividers. Useful with sort or time filters'
-      #   puts '  Ex: --combine'
-      #   puts
-
-      #   puts '--case'.pastel(:green)
-      #   puts '  Exact case match. Defaults to case insensitive'
-      #   puts '  Ex: --case; --name=Jon, --name=jane --case'
-      #   puts
-
-      #   puts '--exact'.pastel(:green)
-      #   puts '  Exact parameter/value match. Defaults to partial match'
-      #   puts '  Ex: --field=CommonPartial --exact'
-      #   puts
-
-      #   puts '--start'.pastel(:green)
-      #   puts '  Show events after specified time. Filtered by the `time` field'
-      #   puts '  Use with `--end` for between selections'
-      #   puts '  Ex: log filter --start="2021-06-22 14:44 UTC" --end="2021-06-22 14:45 UTC"'
-      #   puts
-
-      #   puts '--end'.pastel(:green)
-      #   puts '  Show events before specified time. Filtered by the `time` field'
-      #   puts '  Use with `--start` for between selections'
-      #   puts '  Ex: log filter --end="2021-06-22"'
-      #   puts
-
-      #   puts '--time_zone'.pastel(:green)
-      #   puts '  Manipulate the `time` field into a specific timezone'
-      #   puts '  Ex: log filter --time_zone=EDT'
-      #   puts
-
-      #   puts '--text'.pastel(:green)
-      #   puts '  Full entry text searching (slow)'
-      #   puts '    --text="anything here"'
-      #   puts
-
-      #   puts '--table_style'.pastel(:green)
-      #   puts '  Renderer used for formatted output. basic, ascii, or unicode(default)'
-      #   puts '  Ex: log filter --table_style=base'
-      #   puts
-
-      #   puts '--truncate'.pastel(:green)
-      #   puts '  Truncate field length. On by default (4 rows). Performance issues!'
-      #   puts '  Disable with --truncate=0'.pastel(:bright_red)
-      #   puts '  Ex: --truncate=200, --truncate=2048"'
-      #   puts
-
-      #   puts 'Search specific logs'.pastel(:blue)
-      #   puts '  Any non dash parameters will be the log list to search from'
-      #   puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names"
-      #   puts
-
-      #   puts 'General (Field) Searching'.pastel(:blue)
-      #   puts "  There are different ways filter within specific keys/fields within log. The baseline is #{'--[key]=[value]'.pastel(:green)}"
-      #   puts '  This will, by default, select any entries with any partial match (case insensitive) of [value] in the entry [key]'
-      #   puts '  Example: --path=mirror/pull'
-      #   puts
-
-      #   puts "  You can modify logic with params like: #{'case'.pastel(:green)}, #{'exact'.pastel(:green)}"
-      #   puts '  Ex Case turn on case sensitivity, exact will change from the inclusive to only exact matching.'
-      #   puts
-
-      #   puts "  You can also use multiple #{'--[key]=[value]'.pastel(:green)} to narrow down, or combine it with #{'or'.pastel(:green)} to be more inclusive"
-      #   puts '  Ex nginx/gitlab_access.log --http_user_agent=gitlab-runner --path=/api/v4/jobs/request'
-      #   puts '  This will look through the access logs for anything that is a runner (user agent) and hitting the request endpoint'
-      #   puts
-
-      #   puts " You can also modify whether a field search excluded or a integer (and technically string) comparison by changing the operator:  #{'(!= >= <=)'.pastel(:green)}"
-      #   puts ' Ex "nginx/gitlab_access.log --http_user_agent=gitlab-runner  --http_user_agent!=13.12.0" will exclude any entries with 13.12.0/13.12 runners'
-      #   puts ' Ex "sidekiq/current --duration_s>=5 --class!=LdapSyncWorker" Duration over 5 that is not an ldap job'
-      #   puts
-
-      #   puts 'Pipe to Shell'.pastel(:blue)
-      #   puts '  This is very experimental! The idea is to take all the output from the search and allow a pipe into a traditional shell command.'
-      #   puts '  Due to the arg parsing its not a straight send to the terminal. You may need to escape quotes.'
-      #   puts '  Order is relevant, no arguments after the pipe'
-      #   puts
-      #   puts '  <files> <params> | grep path'
-      #   puts
-
-      #   puts 'Example Queries'.pastel(:blue)
-      #   puts "  Also see #{'examples'.pastel(:bright_blue)} for even more examples"
-      #   puts '  log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message'
-      #   puts '  log filter gitlab-rails/api_json.log --slice=ua --uniq=ua --ua=gitlab-runner'
-
-      #   puts
-      # end
-
       def self.help_index
         {
           title: [
@@ -238,7 +62,8 @@ module GreenHat
 
           total: [
             '--total'.pastel(:green),
-            '  Show total count, duration, start/end time for matching entries'
+            '  Show total count, duration, start/end time for matching entries',
+            '  Add `simple` to only return counts. e.g. --total=simple'
           ],
 
           fields: [
@@ -250,14 +75,20 @@ module GreenHat
             '--slice'.pastel(:green),
             '  Extract specific fields from entries (slice multiple with comma)',
             '  Ex: --slice=path or --slice=path,params'
-
           ],
 
           except: [
             '--except'.pastel(:green),
             '  Exclude specific fields (except multiple with comma)',
             '  Ex: --except=params --except=params,path'
+          ],
 
+          transform: [
+            '--transform'.pastel(:green),
+            '  Copy key to another key within results. Useful for queries that require (timestamp => time)',
+            '  Available Transforms, to_i, default is to copy',
+            '  Ex: --transform=timestamp,time',
+            '      --transform=cpu,cpu,to_i'
           ],
 
           exists: [
@@ -371,6 +202,22 @@ module GreenHat
             '  Ex: --truncate=200, --truncate=2048"'
           ],
 
+          interval: [
+            '--interval'.pastel(:green),
+            '  Split results down into a interval (based on the time field)',
+            '  Example Params: 5m, 1d, 3h, "30 minutes" (chronic duration)',
+            '  Ex: --interval=15m, --interval=1h"'
+          ],
+
+          percentile: [
+            '--percentile'.pastel(:green),
+            '  Stats breakdown on any integer value within the results. No params',
+            '  Best when combined with slice/round to limit/filter results',
+            '  No parameters, Returns 99,95,mean,min,max and entry count',
+            '  Ex: --percentile --round --slice=field',
+            '  Query: gitlab-rails/api_json.log --percentile --round=2 --slice=cpu_s'
+          ],
+
           field_search: [
             'General (Field) Searching'.pastel(:blue),
             "  There are different ways filter within specific keys/fields within log. The baseline is #{'--[key]=[value]'.pastel(:green)}",
@@ -412,7 +259,7 @@ module GreenHat
 
         }
       end
-      # rubocop:enable Metrics/MethodLength,Layout/LineLength
+      # rubocop:enable Metrics/MethodLength,Layout/LineLength,Metrics/ModuleLength
     end
   end
 end

data/lib/greenhat/shell/log.rb

@@ -4,7 +4,6 @@ module GreenHat
     # Logs
     # rubocop:disable Metrics/ModuleLength
     module Log
-      extend Spinner # TODO: DEBUGGING REMOVE
       def self.auto_complete(list, word)
         # Argument Parsing
         files, flags, _args = Args.parse(list)
@@ -79,11 +78,12 @@ module GreenHat
 
       def self.filter_help(raw = {})
         args, flags, _args = Args.parse(raw)
+
         ShellHelper.show(ShellHelper::Filter.help(args.first), flags)
       end
 
       def self.ls(args = [])
-        ShellHelper::List.list(args, Thing.list)
+        ShellHelper::List.list(args, ShellHelper::Log.list)
       end
 
       def self.show(raw = {})
@@ -106,7 +106,7 @@ module GreenHat
         end
 
         name = if raw.empty?
-                 Cli.prompt.ask('Log/name to save the results to? '.pastel(:yellow))
+                 Cli.prompt.ask('Log entry to save the results to? '.pastel(:yellow))
                else
                  raw.first
                end
@@ -125,10 +125,18 @@ module GreenHat
           return false
         end
 
-        Thing.new.query_save(results, name)
+        Thing.new.query_save(save_greenhat_query_info + results, name)
         puts "#{name.pastel(:green)} Saved!"
       end
 
+      def self.save_greenhat_query_info
+        [{
+          time: Time.now,
+          query: ShellHelper::Log.last,
+          msg: 'GreenHat Save/Write'
+        }]
+      end
+
       def self.write(raw = [])
         if ShellHelper::Log.last.nil?
           puts 'No previous query found'.pastel(:red)
@@ -139,7 +147,7 @@ module GreenHat
         end
 
         name = if raw.empty?
-                 Cli.prompt.ask('Log/name to save the results to? '.pastel(:yellow))
+                 Cli.prompt.ask('Filename to save the results to? '.pastel(:yellow))
                else
                  raw.first
                end
@@ -158,7 +166,7 @@ module GreenHat
           return false
         end
 
-        all = results.map { |row| Oj.dump(row) }
+        all = (save_greenhat_query_info + results).map { |row| Oj.dump(row) }
         File.write(name, all.join("\n"))
         puts "#{name.pastel(:green)} File Written!"
       end
@@ -205,16 +213,10 @@ module GreenHat
         files, flags, args = Args.parse(raw)
 
         # Prepare Log List
-        files = ShellHelper.prepare_list(files)
+        files = ShellHelper.prepare_list(files, ShellHelper::Log.list)
 
         results = Query.start(files, flags, args)
 
-        # Skip and Print Total if set
-        if flags[:total]
-          ShellHelper.total_count(results, flags)
-          return true
-        end
-
         # Skip and Print Total if set
         if flags[:fields]
           ShellHelper.fields_print(results)
@@ -222,14 +224,15 @@ module GreenHat
         end
 
         # Check Search Results
-        if results.instance_of?(Hash) && results.values.flatten.empty?
+        if results.empty?
           puts 'No results'.pastel(:red)
           ShellHelper::Log.no_files_warning(files) if ShellHelper.find_things(files, flags).count.zero?
         elsif flags[:pipe]
           Pipe.show(results, flags[:pipe])
         else
+          # Allow for array / multiple table outputs
           # This causes the key 'colorized' output to also be included
-          ShellHelper.show(results.to_a.compact.flatten, flags)
+          ShellHelper.show(results, flags)
         end
       rescue StandardError => e
         LogBot.fatal('Filter', message: e.message)
@@ -275,6 +278,10 @@ module GreenHat
         puts 'Show workhorse duration/URI. Filter by duration bounds'.pastel(:bright_green)
         puts 'gitlab-workhorse/current --duration_ms>=30000 --duration_ms<=45000 --slice=duration_ms,uri'
         puts
+
+        puts 'Show runner statistics'.pastel(:bright_green)
+        puts 'gitlab-rails/api_json.log --path=/api/v4/jobs/request --percentile --round=2'
+        puts
       end
       # rubocop:enable Layout/LineLength
 
@@ -286,7 +293,7 @@ module GreenHat
         files_list, flags, args = Args.parse(raw)
 
         # Prepare Log List
-        files = ShellHelper.prepare_list(files_list)
+        files = ShellHelper.prepare_list(files_list, ShellHelper::Log.list)
 
         results = ShellHelper.search_start(files, flags, args)
 
@@ -387,6 +394,11 @@ module GreenHat
         puts "No matching files found for pattern #{files.to_s.pastel(:yellow)}"
         puts "See #{'ls'.pastel(:blue)} for available files"
       end
+
+      # Limit to Log Files
+      def self.list
+        Thing.all.select(&:log)
+      end
     end
 
     # --------