greenhat 0.3.2 → 0.3.6

data/lib/greenhat/shell/filter_help.rb

@@ -2,8 +2,8 @@ module GreenHat
   # CLI Helper
   module ShellHelper
     # Unify Filter / Filter Help
+    # rubocop:disable Metrics/MethodLength,Metrics/ModuleLength,Layout/LineLength
     module Filter
-      # rubocop:disable Metrics/MethodLength
       def self.help
         puts "\u2500".pastel(:cyan) * 20
         puts 'Filter'.pastel(:yellow)
@@ -38,7 +38,7 @@ module GreenHat
         puts
 
         puts '--total'.pastel(:green)
-        puts '  Print only total count of matching entries'
+        puts '  Show total count, duration, start/end time for matching entries'
         puts
 
         puts '--fields'.pastel(:green)
@@ -138,17 +138,39 @@ module GreenHat
         puts '  Ex: --truncate=200, --truncate=2048"'
         puts
 
-        puts 'Field Searching'.pastel(:blue)
-        puts '  --[key]=[value]'
-        puts '  Search in key for value'
-        puts '  Example: --path=mirror/pull'
-        puts
-
         puts 'Search specific logs'.pastel(:blue)
         puts '  Any non dash parameters will be the log list to search from'
         puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names"
         puts
 
+        puts 'General (Field) Searching'.pastel(:blue)
+        puts "  There are different ways filter within specific keys/fields within log. The baseline is #{'--[key]=[value]'.pastel(:green)}"
+        puts '  This will, by default, select any entries with any partial match (case insensitive) of [value] in the entry [key]'
+        puts '  Example: --path=mirror/pull'
+        puts
+
+        puts "  You can modify logic with params like: #{'case'.pastel(:green)}, #{'exact'.pastel(:green)}"
+        puts '  Ex Case turn on case sensitivity, exact will change from the inclusive to only exact matching.'
+        puts
+
+        puts "  You can also use multiple #{'--[key]=[value]'.pastel(:green)} to narrow down, or combine it with #{'or'.pastel(:green)} to be more inclusive"
+        puts '  Ex nginx/gitlab_access.log --http_user_agent=gitlab-runner --path=/api/v4/jobs/request'
+        puts '  This will look through the access logs for anything that is a runner (user agent) and hitting the request endpoint'
+        puts
+
+        puts " You can also modify whether a field search excluded or a integer (and technically string) comparison by changing the operator:  #{'(!= >= <=)'.pastel(:green)}"
+        puts ' Ex "nginx/gitlab_access.log --http_user_agent=gitlab-runner  --http_user_agent!=13.12.0" will exclude any entries with 13.12.0/13.12 runners'
+        puts ' Ex "sidekiq/current --duration_s>=5 --class!=LdapSyncWorker" Duration over 5 that is not an ldap job'
+        puts
+
+        puts 'Pipe to Shell'.pastel(:blue)
+        puts '  This is very experimental! The idea is to take all the output from the search and allow a pipe into a traditional shell command.'
+        puts '  Due to the arg parsing its not a straight send to the terminal. You may need to escape quotes.'
+        puts '  Order is relevant, no arguments after the pipe'
+        puts
+        puts '  <files> <params> | grep path'
+        puts
+
         puts 'Example Queries'.pastel(:blue)
         puts "  Also see #{'examples'.pastel(:bright_blue)} for even more examples"
         puts '  log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message'
@@ -156,7 +178,208 @@ module GreenHat
 
         puts
       end
-      # rubocop:enable Metrics/MethodLength
+
+      def self.help_index
+        {
+          title: [
+            "\u2500".pastel(:cyan) * 20,
+            'Filter'.pastel(:yellow),
+            "\u2500".pastel(:cyan) * 20
+          ],
+          options: [
+            'Options'.pastel(:blue)
+          ],
+
+          raw: [
+            '--raw'.pastel(:green),
+            '  Disable formatting and page/less'
+          ],
+
+          page: [
+            '--page'.pastel(:green),
+            '  Specifically enable or disable paging',
+            '  E.g. --page (default to true), --page=true, --page=false'
+          ],
+
+          round: ['--round'.pastel(:green),
+                  '  Attempt to round all integers. Default: 2.',
+                  '  E.g. --round, --round=3, --round=0'],
+
+          limit: [
+
+            '--limit'.pastel(:green),
+            '  Limit total output lines. Disabled by default. Default without value is based on screen height',
+            '  E.g. --limit, --limit=5'
+          ],
+
+          json: [
+            '--json'.pastel(:green),
+            '  Print output back into JSON'
+          ],
+
+          or: [
+            '--or'.pastel(:green),
+            '  Filters will use OR instead of AND (all match vs any match)'
+          ],
+
+          total: [
+            '--total'.pastel(:green),
+            '  Show total count, duration, start/end time for matching entries'
+          ],
+
+          fields: [
+            '--fields'.pastel(:green),
+            '  Print only Available fields for selected files'
+          ],
+
+          slice: [
+            '--slice'.pastel(:green),
+            '  Extract specific fields from entries (slice multiple with comma)',
+            '  Ex: --slice=path or --slice=path,params'
+
+          ],
+
+          except: [
+            '--except'.pastel(:green),
+            '  Exclude specific fields (except multiple with comma)',
+            '  Ex: --except=params --except=params,path'
+
+          ],
+
+          exists: [
+            '--exists'.pastel(:green),
+            '  Ensure field exists regardless of contents',
+            '  Ex: --exists=params --exists=params,path'
+
+          ],
+
+          stats: [
+            '--stats'.pastel(:green),
+            '  Order/Count occurrances by field. Combine with `truncate` for key names',
+            '  Ex: --stats=params --except=params,path'
+
+          ],
+
+          uniq: [
+            '--uniq'.pastel(:green),
+            '  Show unique values only',
+            '  Ex: --uniq=params --uniq=params,path'
+
+          ],
+
+          pluck: [
+            '--pluck'.pastel(:green),
+            '  Extract values from entries',
+            '  Ex: --pluck=params --pluck=params,path'
+
+          ],
+
+          archive: [
+            '--archive'.pastel(:green),
+            '  Limit to specific archvie name (partial matching /inclusive). Matching SOS tar.gz name',
+            '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
+
+          ],
+
+          sort: [
+            '--sort'.pastel(:green),
+            '  Sort by multiple fields',
+            '  Ex: --sort=duration_s,db_duration_s'
+
+          ],
+
+          reverse: [
+            '--reverse'.pastel(:green),
+            '  Reverse all results',
+            '  Ex: --reverse'
+
+          ],
+
+          combine: [
+            '--combine'.pastel(:green),
+            '  Omit archive identifier dividers. Useful with sort or time filters',
+            '  Ex: --combine'
+
+          ],
+
+          case: [
+            '--case'.pastel(:green),
+            '  Exact case match. Defaults to case insensitive',
+            '  Ex: --case; --name=Jon, --name=jane --case'
+
+          ],
+
+          exact: [
+            '--exact'.pastel(:green),
+            '  Exact parameter/value match. Defaults to partial match',
+            '  Ex: --field=CommonPartial --exact'
+
+          ],
+
+          start: [
+            '--start'.pastel(:green),
+            '  Show events after specified time. Filtered by the `time` field',
+            '  Use with `--end` for between selections',
+            '  Ex: log filter --start="2021-06-22 14:44  UTC" --end="2021-06-22 14:45 UTC"'
+
+          ],
+
+          end: [
+            '--end'.pastel(:green),
+            '  Show events before specified time. Filtered by the `time` field',
+            '  Use with `--start` for between selections',
+            '  Ex: log filter --end="2021-06-22"'
+          ],
+
+          time_zone: [
+            '--time_zone'.pastel(:green),
+            '  Manipulate the `time` field into a specific timezone',
+            '  Ex: log filter --time_zone=EDT'
+
+          ],
+
+          text: [
+            '--text'.pastel(:green),
+            '  Full entry text searching (slow)',
+            '    --text="anything here"'
+          ],
+
+          table_style: [
+            '--table_style'.pastel(:green),
+            '  Renderer used for formatted output. basic, ascii, or unicode(default)',
+            '  Ex: log filter --table_style=base'
+          ],
+
+          truncate: [
+            '--truncate'.pastel(:green),
+            '  Truncate field length. On by default (4 rows). Performance issues!',
+            '  Disable with --truncate=0'.pastel(:bright_red),
+            '  Ex: --truncate=200, --truncate=2048"'
+          ],
+
+          field: [
+            'Field Searching'.pastel(:blue),
+            '  --[key]=[value]',
+            '  Search in key for value',
+            '  Example: --path=mirror/pull'
+          ],
+
+          ls: [
+            'Search specific logs'.pastel(:blue),
+            '  Any non dash parameters will be the log list to search from',
+            "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names"
+          ],
+
+          examples: [
+            'Example Queries'.pastel(:blue),
+            "  Also see #{'examples'.pastel(:bright_blue)} for even more examples",
+            '  log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message',
+            '  log filter gitlab-rails/api_json.log --slice=ua --uniq=ua --ua=gitlab-runner'
+          ]
+
+        }
+      end
+      # rubocop:enable Metrics/MethodLength,Metrics/ModuleLength,Layout/LineLength
     end
   end
 end

data/lib/greenhat/shell/log.rb

@@ -2,6 +2,7 @@ module GreenHat
   # CLI Helper
   module Shell
     # Logs
+    # rubocop:disable Metrics/ModuleLength
     module Log
       def self.auto_complete(list, word)
         # Argument Parsing
@@ -47,6 +48,7 @@ module GreenHat
         puts '  filter'.pastel(:green)
         puts "    Primary way for log searching within greenhat. See #{'filter_help'.pastel(:blue)}"
         puts '    Time, round, slice/except, and/or, stats, uniq, sort'
+        puts "    #{'filter_help'.pastel(:blue)} supports filtering Ex: #{'filter_help stats'.pastel(:blue)}"
         puts
 
         puts '  show'.pastel(:green)
@@ -57,13 +59,33 @@ module GreenHat
         puts "    General full text by file searching. See #{'search_help'.pastel(:blue)}"
         puts
 
+        puts '  save'.pastel(:green)
+        puts '    Save the last query result into a new searchable object'
+        puts
+
+        puts '  write'.pastel(:green)
+        puts '    Write the last query result into a local file'
+        puts
+
+        puts '  visualize'.pastel(:green)
+        puts '    Load web services and formulate last query for the UI'
+        puts
+
         puts ShellHelper::List.help
 
         puts "See #{'examples'.pastel(:bright_blue)} for query examples"
       end
 
-      def self.filter_help
-        ShellHelper::Filter.help
+      def self.filter_help(args = {})
+        if args.empty?
+          ShellHelper::Filter.help
+        else
+          list = ShellHelper::Filter.help_index.select do |k, _v|
+            k.to_s.include? args.first
+          end
+
+          puts list.values.map { |x| x.join("\n") }.join("\n\n")
+        end
       end
 
       def self.ls(args = [])
@@ -80,6 +102,95 @@ module GreenHat
         ShellHelper.show files.map(&:data).flatten
       end
 
+      def self.save(raw = [])
+        if ShellHelper::Log.last.nil?
+          puts 'No previous query found'.pastel(:red)
+          puts 'Run a query first then save to store as a new log'
+          puts
+          puts "Try #{'nginx/gitlab_access.log --status!=200'.pastel(:green)} then #{'save'.pastel(:green)}"
+          true
+        end
+
+        name = if raw.empty?
+                 Cli.prompt.ask('Log/name to save the results to? '.pastel(:yellow))
+               else
+                 raw.first
+               end
+
+        if name.blank?
+          puts 'Name required'.pastel(:red)
+          return true
+        end
+
+        results = ShellHelper.filter_internal ShellHelper::Log.last
+
+        # Don't save empty results
+        if results.empty?
+          puts 'No results'.pastel(:red)
+          ShellHelper::Log.no_files_warning(files) if ShellHelper.find_things(files, flags).count.zero?
+          return false
+        end
+
+        Thing.new.query_save(results, name)
+        puts "#{name.pastel(:green)} Saved!"
+      end
+
+      def self.write(raw = [])
+        if ShellHelper::Log.last.nil?
+          puts 'No previous query found'.pastel(:red)
+          puts 'Run a query first then write'
+          puts
+          puts "Try #{'nginx/gitlab_access.log --status!=200'.pastel(:green)} then #{'save'.pastel(:green)}"
+          true
+        end
+
+        name = if raw.empty?
+                 Cli.prompt.ask('Log/name to save the results to? '.pastel(:yellow))
+               else
+                 raw.first
+               end
+
+        if name.blank?
+          puts 'Name required'.pastel(:red)
+          return true
+        end
+
+        results = ShellHelper.filter_internal ShellHelper::Log.last
+
+        # Don't save empty results
+        if results.empty?
+          puts 'No results'.pastel(:red)
+          ShellHelper::Log.no_files_warning(files) if ShellHelper.find_things(files, flags).count.zero?
+          return false
+        end
+
+        all = results.map { |row| Oj.dump(row) }
+        File.write(name, all.join("\n"))
+        puts "#{name.pastel(:green)} File Written!"
+      end
+
+      def self.visualize
+        if ShellHelper::Log.last.nil?
+          puts 'No previous query found'.pastel(:red)
+          puts 'Run a query first then visualize to load it in the chart web page'
+          puts
+          puts "Try #{'nginx/gitlab_access.log --status!=200'.pastel(:green)} then #{'visualize'.pastel(:green)}"
+          return true
+        end
+
+        # Load Required Files
+        require 'greenhat/web'
+
+        unless GreenHat::Web.alive?
+          GreenHat::Web.start
+          sleep 0.2
+        end
+
+        url = "http://localhost:4567/chart/time?query=#{CGI.escape(ShellHelper::Log.last)}"
+
+        GreenHat::Platform.open url
+      end
+
       # ========================================================================
       # Filter (See Filter Help)
       # ========================================================================
@@ -94,6 +205,8 @@ module GreenHat
           return true
         end
 
+        ShellHelper::Log.last = raw
+
         # Argument Parsing
         files, flags, args = Args.parse(raw)
 
@@ -104,7 +217,7 @@ module GreenHat
 
         # Skip and Print Total if set
         if flags[:total]
-          ShellHelper.total_count(results)
+          ShellHelper.total_count(results, flags)
           return true
         end
 
@@ -117,13 +230,13 @@ module GreenHat
         # Check Search Results
         if results.instance_of?(Hash) && results.values.flatten.empty?
           puts 'No results'.pastel(:red)
+          ShellHelper::Log.no_files_warning(files) if ShellHelper.find_things(files, flags).count.zero?
+        elsif flags[:pipe]
+          Pipe.show(results, flags[:pipe])
         else
           # This causes the key 'colorized' output to also be included
           ShellHelper.show(results.to_a.compact.flatten, flags)
         end
-
-        # log filter --path='cloud/gitlab-automation' --path='/pull' --all
-        # log filter --project=thingy --other_filter=asdf *
       rescue StandardError => e
         LogBot.fatal('Filter', message: e.message)
         ap e.backtrace
@@ -152,9 +265,25 @@ module GreenHat
         puts 'Count/% occurences for both user and remote ip fields'.pastel(:bright_green)
         puts 'gitlab-rails/api_json.log --stats=meta.user,meta.remote_ip --exists=meta.user'
         puts
-      end
 
+        puts 'Sidekiq jobs that took over 5 seconds excluding LdapSyncWorker jobs'.pastel(:bright_green)
+        puts 'sidekiq/current --duration_s>=5 --class!=LdapSyncWorker'
+        puts
+
+        puts 'Search access logs for runner requests, exclude specific runner version'.pastel(:bright_green)
+        puts 'nginx/gitlab_access.log --http_user_agent=gitlab-runner  --http_user_agent!=13.12.0'
+        puts
+
+        puts 'Get a list of unique Gitaly error messages for a specific project'.pastel(:bright_green)
+        puts 'filter --level=error --grpc.request.glProjectPath=path/to/project  gitaly/current --slice=error --uniq=error'
+        puts
+
+        puts 'Show workhorse duration/URI. Filter by duration bounds'.pastel(:bright_green)
+        puts 'gitlab-workhorse/current --duration_ms>=30000 --duration_ms<=45000 --slice=duration_ms,uri'
+        puts
+      end
       # rubocop:enable Layout/LineLength
+
       # ========================================================================
       # Search (Full Text / String Search)
       # ========================================================================
@@ -169,13 +298,15 @@ module GreenHat
 
         # Skip and Print Total if set
         if flags[:total]
-          ShellHelper.total_count(results)
+          ShellHelper.total_count(results, flags)
           return true
         end
 
         # Check Search Results
         if results.values.flatten.empty?
           puts 'No results'.pastel(:red)
+          ShellHelper::Log.no_files_warning(files) if ShellHelper.find_things(files, flags).count.zero?
+
         else
           # This causes the key 'colorized' output to also be included
           ShellHelper.show(results.to_a.compact.flatten, flags)
@@ -240,7 +371,7 @@ module GreenHat
         puts 'log search --text=BuildHooksWorker --text!=start --slice=enqueued_at sidekiq/current'
         puts
       end
-      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/MethodLength,Metrics/ModuleLength
 
       # ------------------------------------------------------------------------
     end
@@ -251,9 +382,22 @@ module GreenHat
   module ShellHelper
     # Log Helpers
     module Log
+      def self.last=(value)
+        @last = value.join(' ')
+      end
+
+      def self.last
+        @last
+      end
+
       def self.list
         Thing.all.select(&:log)
       end
+
+      def self.no_files_warning(files)
+        puts "No matching files found for pattern #{files.to_s.pastel(:yellow)}"
+        puts "See #{'ls'.pastel(:blue)} for available files"
+      end
     end
     # --------
   end