greenhat 0.6.3 → 0.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: abf4296d91fa1a63e0858e1a1ee0bc461668a6839d7ffb7ab6fbaeadae035dc1
-  data.tar.gz: c7134e262cabb6c1873dc92e56ca6318cef9c64abb4573646228ee52c36be60e
+  metadata.gz: d1d43c5b575df2757d624faaccf19b9474232d1fcdcfbc4ca5b9c7a59c8c0f0d
+  data.tar.gz: 3b3b17030206edca1b23533f1df6624a3c16e8e881c2460f84c78b4fcc0ccef4
 SHA512:
-  metadata.gz: 6ea40f3c0fc24c35722631e61ad489bd1b9eabd2d6453669e2b56476a5e1c4c883d181d062794023555598da608a69615d4f10f0bfd0c8a75bccf6e0faed8189
-  data.tar.gz: 9b947cdfc641164da1ed4d76df4e9739a01ad29a8120bc3485a534b7a0274beb3965c07dc275004e0a3320341844a00031b480e984835e91610edf77b70bf3e4
+  metadata.gz: dd7839606650408a33b2af5fbb20288a9390c16f6909bf12e55501988f5314f6ce27f85ebcea0ce9b6012073c92960bfadafd4ea59171873233a65259a860012
+  data.tar.gz: ad39e169472755c3b08bf5f8c2a6bf9bfed558f5ee7fa7318ba75439eb76d53e712e26b26c51a0e8ca6f651d90c138ff2200a63b9cddca78aae6109e1cca04e7

data/lib/greenhat/accessors/grep.rb

@@ -0,0 +1,66 @@
+module GreenHat
+  # Pretending to do grep stuffs
+  module Grep
+    def self.help
+      puts "\u2500".pastel(:cyan) * 20
+      puts "#{'Grep'.pastel(:yellow)} - gimme... GIMME!"
+      puts 'General text searching through all files (logs, parsed, and raw)'
+      puts "\u2500".pastel(:cyan) * 20
+
+      puts 'Usage'.pastel(:blue)
+      puts '  <search term> <optional: files>'.pastel(:green)
+      puts '  Examples:'
+      puts '    grep external_url'
+      puts '    grep external_url gitlab.rb'
+      puts '    grep "spaced terms"'
+      puts '    grep "System clock" timedatectl'
+      puts '    grep "search in two files" nfsiostat nfsstat'
+      puts
+    end
+
+    def self.grep(raw)
+      files, flags, _args = Args.parse(raw)
+
+      if flags.help
+        help
+        return
+      end
+
+      # Search Term
+      term = files.shift
+
+      things = find_things(files)
+
+      # Renderer
+      output = []
+      things.each do |thing|
+        results = thing.grep(term)
+
+        next if results.empty?
+
+        output.push("#{thing.friendly_name} #{results.count.to_s.pastel(:bright_black)}")
+        output.concat results
+        output.push ''
+      end
+
+      # Output
+      if output.empty?
+        puts 'No results'.pastel(:red)
+        return
+      end
+
+      ShellHelper.show GreenHat::Paper.new(data: output, flags: flags).render
+    end
+
+    def self.find_things(files = [])
+      if files.empty?
+        Thing.all
+      else
+        Thing.all.select { |x| files.any? { |f| x.name.include? f } }
+      end
+    end
+
+    # ------------------
+  end
+  # ------------------
+end

data/lib/greenhat/reports/builder.rb

@@ -6,12 +6,13 @@ module GreenHat
     class Builder
       include GreenHat::Reports::Shared
 
-      attr_accessor :store, :flags, :args
+      attr_accessor :store, :flags, :args, :raw_args
 
-      def initialize(file:, flags:, args:)
+      def initialize(file:, flags:, args:, raw_args:)
         self.store = []
         self.flags = flags
         self.args = args
+        self.raw_args = raw_args
         instance_eval File.read(file)
       rescue StandardError => e
         print "#{e.message}\n"
@@ -88,6 +89,10 @@ module GreenHat
         add(:query_format, query, show_output, block)
       end
 
+      def query_if_exists(query, show_output = true, &block)
+        add(:query_if_exists, query, show_output, block)
+      end
+
       # FastStats Helper
       def faststats(query, subcmd = '')
         add(:faststats, query, subcmd)

data/lib/greenhat/reports/helpers.rb

@@ -52,6 +52,10 @@ def query(query, show_output = true, &block)
   add(:query, query, show_output, block)
 end
 
+def query_if_exists(query, show_output = true, &block)
+  add(:query_if_exists, query, show_output, block)
+end
+
 # Log Query / Assume Format
 def query_format(query, show_output = true, &block)
   add(:query_format, query, show_output, block)

data/lib/greenhat/reports/internal_methods.rb

@@ -37,6 +37,28 @@ module GreenHat
         instance_exec(&block)
       end
 
+      # Log Query Ignore if no Files
+      def query_if_exists(query, show_output, block)
+        files, query_flags, query_args = GreenHat::Args.parse(Shellwords.split(query))
+
+        query_flags[:archive] = [archive.name] # Limit query to archive
+        query_flags[:combine] = true
+
+        # Default to everything
+        files = archive.things if files.empty?
+
+        # Ignore if no files are found
+        return unless ShellHelper.any_things?(files, query_flags)
+
+        results = Query.start(files, query_flags, query_args)
+
+        # Print and Exit of No Block / Show
+        return show(results) if block.nil? && show_output
+
+        output = instance_exec(results, &block)
+        show output if show_output
+      end
+
       # Log Query
       def query(query, show_output, block)
         files, query_flags, query_args = GreenHat::Args.parse(Shellwords.split(query))

data/lib/greenhat/reports/reports/errors.rb

@@ -24,11 +24,13 @@ entries = [
   { header: 'Sidekiq', base: 'sidekiq/current --severity=error', stats: 'message' },
   { header: 'API', base: 'gitlab-rails/api_json.log --severity=error',
     stats: 'exception.class,exception.message,status' },
-  { header: 'Gitaly', base: 'gitaly/current --level=error --truncate=99', stats: 'error' }
+  { header: 'Gitaly', base: 'gitaly/current --level=error --truncate=99', stats: 'error' },
+  { header: 'Praefect', base: 'praefect/current --level=error --truncate=99', stats: 'error' }
 ]
 
 # Filter Logic
 filters = args_select(:filter)
+
 unless filters.empty?
   entries.select! do |entry|
     filters.any? do |filter|
@@ -37,6 +39,17 @@ unless filters.empty?
   end
 end
 
+# Fuzzy Filter Match
+unless raw_args.empty?
+  entries.select! do |entry|
+    # Create a pattern with the file and title
+    globby = [entry.header.downcase, entry.base.split(' ', 2).first].join(' ')
+
+    # Check any Matches
+    raw_args.any? { |x| globby.include? x }
+  end
+end
+
 # Return output
 entries.each do |entry|
   query_string = flags[:verbose] ? entry.base : entry.base + " --stats=#{entry.stats}"

data/lib/greenhat/reports/reports/full.rb

@@ -114,37 +114,43 @@ end
 br
 puts indent('Errors'.pastel(:cyan))
 
-query 'gitlab-rails/production_json.log --status=500' do |data|
+query_if_exists 'gitlab-rails/production_json.log --status=500' do |data|
   color = data.count.zero? ? :green : :red
   indent("#{ljust('Production:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
 end
 
-query 'gitlab-rails/application_json.log --message!="Cannot obtain an exclusive lease" --severity=error' do |data|
+app_query = 'gitlab-rails/application_json.log --message!="Cannot obtain an exclusive lease" --severity=error'
+query_if_exists app_query do |data|
   color = data.count.zero? ? :green : :red
   indent("#{ljust('Application:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
 end
 
-query 'sidekiq/current --severity=error' do |data|
+query_if_exists 'sidekiq/current --severity=error' do |data|
   color = data.count.zero? ? :green : :red
   indent("#{ljust('Sidekiq:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
 end
 
-query 'gitlab-rails/api_json.log --status=500' do |data|
+query_if_exists 'gitlab-rails/api_json.log --status=500' do |data|
   color = data.count.zero? ? :green : :red
   indent("#{ljust('API:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
 end
 
-query 'gitaly/current --level=error' do |data|
+query_if_exists 'gitaly/current --level=error' do |data|
   color = data.count.zero? ? :green : :red
   indent("#{ljust('Gitaly:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
 end
 
-query 'gitlab-workhorse/current --level=error' do |data|
+query_if_exists 'gitlab-workhorse/current --level=error' do |data|
   color = data.count.zero? ? :green : :red
   indent("#{ljust('Workhorse:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
 end
 
-query 'gitlab-rails/exceptions_json.log' do |data|
+query_if_exists 'gitlab-rails/exceptions_json.log' do |data|
   color = data.count.zero? ? :green : :red
   indent("#{ljust('Exceptions:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
 end
+
+query_if_exists('praefect/current --level=error') do |data|
+  color = data.count.zero? ? :green : :red
+  indent("#{ljust('Praefect:', 14, :magenta)} #{data.count.to_s.pastel(color)}", 4)
+end

data/lib/greenhat/reports/reports/gitaly_duration.rb

@@ -0,0 +1,14 @@
+quiet!
+archive_header
+
+fields = %w[
+  duration_ms
+  time_ms
+  grpc.time_ms
+  command.cpu_time_ms
+  command.real_time_ms
+  command.system_time_ms
+  command.user_time_ms
+]
+
+query_format "gitaly/current --percentile --slice=#{fields.join(',')} --round"

data/lib/greenhat/reports/reports/production_log_duration.rb

@@ -0,0 +1,21 @@
+quiet!
+archive_header
+
+fields = %w[
+  duration_s
+  cpu_s
+  db_duration_s
+  gitaly_duration_s
+  redis_duration_s
+  target_duration_s
+  view_duration_s
+  db_main_duration_s
+  db_main_replica_duration_s
+  db_primary_duration_s
+  db_replica_duration_s
+  external_http_duration_s
+  redis_cache_duration_s
+  redis_shared_state_duration_s
+]
+
+query_format "production_json.log --percentile --slice=#{fields.join(',')} --round"

data/lib/greenhat/reports/reports/sidekiq_duration.rb

@@ -0,0 +1,15 @@
+quiet!
+archive_header
+
+fields = %w[
+  duration_s
+  db_duration_s
+  redis_duration_s
+  gitaly_duration_s
+  db_main_duration_s
+  db_replica_duration_s
+  redis_queues_duration_s
+  elasticsearch_duration_s
+]
+
+query_format "sidekiq/current --percentile --slice=#{fields.join(',')} --round"

data/lib/greenhat/reports/runner.rb

@@ -9,18 +9,19 @@ module GreenHat
 
       include InternalMethods
       include Shared
-      attr_accessor :store, :archive, :flags, :args, :output
+      attr_accessor :store, :archive, :flags, :args, :raw_args, :output
 
       # Make Pry Easier
       def inspect
         "#<Runner archive: '#{archive}'>"
       end
 
-      def initialize(archive:, store:, flags:, args:)
+      def initialize(archive:, store:, flags:, args:, raw_args:)
         self.store = store
         self.archive = archive
         self.flags = flags
         self.args = args
+        self.raw_args = raw_args
         self.output = []
 
         # Don't render paper new lines

data/lib/greenhat/reports/shared.rb

@@ -31,6 +31,7 @@ module GreenHat
       def args_select(field)
         args.select { |x| x.field == field.to_sym }
       end
+
       # -=-=-=-=-=-
     end
   end

data/lib/greenhat/reports/shell_helper.rb

@@ -3,14 +3,15 @@ module GreenHat
     # Log Helpers
     module Reports
       # Make Running more consistent
-      def self.run(file:, args:, flags:)
-        report = GreenHat::Reports::Builder.new(file: file, args: args, flags: flags)
+      def self.run(file:, args:, flags:, raw_args:)
+        report = GreenHat::Reports::Builder.new(file: file, args: args, flags: flags, raw_args: raw_args)
         output = Archive.all.map do |archive|
           runner = GreenHat::Reports::Runner.new(
             archive: archive,
             store: report.store.clone,
             flags: flags,
-            args: args
+            args: args,
+            raw_args: raw_args
           )
 
           runner.run!

data/lib/greenhat/shell/platform.rb

@@ -12,12 +12,19 @@ module GreenHat
     end
 
     def self.open(url = 'http://localhost:4567/chart/time')
-      cmd = if platform.linux? || platform.unix?
-              'xdg-open'
-            elsif platform.mac?
+      cmd = if platform.mac?
               'open'
+            elsif platform.linux? || platform.unix?
+              'xdg-open'
             end
 
+      # See if anything was detected -- ignore system call if nothing was
+      if cmd.nil?
+        puts "Unknown OS! #{RbConfig::CONFIG['arch']}"
+        puts url
+        return
+      end
+
       # platform.windows?  # => false
       # platform.unix?     # => true
       # platform.linux?    # => false

data/lib/greenhat/shell/reports.rb

@@ -47,7 +47,8 @@ module GreenHat
         run_list.uniq!
 
         run_list.each do |file|
-          GreenHat::ShellHelper::Reports.run(file: file, args: args, flags: flags)
+          raw_args = list.reject { |x| file.include? x }
+          GreenHat::ShellHelper::Reports.run(file: file, args: args, flags: flags, raw_args: raw_args)
         end
       end
     end

data/lib/greenhat/shell/shell_helper.rb

@@ -331,6 +331,11 @@ module GreenHat
       @thing_list
     end
 
+    # Shortcut to see if any things exist
+    def self.any_things?(file_list, flags = {}, base_list = nil)
+      !find_things(file_list, flags, base_list).empty?
+    end
+
     # Shortcut find things
     def self.find_things(file_list, flags = {}, base_list = nil)
       base_list ||= Thing.all

data/lib/greenhat/shell.rb

@@ -24,6 +24,10 @@ module GreenHat
       Log.default(raw_list)
     end
 
+    def self.grep(raw_list = [])
+      Grep.grep(raw_list)
+    end
+
     def self.df
       Disk.df
     end

data/lib/greenhat/thing/file_types.rb

@@ -18,16 +18,10 @@ module GreenHat
   module FileTypes
     def types
       {
-        'ifconfig' => {
+        'gitlab_rb' => {
           format: :raw,
           pattern: [
-            /ifconfig/
-          ]
-        },
-        'ip_address' => {
-          format: :raw,
-          pattern: [
-            /ip_address/
+            %r{gitlab/gitlab.rb}
           ]
         },
         'cpuinfo' => {
@@ -115,12 +109,6 @@ module GreenHat
             /getenforce/
           ]
         },
-        'gitlab.rb' => {
-          format: :raw,
-          pattern: [
-            %r{gitlab/gitlab.rb}
-          ]
-        },
         'geo-logcursor/current' => {
           format: :raw,
           pattern: [
@@ -387,6 +375,18 @@ module GreenHat
             /hostname/
           ]
         },
+        'ifconfig' => {
+          format: :raw,
+          pattern: [
+            /ifconfig/
+          ]
+        },
+        'ip_address' => {
+          format: :raw,
+          pattern: [
+            /ip_address/
+          ]
+        },
         'lscpu' => {
           format: :colon_split_strip,
           pattern: [
@@ -539,6 +539,27 @@ module GreenHat
             /^ps/
           ]
         },
+        'pressure_cpu.txt' => {
+          format: :raw,
+          log: false,
+          pattern: [
+            /pressure_cpu.txt/
+          ]
+        },
+        'pressure_io.txt' => {
+          format: :raw,
+          log: false,
+          pattern: [
+            /pressure_io.txt/
+          ]
+        },
+        'pressure_mem.txt' => {
+          format: :raw,
+          log: false,
+          pattern: [
+            /pressure_mem.txt/
+          ]
+        },
         'puma/puma_stderr.log' => {
           format: :raw,
           log: false,
@@ -547,6 +568,7 @@ module GreenHat
             %r{webservice.log/puma.stderr.log}
           ]
         },
+
         'puma/puma_stdout.log' => {
           format: :json,
           log: true,
@@ -725,12 +747,6 @@ module GreenHat
             %r{gitlab-rails/gitlab-rails-db-migrate.*}
           ]
         },
-        'rpm_verify' => {
-          format: :raw,
-          pattern: [
-            /rpm_verify/
-          ]
-        },
         'tainted' => {
           format: :raw,
           pattern: [
@@ -856,18 +872,48 @@ module GreenHat
             /vmstat/
           ]
         },
+        'gitlab-kas/config' => {
+          format: :raw,
+          pattern: [
+            %r{gitlab-kas/config}
+          ]
+        },
+        'mailroom/config' => {
+          format: :raw,
+          pattern: [
+            %r{mailroom/config}
+          ]
+        },
         'iotop' => {
           format: :raw,
           pattern: [
             /iotop/
           ]
         },
+        'top_res' => {
+          format: :raw,
+          pattern: [
+            /top_res/
+          ]
+        },
+        'top_cpu' => {
+          format: :raw,
+          pattern: [
+            /top_cpu/
+          ]
+        },
         'ntpq' => {
           format: :raw,
           pattern: [
             /ntpq/
           ]
         },
+        'rpm_verify' => {
+          format: :raw,
+          pattern: [
+            /rpm_verify/
+          ]
+        },
         'gitlab-rails/application.log' => {
           format: :raw,
           pattern: [

data/lib/greenhat/thing.rb

@@ -42,6 +42,23 @@ class Thing < Teron
     save!
   end
 
+  # Helper to make it easier to query through hashed / parsed objects
+  def hash_to_a_query
+    data.to_a.map { |x| x.join(': ') }
+  end
+
+  def grep(term)
+    selectable = if query?
+                   data
+                 elsif process?
+                   hash_to_a_query
+                 else
+                   raw_full
+                 end
+
+    selectable.select { |r| r.to_s.include?(term) }
+  end
+
   # Processor
   def data
     process unless parsed
@@ -91,7 +108,7 @@ class Thing < Teron
 
   # Things that can be queried (Query Helper)
   def query?
-    data.instance_of?(Array)
+    data.instance_of?(Array) || data.instance_of?(Enumerator)
   end
 
   # Helper for all things that can be hash/value filtered

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.6.3'.freeze
+  VERSION = '0.6.6'.freeze
 end

data/lib/greenhat.rb

@@ -22,7 +22,7 @@ require 'tty-reader'
 require 'tty-spinner'
 require 'tty-table'
 require 'tty-which'
-require 'warning'
+# require 'warning' # Ruby 3
 require 'ruby_native_statistics'
 require 'gitlab_chronic_duration'
 
@@ -79,4 +79,4 @@ require 'greenhat/tty/reader'
 require 'greenhat/tty/custom_line'
 require 'greenhat/tty/columns'
 
-Warning.ignore(/The table size exceeds the currently set width/)
+# Warning.ignore(/The table size exceeds the currently set width/) # Ruby 3

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: greenhat
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.6
 platform: ruby
 authors:
 - Davin Walker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-01 00:00:00.000000000 Z
+date: 2022-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: actionview
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +234,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
 - !ruby/object:Gem::Dependency
   name: require_all
   requirement: !ruby/object:Gem::Requirement
@@ -276,6 +290,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: slim
   requirement: !ruby/object:Gem::Requirement
@@ -430,20 +458,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-- !ruby/object:Gem::Dependency
-  name: warning
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
 description: Experimental SOS and Log Parser for GitLab
 email:
 - dwalker@gitlab.com
@@ -457,6 +471,7 @@ files:
 - lib/greenhat.rb
 - lib/greenhat/accessors/disk.rb
 - lib/greenhat/accessors/gitlab.rb
+- lib/greenhat/accessors/grep.rb
 - lib/greenhat/accessors/logs/production.rb
 - lib/greenhat/accessors/logs/sidekiq.rb
 - lib/greenhat/accessors/memory.rb
@@ -479,6 +494,9 @@ files:
 - lib/greenhat/reports/reports/faststats.rb
 - lib/greenhat/reports/reports/full.rb
 - lib/greenhat/reports/reports/full_markdown.rb
+- lib/greenhat/reports/reports/gitaly_duration.rb
+- lib/greenhat/reports/reports/production_log_duration.rb
+- lib/greenhat/reports/reports/sidekiq_duration.rb
 - lib/greenhat/reports/runner.rb
 - lib/greenhat/reports/shared.rb
 - lib/greenhat/reports/shell_helper.rb
@@ -590,14 +608,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: 2.7.5
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: GitLab SOS Tool