greenhat 0.3.2 → 0.3.6

data/lib/greenhat/shell/shell_helper.rb

@@ -103,7 +103,8 @@ module GreenHat
                           StringColor.do(key, entry)
                         end
 
-      if flags[:truncate]
+      # Stats truncation handled separately
+      if flags[:truncate] && !flags[:stats]
         entry_truncate(formatted_entry, flags[:truncate])
       else
         formatted_entry
@@ -150,8 +151,21 @@ module GreenHat
       format_table_entry(flags, val)
     end
 
+    # Internal Query Helper
+    # query = 'gitlab-rails/application_json.log --message!="Cannot obtain an exclusive lease" --severity=error'
+    # ShellHelper.filter_internal(query)
+    def self.filter_internal(search = '')
+      files, flags, args = Args.parse(Shellwords.split(search))
+      flags[:combine] = true
+
+      # Default to everything
+      files = Thing.all.map(&:name) if files.empty?
+
+      ShellHelper.filter_start(files, flags, args)
+    end
+
     # Main Entry Point for Filtering
-    def self.filter_start(files, flags, args)
+    def self.filter_start(files, flags = {}, args = {})
       # Convert to Things
       logs = ShellHelper.find_things(files, flags).select(&:processed?)
 
@@ -167,11 +181,17 @@ module GreenHat
 
           # Include Total Count in Name
           results = ShellHelper.filter(log.data, flags, args)
+          duration = calculate_duration(results)
+
           title = [
             log.friendly_name,
             " #{results.count}".pastel(:bright_black)
+
           ]
 
+          # Append Duration
+          title.push(" #{duration.pastel(:cyan, :dim)}") unless duration.blank?
+
           # Save unless empty
           obj[title.join] = results unless results.count.zero?
 
@@ -180,6 +200,31 @@ module GreenHat
       end
     end
 
+    def self.calculate_duration(results)
+      only_with_time = results.select(&:time)
+
+      # If slice is used ignore
+      return nil if only_with_time.empty?
+
+      sorted = only_with_time.map(&:time).sort
+      humanize_time(sorted.first, sorted.last)
+    end
+
+    # Replace TimeDifference with https://stackoverflow.com/a/4136485/1678507
+    def self.humanize_time(time_start, time_end, increments = 2)
+      miliseconds = (time_end - time_start) * 1000
+
+      list = [[1000, :ms], [60, :s], [60, :m], [24, :h]].map do |count, name|
+        next unless miliseconds.positive?
+
+        miliseconds, n = miliseconds.divmod(count)
+
+        "#{n.to_i}#{name}" unless n.to_i.zero?
+      end
+
+      list.compact.reverse[0..increments - 1].join(' ')
+    end
+
     # Filter Logic
     # TODO: Simplify
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
@@ -189,7 +234,7 @@ module GreenHat
       # Experimenting with deep clone
       results = Marshal.load(Marshal.dump(data))
       results.select! do |row|
-        args.send(flags.logic) do |arg|
+        args.send(flags.logic || :all?) do |arg|
           filter_row_key(row, arg, flags)
         end
       end
@@ -255,7 +300,6 @@ module GreenHat
     end
 
     # Filter Start and End Times
-    # rubocop:disable Metrics/MethodLength
     # TODO: This is a bit icky, simplify/dry
     def self.filter_time(results, flags)
       if flags.key?(:start)
@@ -292,7 +336,6 @@ module GreenHat
 
       results
     end
-    # rubocop:enable Metrics/MethodLength
 
     def self.filter_except(results, except)
       # Avoid Empty Results
@@ -368,7 +411,10 @@ module GreenHat
 
       # Loop through Stats, Separate Hash/Tables
       stats.map do |field|
-        occurrences = filter_count_occurrences(results, field)
+        occurrences = filter_count_occurrences(results, field, flags)
+
+        # Use Truncate For Long Keys
+        occurrences.transform_keys! { |key| key.to_s[0..flags[:truncate]] } if flags[:truncate]
 
         # Total Occurences
         total = occurrences.values.sum
@@ -387,13 +433,6 @@ module GreenHat
         # Append Header / Total with field name
         output.unshift([field.to_s.pastel(:bright_black), total])
 
-        # Use Truncate For Long Keys
-        if flags[:truncate]
-          output.map! do |key, value|
-            [key.to_s[0..flags[:truncate]], value]
-          end
-        end
-
         # Format
         output.to_h
       end
@@ -405,10 +444,17 @@ module GreenHat
     end
 
     # Helper to Count occurrences
-    def self.filter_count_occurrences(results, field)
+    def self.filter_count_occurrences(results, field, flags = {})
       results.each_with_object(Hash.new(0)) do |entry, counts|
         if entry.key? field
-          counts[entry[field]] += 1
+          # Rounding in pagination breaks stats
+          key = if flags.key?(:round) && entry[field].numeric?
+                  entry[field].to_f.round(flags.round)
+                else
+                  entry[field]
+                end
+
+          counts[key] += 1
         else
           counts['None'.pastel(:bright_black)] += 1
         end
@@ -426,48 +472,83 @@ module GreenHat
     end
 
     # Break out filter row logic into separate method
-
     def self.filter_row_key(row, arg, flags)
       # Ignore Other Logic if Field isn't even included / Full Text Searching
       return false unless row.key?(arg[:field]) || arg[:field] == :text
 
       # Sensitivity Check / Check for Match / Full Text Searching
-      included = if arg[:field] == :text
-                   filter_row_entry(row.to_s, arg, flags)
-                 else
-                   filter_row_entry(row[arg.field].to_s, arg, flags)
-                 end
+      search_data = arg[:field] == :text ? row : row[arg.field]
+      match = filter_row_entry(search_data.to_s, arg, flags)
 
       # Pivot of off include vs exclude
       if arg.bang
-        !included
+        !match
       else
-        included
+        match
       end
     end
 
     # Field Partial / Case / Exact search
     def self.filter_row_entry(entry, arg, flags)
       # Exact Matching / Unless doing full text search
-      return entry.to_s == arg.value.to_s if flags.key?(:exact) && arg.field != :text
+      return entry == arg.value.to_s if flags.key?(:exact) && arg.field != :text
 
-      if flags.key?(:case)
-        entry.include? arg.value.to_s
-      else
-        entry.downcase.include? arg.value.to_s.downcase
+      # Cast to String/Integer Helper
+      entry, value = filter_entry_cast(entry, arg, flags)
+
+      entry.send(arg.logic, value)
+    end
+
+    # Handle casting to strings or integers
+    def self.filter_entry_cast(entry, arg, flags)
+      # Cast to String
+      value = arg.value.to_s
+
+      case arg.logic
+      when :include?
+
+        # Exact Case argument
+        unless flags.key?(:case)
+          entry.downcase!
+          value.downcase!
+        end
+      when :>=, :<=
+        entry = entry.to_i if entry.numeric?
+        value = value.to_i if value&.numeric?
       end
+
+      [entry, value]
+    end
+
+    def self.filter_entry_logic(entry, arg)
+      entry.send(arg.logic, arg.value)
     end
 
     # Total Count Helper
-    def self.total_count(results)
-      results.each do |k, v|
+    def self.total_count(results, flags)
+      results.map do |k, v|
         puts k
-        puts "Total: #{v.count.to_s.pastel(:blue)}"
+        # puts "Total: #{v.count.to_s.pastel(:blue)}"
+        # puts "Time Covered: #{calculate_duration(v).pastel(:cyan)}"
+
+        output = {}
+        output[:total] = v.count
+        output[:duration] = calculate_duration(v)
+
+        # Sort / Get first and Last
+        list = v.select(&:time).map(&:time)
+        unless list.blank?
+          output[:start] = list.first
+          output[:end] = list.last
+        end
+
+        puts render_table(output, flags)
+
         puts
       end
     end
 
-    # Total Count Helper
+    # Table Printer Helper
     def self.fields_print(results)
       results.each do |k, v|
         puts k
@@ -531,7 +612,7 @@ module GreenHat
         return file if file.instance_of?(Thing)
 
         if flags.fuzzy_file_match
-          Thing.all.select { |x| x.name.include? file }
+          Thing.all.select { |x| x.name.include?(file) || x.type.include?(file) }
         else
           Thing.where name: file
         end
@@ -543,62 +624,6 @@ module GreenHat
       things
     end
 
-    # Main Entry Point for Searching
-    # def self.search_start(log_list, filter_type, args, opts)
-    def self.search_start(files, flags, args)
-      # Convert to Things
-      logs = ShellHelper.find_things(files, flags)
-
-      logs.each_with_object({}) do |log, obj|
-        # Ignore Empty Results / No Thing
-        next if log&.data.blank?
-
-        obj[log.friendly_name] = ShellHelper.search(log.data, flags, args)
-
-        obj
-      end
-    end
-
-    # Generic Search Helper / String/Regex
-    def self.search(data, flags = {}, args = {})
-      results = data.clone.flatten.compact
-      results.select! do |row|
-        args.send(flags.logic) do |arg|
-          search_row(row, arg, flags)
-        end
-      end
-
-      # Strip Results if Slice is defined
-      results.map! { |row| row.slice(*flags[:slice]) } if flags[:slice]
-
-      # Strip Results if Except is defined
-      results.map! { |row| row.except(*flags[:except]) } if flags[:except]
-
-      # Remove Blank from either slice or except
-      results.reject!(&:empty?)
-
-      results
-    end
-
-    # Break out filter row logic into separate method
-    def self.search_row(row, arg, flags)
-      # Sensitivity Check / Check for Match
-      included = filter_row_entry(row.to_s, arg, flags)
-
-      # Pivot of off include vs exclude
-      if arg.bang
-        !included
-      else
-        included
-      end
-    end
-
-    # TODO: Remove?
-    # Color Reader Helper
-    # def self.pastel
-    #   @pastel ||= Pastel.new
-    # end
-
     # Number Helper
     # https://gitlab.com/zedtux/human_size_to_number/-/blob/master/lib/human_size_to_number/helper.rb
     def self.human_size_to_number(string)
@@ -620,23 +645,6 @@ module GreenHat
       number.round
     end
 
-    # TODO: Needed?
-    def self.filter_and(data, params = {})
-      result = data.clone.flatten.compact
-      params.each do |k, v|
-        result.select! do |row|
-          if row.key? k.to_sym
-            row[k.to_sym].include? v
-          else
-            false
-          end
-        end
-        next
-      end
-
-      result
-    end
-
     # General Helper for `show`
     def self.common_opts
       puts 'Common Options'.pastel(:blue)

data/lib/greenhat/shell.rb

@@ -7,6 +7,13 @@ module GreenHat
       # rubocop:enable Lint/Debugger
     end
 
+    def self.web
+      # Load Required Files
+      require 'greenhat/web'
+
+      GreenHat::Web.toggle
+    end
+
     def self.df
       Disk.df
     end

data/lib/greenhat/thing/file_types.rb

@@ -43,6 +43,13 @@ module GreenHat
             /dmesg/
           ]
         },
+        'lets-encrypt/renewal' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            %r{lets-encrypt/renewal}
+          ]
+        },
         'repmgrd/current' => {
           format: :bracket_log,
           log: true,
@@ -87,6 +94,12 @@ module GreenHat
             /getenforce/
           ]
         },
+        'geo-logcursor/current' => {
+          format: :raw,
+          pattern: [
+            %r{geo-logcursor/current}
+          ]
+        },
         'gitaly/current' => {
           format: :json_shell,
           log: true,
@@ -225,7 +238,8 @@ module GreenHat
           format: :json,
           log: true,
           pattern: [
-            %r{gitlab-rails/audit_json.log}
+            %r{gitlab-rails/audit_json.log},
+            %r{webservice.log/audit_json.log}
           ]
         },
         'gitlab-rails/auth.log' => {
@@ -336,6 +350,20 @@ module GreenHat
             %r{nginx/current}
           ]
         },
+        'nginx-ingress.log' => {
+          format: :kube_nginx,
+          log: true,
+          pattern: [
+            /nginx-ingress.log/
+          ]
+        },
+        'nginx/access.log' => {
+          format: :nginx,
+          log: true,
+          pattern: [
+            %r{nginx/access.log}
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
           format: :nginx,
           log: true,
@@ -422,7 +450,7 @@ module GreenHat
           ]
         },
         'registry/current' => {
-          format: :time_shellwords,
+          format: :time_registry,
           log: true,
           pattern: [
             %r{registry/current},
@@ -444,16 +472,18 @@ module GreenHat
         },
         'puma/puma_stderr.log' => {
           format: :raw,
-          log: true,
+          log: false,
           pattern: [
-            %r{puma/puma_stderr.log}
+            %r{puma/puma_stderr.log},
+            %r{webservice.log/puma.stderr.log}
           ]
         },
         'puma/puma_stdout.log' => {
           format: :json,
           log: true,
           pattern: [
-            %r{puma/puma_stdout.log}
+            %r{puma/puma_stdout.log},
+            %r{webservice.log/puma.stdout.log}
           ]
         },
         'gitlab-pages/current' => {
@@ -529,6 +559,20 @@ module GreenHat
             %r{unicorn/current}
           ]
         },
+        'unicorn_stats' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            /unicorn_stats/
+          ]
+        },
+        'unicorn/unicorn_stdout.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{unicorn/unicorn_stdout.log}
+          ]
+        },
         'running_swappiness' => {
           format: :raw,
           pattern: [
@@ -566,6 +610,7 @@ module GreenHat
           log: true,
           pattern: [
             %r{sidekiq/current},
+            /sidekiq_current/,
             %r{sidekiq/@.*}
           ]
         },
@@ -629,6 +674,12 @@ module GreenHat
             /uname/
           ]
         },
+        'gitlab_socket_stats' => {
+          format: :gitlab_socket_stats,
+          pattern: [
+            /gitlab_socket_stats/
+          ]
+        },
         'uptime' => {
           format: :raw,
           pattern: [
@@ -745,13 +796,15 @@ module GreenHat
         'gitlab-rails/application.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/application.log}
+            %r{gitlab-rails/application.log},
+            %r{webservice.log/application.log}
           ]
         },
         'gitlab-rails/production.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/production.log}
+            %r{gitlab-rails/production.log},
+            %r{webservice.log/production.log}
           ]
         },
         'gitlab/version-manifest.txt' => {
@@ -765,6 +818,72 @@ module GreenHat
           pattern: [
             %r{sidekiq/perf.data}
           ]
+        },
+
+        # ======================================================================
+        # KubeSoS TODO Section
+        # Things I am going to shortcut and set to raw for now
+        # ======================================================================
+        # Attempted Parsing
+        'kubesos_json' => {
+          log: true,
+          format: :kube_json,
+          pattern: [
+            /gitaly.log/
+          ]
+        },
+
+        'kube_webservice' => {
+          log: false,
+          format: :kube_webservice,
+          pattern: [
+            /^webservice\.log$/
+          ]
+        },
+
+        'kubesos' => {
+          format: :raw,
+          pattern: [
+            /all_values.yaml/,
+            %r{webservice.log/sidekiq_client.log},
+            /chart-version/,
+            /configmaps/,
+            /describe_deployments/,
+            /describe_ingress/,
+            /describe_nodes/,
+            /describe_pods/,
+            /describe_pv/,
+            /describe_pvc/,
+            /events/,
+            /get_deployments/,
+            /get_endpoints/,
+            /get_jobs/,
+            /get_pods/,
+            /get_pv/,
+            /get_pvc/,
+            /get_services/,
+            /gitaly.log/,
+            /gitlab-exporter.log/,
+            /gitlab-pages.log/,
+            /gitlab-shell.log/,
+            /grafana.log/,
+            /helm-version/,
+            /kubectl-check/,
+            /migrations.log/,
+            /minio.log/,
+            /nfs-client-provisioner.log/,
+            /operator.log/,
+            /postgresql.log/,
+            /prometheus.log/,
+            /redis.log/,
+            /registry.log/,
+            /secrets/,
+            /sidekiq.log/,
+            /task-runner.log/,
+            /top_nodes/,
+            /top_pods/,
+            /user_supplied_values.yaml/
+          ]
         }
       }
     end

data/lib/greenhat/thing/formatters/json.rb

@@ -43,6 +43,10 @@ module GreenHat
     # Check for Common Fields
     def format_json_time(result)
       result.time = format_time_parse(result.time) if result.key? :time
+
+      # If timestamp exists but time doesn't, add time
+      result.time = format_time_parse(result.timestamp) if result.key?(:timestamp) && !result.key?(:time)
+
       result.created_at = format_time_parse(result.created_at) if result.key? :created_at
       result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
     rescue StandardError => e

data/lib/greenhat/thing/formatters/kube_json.rb

@@ -0,0 +1,36 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # K8s logs seem to have a lot of non-json entries
+    # Extract all json, ignore everything else
+    # ==========================================================================
+    def format_kube_json
+      self.result = raw.map do |row|
+        # Skip all non-json
+        next unless row.first == '{'
+
+        result = begin
+          Oj.load row
+        rescue EncodingError
+          puts
+          next
+        end
+
+        # Parsing Time
+        format_json_traverse result
+
+        result.sort.to_h
+      rescue StandardError => e
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('JSON Parse', e.message)
+        next
+      end.compact
+
+      :ok
+    end
+    # =========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/kube_nginx.rb

@@ -0,0 +1,48 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    def format_kube_nginx
+      self.result = raw.map do |row|
+        ip, _sym, remote_user, rest = row.split(' ', 4)
+
+        time, rest = rest.split(']', 2)
+        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+
+        verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
+
+        method, path, http_version = verb.split
+
+        {
+          remote_addr: ip,
+          remote_user: remote_user,
+          method: method,
+          path: path,
+          status: status,
+          bytes: bytes,
+          http_version: http_version,
+          http_referer: http_referer,
+          http_user_agent: http_user_agent,
+          gzip_ratio: gzip_ratio,
+          time: time
+        }
+
+        # Fall back for malformed logs
+      rescue StandardError
+        { message: row }
+      end
+
+      :ok
+    end
+
+    # rubocop:disable Layout/LineLength
+    # NGINX Conf: /var/opt/gitlab/nginx/conf/nginx.conf
+    # log_format gitlab_access '$remote_addr - $remote_user [$time_local] "$request_method $filtered_request_uri $server_protocol" $status $body_bytes_sent "$filtered_http_referer" "$http_user_agent" $gzip_ratio';
+    # rubocop:enable Layout/LineLength
+
+    # ==========================================================================
+  end
+end