greenhat 0.3.0 → 0.3.4

data/lib/greenhat/thing/file_types.rb

@@ -43,6 +43,13 @@ module GreenHat
             /dmesg/
           ]
         },
+        'lets-encrypt/renewal' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            %r{lets-encrypt/renewal}
+          ]
+        },
         'repmgrd/current' => {
           format: :bracket_log,
           log: true,
@@ -225,7 +232,8 @@ module GreenHat
           format: :json,
           log: true,
           pattern: [
-            %r{gitlab-rails/audit_json.log}
+            %r{gitlab-rails/audit_json.log},
+            %r{webservice.log/audit_json.log}
           ]
         },
         'gitlab-rails/auth.log' => {
@@ -336,15 +344,22 @@ module GreenHat
             %r{nginx/current}
           ]
         },
+        'nginx-ingress.log' => {
+          format: :kube_nginx,
+          log: true,
+          pattern: [
+            /nginx-ingress.log/
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
-          format: :raw,
+          format: :nginx,
           log: true,
           pattern: [
             %r{nginx/gitlab_pages_access.log}
           ]
         },
         'nginx/gitlab_access.log' => {
-          format: :raw,
+          format: :nginx,
           log: true,
           pattern: [
             %r{nginx/gitlab_access.log}
@@ -422,7 +437,7 @@ module GreenHat
           ]
         },
         'registry/current' => {
-          format: :time_shellwords,
+          format: :time_registry,
           log: true,
           pattern: [
             %r{registry/current},
@@ -444,16 +459,18 @@ module GreenHat
         },
         'puma/puma_stderr.log' => {
           format: :raw,
-          log: true,
+          log: false,
           pattern: [
-            %r{puma/puma_stderr.log}
+            %r{puma/puma_stderr.log},
+            %r{webservice.log/puma.stderr.log}
           ]
         },
         'puma/puma_stdout.log' => {
           format: :json,
           log: true,
           pattern: [
-            %r{puma/puma_stdout.log}
+            %r{puma/puma_stdout.log},
+            %r{webservice.log/puma.stdout.log}
           ]
         },
         'gitlab-pages/current' => {
@@ -489,7 +506,8 @@ module GreenHat
             %r{pgbouncer-exporter/config},
             %r{repmgrd/config},
             %r{sentinel/config},
-            %r{consul/config}
+            %r{consul/config},
+            %r{mailroom/current}
           ]
         },
         'redis/current' => {
@@ -500,7 +518,6 @@ module GreenHat
             %r{redis/@.*}
           ]
         },
-
         'nginx/gitlab_error.log' => {
           format: :time_space,
           log: true,
@@ -508,6 +525,13 @@ module GreenHat
             %r{nginx/gitlab_error.log}
           ]
         },
+        'nginx/error.log' => {
+          format: :raw,
+          log: false,
+          pattern: [
+            %r{nginx/error.log}
+          ]
+        },
         'nginx/gitlab_registry_error.log' => {
           format: :time_space,
           log: true,
@@ -529,7 +553,7 @@ module GreenHat
           ]
         },
         'nginx/gitlab_registry_access.log' => {
-          format: :raw,
+          format: :nginx,
           log: true,
           pattern: [
             %r{nginx/gitlab_registry_access.log}
@@ -738,13 +762,15 @@ module GreenHat
         'gitlab-rails/application.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/application.log}
+            %r{gitlab-rails/application.log},
+            %r{webservice.log/application.log}
           ]
         },
         'gitlab-rails/production.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/production.log}
+            %r{gitlab-rails/production.log},
+            %r{webservice.log/production.log}
           ]
         },
         'gitlab/version-manifest.txt' => {
@@ -758,6 +784,72 @@ module GreenHat
           pattern: [
             %r{sidekiq/perf.data}
           ]
+        },
+
+        # ======================================================================
+        # KubeSoS TODO Section
+        # Things I am going to shortcut and set to raw for now
+        # ======================================================================
+        # Attempted Parsing
+        'kubesos_json' => {
+          log: true,
+          format: :kube_json,
+          pattern: [
+            /gitaly.log/
+          ]
+        },
+
+        'kube_webservice' => {
+          log: false,
+          format: :kube_webservice,
+          pattern: [
+            /^webservice\.log$/
+          ]
+        },
+
+        'kubesos' => {
+          format: :raw,
+          pattern: [
+            /all_values.yaml/,
+            %r{webservice.log/sidekiq_client.log},
+            /chart-version/,
+            /configmaps/,
+            /describe_deployments/,
+            /describe_ingress/,
+            /describe_nodes/,
+            /describe_pods/,
+            /describe_pv/,
+            /describe_pvc/,
+            /events/,
+            /get_deployments/,
+            /get_endpoints/,
+            /get_jobs/,
+            /get_pods/,
+            /get_pv/,
+            /get_pvc/,
+            /get_services/,
+            /gitaly.log/,
+            /gitlab-exporter.log/,
+            /gitlab-pages.log/,
+            /gitlab-shell.log/,
+            /grafana.log/,
+            /helm-version/,
+            /kubectl-check/,
+            /migrations.log/,
+            /minio.log/,
+            /nfs-client-provisioner.log/,
+            /operator.log/,
+            /postgresql.log/,
+            /prometheus.log/,
+            /redis.log/,
+            /registry.log/,
+            /secrets/,
+            /sidekiq.log/,
+            /task-runner.log/,
+            /top_nodes/,
+            /top_pods/,
+            /user_supplied_values.yaml/
+          ]
         }
       }
     end

data/lib/greenhat/thing/formatters/kube_json.rb

@@ -0,0 +1,36 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # K8s logs seem to have a lot of non-json entries
+    # Extract all json, ignore everything else
+    # ==========================================================================
+    def format_kube_json
+      self.result = raw.map do |row|
+        # Skip all non-json
+        next unless row.first == '{'
+
+        result = begin
+          Oj.load row
+        rescue EncodingError
+          puts
+          next
+        end
+
+        # Parsing Time
+        format_json_traverse result
+
+        result.sort.to_h
+      rescue StandardError => e
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('JSON Parse', e.message)
+        next
+      end.compact
+
+      :ok
+    end
+    # =========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/kube_nginx.rb

@@ -0,0 +1,48 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    def format_kube_nginx
+      self.result = raw.map do |row|
+        ip, _sym, remote_user, rest = row.split(' ', 4)
+
+        time, rest = rest.split(']', 2)
+        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+
+        verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
+
+        method, path, http_version = verb.split
+
+        {
+          remote_addr: ip,
+          remote_user: remote_user,
+          method: method,
+          path: path,
+          status: status,
+          bytes: bytes,
+          http_version: http_version,
+          http_referer: http_referer,
+          http_user_agent: http_user_agent,
+          gzip_ratio: gzip_ratio,
+          time: time
+        }
+
+        # Fall back for malformed logs
+      rescue StandardError
+        { message: row }
+      end
+
+      :ok
+    end
+
+    # rubocop:disable Layout/LineLength
+    # NGINX Conf: /var/opt/gitlab/nginx/conf/nginx.conf
+    # log_format gitlab_access '$remote_addr - $remote_user [$time_local] "$request_method $filtered_request_uri $server_protocol" $status $body_bytes_sent "$filtered_http_referer" "$http_user_agent" $gzip_ratio';
+    # rubocop:enable Layout/LineLength
+
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/kube_webservice.rb

@@ -0,0 +1,51 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Gitlab Tail Formatter
+    # ==========================================================================
+    def format_kube_webservice
+      # Revert to raw for cats
+      self.kind = :raw
+
+      output = {}
+      current_log = nil
+
+      raw.each do |line|
+        next if line.blank?
+
+        if line.include? '*** '
+          current_log = /\*\*\* (.+?) \*\*\*/.match(line).captures.first
+        else
+          output[current_log] ||= [] unless current_log.nil?
+          output[current_log].push line unless current_log.nil?
+        end
+      end
+
+      # Remove Empty Entries
+      output.reject { |_k, v| v.empty? }
+
+      # Root Dir
+      root_dir = "#{$TMP}/#{name}"
+      Dir.mkdir(root_dir)
+
+      # Write Files / Create Things
+      output.each do |k, v|
+        file_name = k.gsub('/var/log/gitlab/', '')
+
+        dir = "#{root_dir}/gitlab-rails"
+        Dir.mkdir(dir) unless File.exist?(dir)
+
+        File.write("#{root_dir}/#{file_name}", v.join("\n"))
+
+        # Thing Setup
+        archive.things_create(file: "#{root_dir}/#{file_name}").setup
+      end
+
+      # Link
+      self.result = raw
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/nginx.rb

@@ -0,0 +1,44 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    def format_nginx
+      self.result = raw.map do |row|
+        ip, _sym, remote_user, rest = row.split(' ', 4)
+
+        time, rest = rest.split(']', 2)
+        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+
+        verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
+
+        method, path, http_version = verb.split
+
+        {
+          remote_addr: ip,
+          remote_user: remote_user,
+          method: method,
+          path: path,
+          status: status,
+          bytes: bytes,
+          http_version: http_version,
+          http_referer: http_referer,
+          http_user_agent: http_user_agent,
+          gzip_ratio: gzip_ratio,
+          time: time
+        }
+      end
+
+      :ok
+    end
+
+    # rubocop:disable Layout/LineLength
+    # NGINX Conf: /var/opt/gitlab/nginx/conf/nginx.conf
+    # log_format gitlab_access '$remote_addr - $remote_user [$time_local] "$request_method $filtered_request_uri $server_protocol" $status $body_bytes_sent "$filtered_http_referer" "$http_user_agent" $gzip_ratio';
+    # rubocop:enable Layout/LineLength
+
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/registry.rb

@@ -0,0 +1,47 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Registry Split
+    # ==========================================================================
+    # https://docs.docker.com/registry/configuration/#log
+    # Formatters text, json
+    # registry['log_formatter'] = "json"
+    # TODO: Logstash (Not working in 14.3)
+
+    def format_time_registry
+      self.result = raw.map do |row|
+        result = row[0] == '{' ? registry_json(row) : registry_shell_words(row)
+
+        # Timestamp Parsing
+        result.ts = Time.parse result.ts if result.key? 'ts'
+        result.time = Time.parse(result.time)
+
+        result
+      end
+    end
+
+    def registry_shell_words(row)
+      time, msg = row.split(' ', 2)
+
+      output = Shellwords.split(msg).each_with_object({}) do |x, h|
+        key, value = x.split('=')
+        next if value.nil?
+
+        h[key.to_sym] = value.numeric? ? value.to_f : value
+      end
+
+      output[:time] = time
+
+      output
+    end
+
+    def registry_json(msg)
+      Oj.load(msg)
+    rescue EncodingError
+      { message: msg }
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing.rb

@@ -26,6 +26,7 @@ class Thing < Teron
   field :parsed # Flag for Parsing
   field :result # Processed Data
   field :raw_result # Flag for reading raw data
+  field :result_fields # All available fields
 
   def friendly_name
     "#{archive.friendly_name.pastel(:blue)} #{name.pastel(:green)}"
@@ -49,6 +50,13 @@ class Thing < Teron
     result
   end
 
+  # Processor
+  def fields
+    process unless parsed
+
+    result_fields
+  end
+
   def raw
     raw_read if raw_result.nil?
 
@@ -116,6 +124,20 @@ class Thing < Teron
 
     self.parsed = true
 
+    self.result_fields = field_processing
+
     save!
   end
+
+  def field_processing
+    if data.instance_of?(Array)
+      data.select { |x| x.instance_of?(Hash) }.map(&:keys).flatten.uniq
+    else
+      []
+    end
+  rescue StandardError => e
+    LogBot.fatal('Process', message: e.message, backtrace: e.backtrace.first)
+
+    []
+  end
 end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.3.4'.freeze
 end

data/lib/greenhat.rb

@@ -21,6 +21,7 @@ require 'tty-prompt'
 require 'tty-reader'
 require 'tty-spinner'
 require 'tty-table'
+require 'tty-which'
 require 'warning'
 
 # Custom Gem

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: greenhat
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.4
 platform: ruby
 authors:
 - Davin Walker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-01 00:00:00.000000000 Z
+date: 2021-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -360,6 +360,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: tty-which
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: warning
   requirement: !ruby/object:Gem::Requirement
@@ -405,11 +419,13 @@ files:
 - lib/greenhat/shell/color_string.rb
 - lib/greenhat/shell/disk.rb
 - lib/greenhat/shell/faststats.rb
+- lib/greenhat/shell/field_helper.rb
 - lib/greenhat/shell/filter_help.rb
 - lib/greenhat/shell/gitlab.rb
 - lib/greenhat/shell/help.rb
 - lib/greenhat/shell/list.rb
 - lib/greenhat/shell/log.rb
+- lib/greenhat/shell/markdown.rb
 - lib/greenhat/shell/memory.rb
 - lib/greenhat/shell/network.rb
 - lib/greenhat/shell/page.rb
@@ -429,8 +445,13 @@ files:
 - lib/greenhat/thing/formatters/gitlab_status.rb
 - lib/greenhat/thing/formatters/json.rb
 - lib/greenhat/thing/formatters/json_shellwords.rb
+- lib/greenhat/thing/formatters/kube_json.rb
+- lib/greenhat/thing/formatters/kube_nginx.rb
+- lib/greenhat/thing/formatters/kube_webservice.rb
 - lib/greenhat/thing/formatters/multiline_json.rb
+- lib/greenhat/thing/formatters/nginx.rb
 - lib/greenhat/thing/formatters/raw.rb
+- lib/greenhat/thing/formatters/registry.rb
 - lib/greenhat/thing/formatters/shellwords.rb
 - lib/greenhat/thing/formatters/syslog.rb
 - lib/greenhat/thing/formatters/table.rb