greenhat 0.1.4

data/lib/greenhat/thing.rb

@@ -0,0 +1,121 @@
+# Top Level what everything is
+class Thing < Teron
+  include GreenHat::Formatters
+  include GreenHat::ThingHelpers
+  include GreenHat::Spinner
+  include GreenHat::Kind
+  include GreenHat::FileTypes
+  include ActionView::Helpers::DateHelper
+  include ActionView::Helpers::NumberHelper
+
+  belongs_to :host
+  belongs_to :archive
+
+  # File
+  field :file # Full File Path / File Load
+  field :name # Base File Name
+  field :path # Symbol Name Helper
+  field :size # Total File Size
+
+  # Classification
+  field :kind # File Information
+  field :type # Shared Naming Convention (Based off of SOS)
+  field :log # Parsable / Include in log formats
+
+  # Preloader Flags
+  field :parsed # Flag for Parsing
+  field :result # Processed Data
+  field :raw_result # Flag for reading raw data
+
+  def friendly_name
+    "#{archive.friendly_name.colorize(:blue)} #{name.colorize(:green)} #{id.colorize(:black)}"
+  end
+
+  def setup
+    self.name = build_path('/')
+    self.path = build_path.gsub(/[^\w\s]/, '_')
+    self.size = number_to_human_size File.size(file)
+
+    kind_collect # Set Kind and Type
+    kind_setup
+
+    save!
+  end
+
+  # Processor
+  def data
+    process unless parsed
+
+    result
+  end
+
+  def raw
+    raw_read if raw_result.nil?
+
+    raw_result
+  end
+
+  # Filter / Archive Pattern Matching
+  def archive?(archive_names)
+    archive_names.map(&:to_s).any? { |x| archive.name.include? x }
+  end
+
+  # Helper for empty logs
+  def blank?
+    data.blank?
+  end
+
+  def raw_read
+    spin_start("Read  #{name.colorize(:blue)} #{size.colorize(:light_black)}")
+    self.raw_result = File.read(file).split("\n")
+  rescue StandardError => e
+    LogBot.fatal('Raw Read', message: e.message, backtrace: e.backtrace.first)
+    self.raw_result = ''
+  ensure
+    spin_done
+  end
+
+  def output(print_it = true)
+    if print_it
+      puts raw.join("\n")
+    else
+      raw
+    end
+  end
+
+  # Log Identifier Helper
+  def log?
+    GreenHat::SuperLog.log? kind
+  end
+
+  # Hashed values searching
+  def processed?
+    kind != :raw
+  end
+
+  # Helper Formatter Method
+  def formatter
+    "format_#{kind}".to_sym
+  end
+
+  def process
+    # Read First if Needed (Spinner)
+    raw_read if raw_result.nil?
+
+    if methods.include? formatter
+      spin_start("Parse #{name.colorize(:blue)} #{kind.to_s.colorize(:light_black)} ")
+      begin
+        send(formatter)
+      rescue StandardError => e
+        LogBot.fatal('Process', message: e.message, backtrace: e.backtrace.first)
+      end
+      spin_done
+    else
+      LogBot.fatal('Thing', "No Formatter for #{formatter}")
+    end
+
+    self.parsed = true
+
+    save!
+  end
+end

data/lib/greenhat/thing/file_types.rb

@@ -0,0 +1,705 @@
+module GreenHat
+  # Overall Type Parsing
+  # rubocop:disable Metrics/MethodLength, Metrics/ModuleLength
+  module FileTypes
+    def types
+      {
+        'cpuinfo' => {
+          format: :raw,
+          pattern: [
+            /cpuinfo/
+          ]
+        },
+        'df_h' => {
+          format: :table,
+          pattern: [
+            /df_h/
+          ]
+        },
+        'df_inodes' => {
+          format: :table,
+          pattern: [
+            /df_inodes/
+          ]
+        },
+        'dmesg' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            /dmesg/
+          ]
+        },
+        'log/syslog' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            %r{log/syslog}
+          ]
+        },
+        'etc/fstab' => {
+          format: :clean_raw,
+          pattern: [
+            %r{etc/fstab}
+          ]
+        },
+        'etc/os-release' => {
+          format: :dotenv,
+          pattern: [
+            %r{etc/os-release}
+          ]
+        },
+        'sysctl_a' => {
+          format: :dotenv,
+          pattern: [
+            /sysctl_a/
+          ]
+        },
+        'free_m' => {
+          format: :free_m,
+          pattern: [
+            /free_m/
+          ]
+        },
+        'getenforce' => {
+          format: :raw,
+          pattern: [
+            /getenforce/
+          ]
+        },
+        'gitaly/current' => {
+          format: :json_shell,
+          log: true,
+          pattern: [
+            %r{gitaly/current},
+            %r{gitaly/@.*}
+          ]
+        },
+        'consul/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{consul/current}
+          ]
+        },
+        'pgbouncer/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{pgbouncer/current}
+          ]
+        },
+        'mailroom/mail_room_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{mailroom/mail_room_json.log}
+          ]
+        },
+        'gitlab-rails/git_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/git_json.log}
+          ]
+        },
+        'gitaly/gitaly_ruby_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitaly/gitaly_ruby_json.log}
+          ]
+        },
+        'gitlab_status' => {
+          format: :gitlab_status,
+          pattern: [
+            /gitlab_status/
+          ]
+        },
+        'postgresql/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{postgresql/current},
+            %r{postgresql/@.*}
+          ]
+        },
+        'gitlab-ctl/tail' => {
+          format: :gitlab_tail,
+          log: true,
+          pattern: [
+            /gitlab-ctl-tail.log/
+          ]
+        },
+        'patroni/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{patroni/current}
+          ]
+        },
+        'gitlab-exporter/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{gitlab-exporter/current}
+          ]
+        },
+        'gitlab-monitor/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{gitlab-monitor/current}
+          ]
+        },
+        'logrotate/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{logrotate/current}
+          ]
+        },
+        'gitlab-rails/api_json.log' => {
+          format: :api_json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/api_json.log}
+          ]
+        },
+        'gitlab-rails/application_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/application_json.log}
+          ]
+        },
+        'gitlab-rails/audit_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/audit_json.log}
+          ]
+        },
+        'gitlab-rails/auth.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/auth.log}
+          ]
+        },
+        'gitlab-rails/exceptions_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/exceptions_json.log}
+          ]
+        },
+        'gitlab-rails/graphql_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/graphql_json.log}
+          ]
+        },
+        'gitlab-rails/importer.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/importer.log}
+          ]
+        },
+        'gitlab-rails/integrations_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/integrations_json.log}
+          ]
+        },
+        'gitlab-rails/production_json.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/production_json.log}
+          ]
+        },
+        'gitlab-rails/sidekiq_client.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/sidekiq_client.log}
+          ]
+        },
+        'gitlab-shell/gitlab-shell.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-shell/gitlab-shell.log}
+          ]
+        },
+        'gitlab-workhorse/current' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-workhorse/current},
+            %r{gitlab-workhorse/@.*}
+          ]
+        },
+        'gitlab/version-manifest.json' => {
+          format: :multiline_json,
+          pattern: [
+            %r{gitlab/version-manifest.json}
+          ]
+        },
+        'gitlabsos.log' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            /gitlabsos.log/
+          ]
+        },
+        'hostname' => {
+          format: :raw,
+          pattern: [
+            /hostname/
+          ]
+        },
+        'lscpu' => {
+          format: :colon_split_strip,
+          pattern: [
+            /lscpu/
+          ]
+        },
+        'meminfo' => {
+          format: :colon_split_strip,
+          pattern: [
+            /meminfo/
+          ]
+        },
+        'mount' => {
+          format: :raw,
+          pattern: [
+            /mount/
+          ]
+        },
+        'nginx/gitlab_pages_access.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_pages_access.log}
+          ]
+        },
+        'nginx/gitlab_access.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_access.log}
+          ]
+        },
+        'gitlab-rails/sidekiq_exporter.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/sidekiq_exporter.log}
+          ]
+        },
+        'crond/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{crond/current}
+          ]
+        },
+        'grafana/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{grafana/current}
+          ]
+        },
+        'pgbouncer-exporter/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{pgbouncer-exporter/current}
+          ]
+        },
+        'puma/current' => {
+          format: :time_json,
+          log: true,
+          pattern: [
+            %r{puma/current}
+          ]
+        },
+        'reconfigure.log' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            %r{^reconfigure/.*log$}
+          ]
+        },
+        'node-exporter/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{node-exporter/current}
+          ]
+        },
+        'postgres-exporter/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{postgres-exporter/current}
+          ]
+        },
+        'redis-exporter/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{redis-exporter/current},
+            %r{redis-exporter/@.*}
+          ]
+        },
+        'prometheus/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{prometheus/current}
+          ]
+        },
+        'registry/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{registry/current},
+            %r{registry/@.*}
+          ]
+        },
+        'alertmanager/current' => {
+          format: :time_shellwords,
+          log: true,
+          pattern: [
+            %r{alertmanager/current}
+          ]
+        },
+        'ps' => {
+          format: :table,
+          pattern: [
+            /^ps/
+          ]
+        },
+        'puma/puma_stderr.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{puma/puma_stderr.log}
+          ]
+        },
+        'puma/puma_stdout.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{puma/puma_stdout.log}
+          ]
+        },
+        'gitlab-pages/current' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-pages/current}
+          ]
+        },
+
+        # General Purpose Raw
+        'raw' => {
+          format: :raw,
+          pattern: [
+            %r{alertmanager/config},
+            %r{gitaly/config},
+            %r{gitlab-exporter/config},
+            %r{gitlab-pages/config},
+            %r{gitlab-workhorse/config},
+            %r{grafana/config},
+            %r{logrotate/config},
+            %r{nginx/config},
+            %r{node-exporter/config},
+            %r{postgres-exporter/config},
+            %r{postgresql/config},
+            %r{prometheus/config},
+            %r{puma/config},
+            %r{redis/config},
+            %r{redis-exporter/config},
+            %r{registry/config},
+            %r{sidekiq/config}
+          ]
+        },
+        'redis/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{redis/current},
+            %r{redis/@.*}
+          ]
+        },
+
+        'nginx/gitlab_error.log' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_error.log}
+          ]
+        },
+        'nginx/gitlab_registry_error.log' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_registry_error.log}
+          ]
+        },
+        'unicorn/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{unicorn/current}
+          ]
+        },
+        'running_swappiness' => {
+          format: :raw,
+          pattern: [
+            /running_swappiness/
+          ]
+        },
+        'nginx/gitlab_registry_access.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_registry_access.log}
+          ]
+        },
+        'security/limits.conf' => {
+          format: :clean_raw,
+          pattern: [
+            %r{security/limits.conf}
+          ]
+        },
+        'unicorn/unicorn_stderr.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{unicorn/unicorn_stderr.log}
+          ]
+        },
+        'selinux/config' => {
+          format: :dotenv,
+          pattern: [
+            %r{selinux/config}
+          ]
+        },
+        'sidekiq/current' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{sidekiq/current},
+            %r{sidekiq/@.*}
+          ]
+        },
+        'gitlab-rails/service_measurement.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/service_measurement.log}
+          ]
+        },
+        # General Purpose JSON
+        'json' => {
+          format: :json,
+          log: true,
+          pattern: []
+        },
+
+        'gitlab-rails/elasticsearch.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/elasticsearch.log}
+          ]
+        },
+        'sshd/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{sshd/current}
+          ]
+        },
+        'gitlab-rails/sidekiq.log' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/sidekiq.log}
+          ]
+        },
+        'gitlab-rails/gitlab-rails-db-migrate' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/gitlab-rails-db-migrate.*}
+          ]
+        },
+        'tainted' => {
+          format: :raw,
+          pattern: [
+            /tainted/
+          ]
+        },
+        'ulimit' => {
+          format: :raw,
+          pattern: [
+            /ulimit/
+          ]
+        },
+        'uname' => {
+          format: :raw,
+          pattern: [
+            /uname/
+          ]
+        },
+        'uptime' => {
+          format: :raw,
+          pattern: [
+            /uptime/
+          ]
+        },
+        'mpstat' => {
+          format: :raw,
+          pattern: [
+            /mpstat/
+          ]
+        },
+        'isostat' => {
+          format: :raw,
+          pattern: [
+            /isostat/
+          ]
+        },
+        'netstat' => {
+          format: :netstat,
+          pattern: [
+            /^netstat$/
+          ]
+        },
+        'netstat_i' => {
+          format: :netstat_i,
+          pattern: [
+            /^netstat_i$/
+          ]
+        },
+        'iostat' => {
+          format: :raw,
+          pattern: [/^iostat/]
+        },
+        'nfsiostat' => {
+          format: :raw,
+          pattern: [
+            /^nfsiostat/
+          ]
+        },
+        'nfsstat' => {
+          format: :raw,
+          pattern: [
+            /nfsstat/
+          ]
+        },
+        'pidstat' => {
+          format: :pidstat,
+          pattern: [
+            /pidstat/
+          ]
+        },
+        'sar_dev' => {
+          format: :sar_dev,
+          pattern: [
+            /sar_dev/
+          ]
+        },
+        'sar_tcp' => {
+          format: :sar_tcp,
+          pattern: [
+            /sar_tcp/
+          ]
+        },
+        'sestatus' => {
+          format: :colon_split_strip,
+          pattern: [
+            /sestatus/
+          ]
+        },
+        'sockstat' => {
+          format: :raw,
+          pattern: [
+            /sockstat/
+          ]
+        },
+        'systemctl_unit_files' => {
+          format: :table,
+          pattern: [
+            /systemctl_unit_files/
+          ]
+        },
+        'timedatectl' => {
+          format: :colon_split_strip,
+          pattern: [
+            /timedatectl/
+          ]
+        },
+        'log/messages' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{log/messages}
+          ]
+        },
+        'vmstat' => {
+          format: :raw,
+          pattern: [
+            /vmstat/
+          ]
+        },
+        'iotop' => {
+          format: :raw,
+          pattern: [
+            /iotop/
+          ]
+        },
+        'ntpq' => {
+          format: :raw,
+          pattern: [
+            /ntpq/
+          ]
+        },
+        'gitlab-rails/application.log' => {
+          format: :raw,
+          pattern: [
+            %r{gitlab-rails/application.log}
+          ]
+        },
+        'gitlab-rails/production.log' => {
+          format: :raw,
+          pattern: [
+            %r{gitlab-rails/production.log}
+          ]
+        },
+        'gitlab/version-manifest.txt' => {
+          format: :raw,
+          pattern: [
+            %r{gitlab/version-manifest.txt}
+          ]
+        },
+        'ignore' => {
+          format: :ignore,
+          pattern: [
+            %r{sidekiq/perf.data}
+          ]
+        }
+      }
+    end
+  end
+  # rubocop:enable Metrics/MethodLength, Metrics/ModuleLength
+end