greenhat 0.1.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fbb51c051ab366c7bb7a172c4a917d7646bd5bdbc00c894217403655130bb2f3
+  data.tar.gz: b4e997388954091b257df23a1f67093490b8c1996635c860b3e462fa3f44213c
+SHA512:
+  metadata.gz: 965baad78deb383ed1c1cb569a219439daa4c73adf7fd8b3f2852ee1b9703e758e91b0784ac1c1ac999045dd8a0d871927680d2718829ebcc3740dcc4c334a00
+  data.tar.gz: 17111df14082319b62bca88224ccf4b0b433ac9bcda5d1fe9c922e08477632a6c0831f61dc8615f633f0d8c4b70c58d54fe765c63172c4f4573b6812fc61aee1

data/README.md

@@ -0,0 +1,64 @@
+# GreenHat
+
+Experimental SOS and Log Parser for GitLab
+
+12.x and above only (due to structured logging)
+
+# Requirements
+
+`bsdtar` - Used to extract archives
+
+### Ubuntu
+
+`apt-get install libarchive-tools`
+
+## Installation
+
+While potentially so in the future, this is currently not a published gem. To install:
+
+```
+git clone https://gitlab.com/gitlab-com/support/toolbox/greenhat
+cd greenhat
+bundle install
+bundle exec rake install
+```
+
+## Usage
+
+```
+greenhat sos-archive.tar.gz
+greenhat production_log.json
+# launches a console
+>> help # the program is self-documented through the builtin help command
+```
+
+## Testing
+
+```
+bundle exec rake
+```
+
+## Release Process
+
+- Update CHANGELOG
+- Increment `version.rb`
+- Create MR
+- Merge to Master
+
+```
+# Build and Push
+bundle exec rake build
+gem push pkg/greenhat-1.x.x.gem
+```
+
+## Contributing
+
+Bug reports and merge requests are welcome on GitLab at https://gitlab.com/gitlab-com/support/toolbox/greenhat. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## Limitations
+
+- Shell Helpers cannot use Camel Case
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/bin/greenhat

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require 'greenhat'
+
+# DEV ONLY
+require 'pry'
+
+Dir.mktmpdir('greenhat-sauce') do |tmp|
+  $TMP = tmp
+  $STDOUT_CLONE = $stdout.clone
+  GreenHat::Cli.start ARGV
+end

data/lib/greenhat.rb

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+require 'action_view'
+require 'benchmark'
+require 'find'
+require 'tty-prompt'
+require 'tty-spinner'
+require 'tty-table'
+require 'tty-pager'
+require 'tty-cursor'
+require 'tty-reader'
+require 'hash_dot'
+require 'oj'
+require 'slim'
+require 'active_support'
+require 'active_support/core_ext'
+require 'pry'
+require 'amazing_print'
+require 'colorize'
+require 'semantic'
+require 'tty-progressbar'
+require 'require_all'
+require 'warning'
+require 'dotenv'
+
+# Custom Gem
+require 'teron'
+
+# Oj Settings
+Oj.default_options = { symbol_keys: true, mode: :compat }
+
+# Amazing Print
+AmazingPrint.defaults = {
+  indent: -2,
+  ruby19_syntax: true,
+  index: false,
+  sort_keys: true,
+  sort_vars: true
+}
+
+# HashDot Instead of Struct
+Hash.use_dot_syntax = true
+Hash.hash_dot_use_default = true
+
+# Load Required Files
+require 'greenhat/version'
+require 'greenhat/cli'
+require 'greenhat/archive'
+require 'greenhat/host'
+require 'greenhat/logbot'
+require 'greenhat/settings'
+
+# Formatters - Loads Required Files Automatically
+require 'greenhat/thing/super_log'
+require 'greenhat/thing/file_types'
+require 'greenhat/thing/kind'
+require 'greenhat/thing/history'
+
+# Thing
+require 'greenhat/thing/helpers'
+require 'greenhat/thing/spinner'
+require 'greenhat/thing'
+
+# Shell - Loads Required Files Automatically
+require 'greenhat/shell'
+
+# TODO: Confirm
+# require 'greenhat/thing/log_format'
+# require 'greenhat/host'
+# require 'greenhat/web'
+
+require 'greenhat/pry_helpers'
+
+## TTY Shims
+
+require 'greenhat/tty/line'
+require 'greenhat/tty/reader'
+require 'greenhat/tty/custom_line'
+
+Warning.ignore(/The table size exceeds the currently set width/)

data/lib/greenhat/accessors/disk.rb

@@ -0,0 +1,27 @@
+module GreenHat
+  # Sidekiq Log Helpers
+  module Disk
+    def self.data
+      df.map(&:data).flatten.compact
+    end
+
+    # Get Max Size
+    def self.max_padding(disks, key = :mounted_on)
+      list = disks.map do |x|
+        x.slice(key)
+      end
+
+      list.map(&:values).flatten.max_by(&:size).size + 4
+    end
+
+    def self.padding(disks, keys = %i[mounted_on size used avail])
+      keys.map do |key|
+        max_padding(disks, key)
+      end
+    end
+
+    def self.df
+      Thing.where(name: 'df_h')
+    end
+  end
+end

data/lib/greenhat/accessors/logs/production.rb

@@ -0,0 +1,41 @@
+module GreenHat
+  # Sidekiq Log Helpers
+  module Production
+    def self.fast_stats
+      things.each do |thing|
+        puts `fast-stats #{thing.file}`
+      end
+
+      :ok!
+    end
+
+    def self.logs
+      @logs ||= things.map(&:data).flatten.compact
+    end
+
+    def self.raw
+      @raw ||= things.map(&:raw).flatten.compact
+    end
+
+    def self.things
+      Thing.where(name: 'gitlab_rails_production_json_log')
+    end
+
+    def self.errors
+      logs.select { |x| x.severity == 'ERROR' }
+    end
+
+    def self.pages
+      show logs
+    end
+
+    def self.queue_duration(data = nil)
+      data ||= logs
+      data.select { |x| x.key? :enqueued_at }.each do |row|
+        next if row.key? :queue_duration
+
+        row[:queue_duration] = row.enqueued_at - row.created_at
+      end
+    end
+  end
+end

data/lib/greenhat/accessors/logs/sidekiq.rb

@@ -0,0 +1,41 @@
+module GreenHat
+  # Sidekiq Log Helpers
+  module Sidekiq
+    def self.fast_stats
+      things.each do |thing|
+        puts `fast-stats #{thing.file}`
+      end
+
+      :ok!
+    end
+
+    def self.logs
+      @logs ||= things.map(&:data).flatten.compact
+    end
+
+    def self.raw
+      @raw ||= things.map(&:raw).flatten.compact
+    end
+
+    def self.things
+      Thing.where(name: 'sidekiq_current')
+    end
+
+    def self.errors
+      logs.select { |x| x.severity == 'ERROR' }
+    end
+
+    def self.pages
+      show logs
+    end
+
+    def self.queue_duration(data = nil)
+      data ||= logs
+      data.select { |x| x.key? :enqueued_at }.each do |row|
+        next if row.key? :queue_duration
+
+        row[:queue_duration] = row.enqueued_at - row.created_at
+      end
+    end
+  end
+end

data/lib/greenhat/accessors/memory.rb

@@ -0,0 +1,46 @@
+module GreenHat
+  # Sidekiq Log Helpers
+  module Memory
+    def self.free
+      Thing.where(name: 'free_m')
+    end
+
+    def self.percentage(used, total)
+      return 0 if used.to_i.zero?
+
+      # Show at least one bar if below 1%
+      [1, ((used.to_i / total.to_f).round(1) * 100).round].max
+    end
+
+    def self.memory_row(mem)
+      total = mem.total.to_i
+      total_name = number_to_human_size(total.megabytes)
+      used_name = number_to_human_size(mem.used.to_i.megabytes)
+
+      bar = [
+        '|'.colorize(:green) * percentage(mem.used, total),
+        '|'.colorize(:blue) * percentage(mem.shared, total),
+        '|'.colorize(:teal) * percentage(mem.buffcache, total),
+        ' ' * percentage(mem.free, total)
+      ].join
+
+      # Make Even
+      padding = 125 - bar.uncolorize.size
+      bar += ' ' * padding if padding.positive?
+
+      [
+        mem.kind.ljust(4).colorize(:cyan),
+        ' ['.colorize(:light_black),
+        bar.ljust(120),
+        used_name.colorize(:magenta),
+        ' / '.colorize(:light_black),
+        total_name.colorize(:blue),
+        ']'.colorize(:light_black)
+      ].join
+    end
+
+    def self.number_to_human_size(num)
+      ActiveSupport::NumberHelper.number_to_human_size num
+    end
+  end
+end

data/lib/greenhat/accessors/network.rb

@@ -0,0 +1,8 @@
+module GreenHat
+  # Sidekiq Log Helpers
+  module Network
+    def self.netstat
+      Thing.where(name: 'netstat')
+    end
+  end
+end

data/lib/greenhat/accessors/process.rb

@@ -0,0 +1,8 @@
+module GreenHat
+  # Sidekiq Log Helpers
+  module Ps
+    def self.ps
+      Thing.where(name: 'ps')
+    end
+  end
+end

data/lib/greenhat/archive.rb

@@ -0,0 +1,108 @@
+# Top level namespace
+module GreenHat
+  # Archive Manipulator
+  module ArchiveLoader
+    def self.load(archive_path)
+      path = "#{$TMP}/#{SecureRandom.uuid}"
+      Dir.mkdir path
+
+      # Initially Copy file into tmpdir
+      FileUtils.cp(archive_path, "#{path}/")
+
+      # Extract Everything
+      loop do
+        archive_list = Find.find(path).reject { |x| File.directory? x }.select { |y| archive? y }
+        break if archive_list.empty?
+
+        archive_list.each do |archive|
+          # Different File Type Handler
+          unpack(archive, path)
+        end
+      end
+
+      # Ignore Directories
+      list = Find.find(path).reject { |x| File.directory? x }
+
+      # Ignore Empty Files
+      list.reject! { |x| File.size(x).zero? }
+
+      archive = Archive.new(name: archive_path, path: path)
+      archive.save
+      # file = list[2]
+      # thing = archive.things_create(file: file)
+      # thing.setup
+
+      list.each do |file|
+        # Ensure Valid Content
+        next if missing_command(file)
+
+        # Thread.new do
+        thing = archive.things_create(file: file)
+        thing.setup
+        # end
+      end
+    end
+
+    # Ignore No Commands
+    # rubocop:disable Style/SymbolProc
+    def self.missing_command(file)
+      first_line = File.open(file) { |f| f.readline }
+      ['command not found', ': not found'].any? { |x| first_line.include? x }
+    end
+    # rubocop:enable Style/SymbolProc
+
+    # Handle Different Types of Archives
+    def self.unpack(archive_path, path)
+      case File.extname archive_path
+      when '.tar'
+        `bsdtar -xf "#{archive_path}" -C #{path}`
+        FileUtils.rm(archive_path)
+      when '.gz'
+        `gzip -d "#{archive_path}"`
+      when '.s'
+        # Find Original Directory, Split Path, Rename to .gz
+        base_path = archive_path.gsub(File.basename(archive_path), '')
+        FileUtils.mv(archive_path, "#{base_path}/#{File.basename(archive_path, '.s')}.gz")
+      else
+        FileUtils.cp(archive_path, "#{path}/#{archive_path}")
+      end
+    end
+
+    def self.archive?(file_name)
+      archive_types.any? do |type|
+        file_name.match?(/(\.#{type})$/)
+      end
+    end
+
+    def self.archive_types
+      %w[s gz tar]
+    end
+    # -- Archive Loader ----------------------------------------------
+  end
+  # Archive End
+end
+
+# Archive Parent
+class Archive < Teron
+  has_one :host
+  has_many :things
+
+  field :path
+  field :name
+
+  # TODO: Fix from Number of Files / Needed?
+  def friendly_name
+    # Difficult with multiple Archives
+    # File.basename(name, '.tar.gz').gsub('gitlabsos.', '').split('_', 2).first
+
+    File.basename(name, File.extname(name)).gsub('gitlabsos.', '')
+  end
+
+  def inspect
+    "#<Archive name: '#{name}'>"
+  end
+
+  def report
+    GreenHat::Report.new(self)
+  end
+end