gitrob 0.0.6 → 1.0.0

data/lib/gitrob/cli/commands/banner.rb

@@ -0,0 +1,25 @@
+module Gitrob
+  class CLI
+    module Commands
+      class Banner < Gitrob::CLI::Command
+        def initialize(options)
+          @options = options
+          output banner if options[:banner]
+          info "Starting Gitrob version #{Gitrob::VERSION} " \
+          "at #{Time.now.strftime('%Y-%m-%d %H:%M %Z')}"
+          debug "Debugging mode enabled"
+        end
+
+        private
+
+        def banner
+          "     _ _           _\n" \
+          " ___|_| |_ ___ ___| |_\n" \
+          "| . | |  _|  _| . | . |\n" \
+          "|_  |_|_| |_| |___|___|\n" \
+          "|___|".light_blue + " By @michenriksen\n\n".light_white
+        end
+      end
+    end
+  end
+end

data/lib/gitrob/cli/commands/configure.rb

@@ -0,0 +1,123 @@
+module Gitrob
+  class CLI
+    module Commands
+      class Configure < Gitrob::CLI::Command
+        CONFIGURATION_FILE_PATH = File.join(Dir.home, ".gitrobrc")
+
+        class ConfigurationError < StandardError; end
+        class ConfigurationFileNotFound < ConfigurationError; end
+        class ConfigurationFileNotReadable < ConfigurationError; end
+        class ConfigurationFileCorrupt < ConfigurationError; end
+
+        def initialize(options)
+          @options = options
+          info("Starting Gitrob configuration wizard")
+          return unless agree_to_overwrite?
+          config = gather_configuration
+          task("Saving configuration to #{CONFIGURATION_FILE_PATH}") do
+            save_configuration(config)
+          end
+        end
+
+        def self.configured?
+          File.exist?(CONFIGURATION_FILE_PATH)
+        end
+
+        def self.load_configuration!
+          fail ConfigurationFileNotFound \
+            unless File.exist?(CONFIGURATION_FILE_PATH)
+          fail ConfigurationFileNotReadable \
+            unless File.readable?(CONFIGURATION_FILE_PATH)
+          YAML.load(File.read(CONFIGURATION_FILE_PATH))
+        rescue Psych::SyntaxError
+          raise ConfigurationFileCorrupt
+        end
+
+        private
+
+        def agree_to_overwrite?
+          return true unless self.class.configured?
+          warn("Configuration file already exists\n")
+          agree(
+            "Proceed and overwrite existing configuration file? (y/n): ")
+        end
+
+        def gather_configuration
+          {
+            :hostname      => gather_hostname,
+            :port          => gather_port,
+            :username      => gather_username,
+            :password      => gather_password,
+            :database      => gather_database,
+            :access_tokens => gather_access_tokens
+          }
+        end
+
+        def gather_hostname
+          ask("Enter PostgreSQL hostname: ") do |q|
+            q.default = "localhost"
+          end
+        end
+
+        def gather_port
+          ask("Enter PostgreSQL port: |5432| ", Integer) do |q|
+            q.default = 5432
+            q.in = 1..65_535
+          end
+        end
+
+        def gather_username
+          ask("Enter PostgreSQL username: ") do |q|
+            q.default = "gitrob"
+          end
+        end
+
+        def gather_password
+          ask("Enter PostgreSQL password (masked): ") do |q|
+            q.echo = "x"
+          end
+        end
+
+        def gather_database
+          ask("Enter PostgreSQL database name: ") do |q|
+            q.default = "gitrob"
+          end
+        end
+
+        def gather_access_tokens
+          tokens = []
+          while tokens.uniq.empty?
+            tokens = ask("Enter GitHub access tokens (blank line to stop):",
+                         ->(ans) { ans =~ /[a-f0-9]{40}/ ? ans : nil }) do |q|
+                           q.gather = ""
+                         end
+          end
+          tokens
+        end
+
+        def save_configuration(config)
+          File.open(CONFIGURATION_FILE_PATH, "w") do |file|
+            file.write(build_yaml(config))
+          end
+        end
+
+        def make_connection_uri(username, password, hostname, port, database)
+          "postgres://#{username}:#{password}@#{hostname}:#{port}/#{database}"
+        end
+
+        def build_yaml(config)
+          YAML.dump(
+            "sql_connection_uri" => make_connection_uri(
+              config[:username],
+              config[:password],
+              config[:hostname],
+              config[:port],
+              config[:database]
+            ),
+            "github_access_tokens" => config[:access_tokens]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/gitrob/cli/commands/server.rb

@@ -0,0 +1,21 @@
+module Gitrob
+  class CLI
+    module Commands
+      class Server < Gitrob::CLI::Command
+        def initialize(options)
+          @options = options
+          info "Starting web application on port #{options[:port]}..."
+
+          if debugging_enabled?
+            Sequel::Model.db.logger = QueryLogger.new(STDOUT)
+          end
+
+          Gitrob::WebApp.run!(
+            :port => options[:port].to_i,
+            :bind => options[:bind_address]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/gitrob/cli/progress_bar.rb

@@ -0,0 +1,47 @@
+module Gitrob
+  class CLI
+    class ProgressBar
+      def initialize(message, options={})
+        @options = {
+          :format =>
+            "#{'[*]'.light_blue} %t %c/%C %B %j% %e",
+          :progress_mark => "|".light_blue,
+          :remainder_mark => "|"
+        }.merge(options)
+        @mutex = Mutex.new
+        Gitrob::CLI.info(message)
+        @progress_bar = ::ProgressBar.create(@options)
+      end
+
+      def finish
+        progress_bar.finish
+      end
+
+      def info(message)
+        progress_bar.log("#{'[+]'.light_blue} #{message}")
+      end
+
+      def error(message)
+        progress_bar.log("#{'[!]'.light_red} #{message}")
+      end
+
+      def warn(message)
+        progress_bar.log("#{'[!]'.light_yellow} #{message}")
+      end
+
+      def method_missing(method, *args, &block)
+        if progress_bar.respond_to?(method)
+          progress_bar.send(method, *args, &block)
+        else
+          super
+        end
+      end
+
+      private
+
+      def progress_bar
+        @mutex.synchronize { @progress_bar }
+      end
+    end
+  end
+end

data/lib/gitrob/cli.rb

@@ -0,0 +1,213 @@
+module Gitrob
+  class QueryLogger < Logger
+    def format_message(_severity, _timestamp, _progname, msg)
+      "#{msg}\n".cyan
+    end
+  end
+
+  class CLI < Thor
+    HELP_COMMANDS = %w(help h --help -h)
+    package_name "Gitrob"
+
+    attr_reader :configuration
+
+    DB_MIGRATIONS_PATH = File.expand_path(
+      "../../../db/migrations", __FILE__
+    )
+
+    class_option :bind_address,
+                 :type    => :string,
+                 :banner  => "ADDRESS",
+                 :default => "127.0.0.1",
+                 :desc    => "Address to bind web server to"
+    class_option :port,
+                 :type    => :numeric,
+                 :default => 9393,
+                 :desc    => "Port to run web server on"
+    class_option :access_tokens,
+                 :type    => :array,
+                 :banner  => "TOKENS",
+                 :desc    => "GitHub API tokens to use " \
+                             "instead of what has been configured"
+    class_option :color,
+                 :type    => :boolean,
+                 :default => true,
+                 :desc    => "Colorize or don't colorize output"
+    class_option :banner,
+                 :type    => :boolean,
+                 :default => true,
+                 :desc    => "Show or don't show Gitrob banner"
+    class_option :debug,
+                 :type    => :boolean,
+                 :default => false,
+                 :desc    => "Show or don't show debugging information"
+
+    def initialize(*args)
+      super
+      self.class.enable_debugging if options[:debug]
+      String.disable_colorization(!options[:color])
+      return if help_command?
+      banner
+      configure unless configured?
+      load_configuration
+      prepare_database
+    end
+
+    desc "analyze TARGETS", "Analyze one or more organizations or users"
+    option :title,
+           :type => :string,
+           :desc => "Give assessment a custom title"
+    option :threads,
+           :type    => :numeric,
+           :default => 5,
+           :desc    => "Number of threads to use"
+    option :server,
+           :type    => :boolean,
+           :default => true,
+           :desc    => "Start or don't start web server after assessment"
+    option :endpoint,
+           :type    => :string,
+           :banner  => "URL",
+           :default => "https://api.github.com",
+           :desc    => "Specify a URL for a custom GitHub Enterprise API"
+    option :site,
+           :type    => :string,
+           :banner  => "URL",
+           :default => "https://github.com",
+           :desc    => "Specify a URL for a custom GitHub Enterprise site"
+    option :verify_ssl,
+           :type    => :boolean,
+           :default => true,
+           :desc    => "Verify or don't verify SSL connection (careful here)"
+    def analyze(targets)
+      accept_tos
+      Gitrob::CLI::Commands::Analyze.start(targets, options)
+    end
+
+    desc "server", "Start web server"
+    def server
+      accept_tos
+      Gitrob::CLI::Commands::Server.start(options)
+    end
+
+    desc "configure", "Start configuration wizard"
+    def configure
+      Gitrob::CLI::Commands::Configure.start(options)
+    end
+
+    desc "banner", "Print Gitrob banner", :hide => true
+    def banner
+      Gitrob::CLI::Commands::Banner.start(options)
+    end
+
+    desc "accept-tos", "Accept Terms Of Use", :hide => true
+    def accept_tos
+      Gitrob::CLI::Commands::AcceptTermsOfUse.start(options)
+    end
+
+    no_commands do
+      def help_command?
+        !ENV["GITROB_ENV"] == "TEST" &&
+          (ARGV.empty? || HELP_COMMANDS.include?(ARGV.first.downcase))
+      end
+
+      def configured?
+        Gitrob::CLI::Commands::Configure.configured?
+      end
+
+      def load_configuration
+        self.class.task("Loading configuration...", true) do
+          @configuration = Gitrob::CLI::Commands::Configure.load_configuration!
+          self.class.configuration = @configuration
+        end
+      end
+
+      def prepare_database
+        self.class.task("Preparing database...", true) do
+          Sequel.extension :migration, :core_extensions
+          db = Sequel.connect(configuration["sql_connection_uri"])
+          Sequel::Migrator.run(db, DB_MIGRATIONS_PATH)
+          Sequel::Model.raise_on_save_failure = true
+          Sequel::Model.db = db
+          Sequel::Model.plugin :validation_helpers
+          Sequel::Model.plugin :timestamps
+          load_models
+        end
+      end
+
+      def load_models
+        require "gitrob/models/assessment"
+        require "gitrob/models/github_access_token"
+        require "gitrob/models/owner"
+        require "gitrob/models/repository"
+        require "gitrob/models/blob"
+        require "gitrob/models/flag"
+        require "gitrob/models/comparison"
+      end
+    end
+
+    def self.info(message)
+      output "[*]".light_blue + " #{message}\n"
+    end
+
+    def self.task(message, fatal_error=false, &block)
+      output "[*]".light_blue + " #{message}"
+      yield block
+      output " done\n".light_green
+    rescue => e
+      output " failed\n".light_red
+      output_failed_task(e, fatal_error)
+    end
+
+    def self.output_failed_task(exception, fatal_error)
+      message = "#{exception.class}: #{exception.message}"
+      debug exception.backtrace.join("\n")
+      if fatal_error
+        fatal message
+      else
+        error message
+      end
+    end
+
+    def self.warn(message)
+      output "[!]".light_yellow + " #{message}\n"
+    end
+
+    def self.error(message)
+      output "[!]".light_red + " #{message}\n"
+    end
+
+    def self.fatal(message)
+      output "[!]".light_white.on_red + " #{message}\n"
+      exit(1)
+    end
+
+    def self.debug(message)
+      output "[#]".light_cyan + " #{message}\n" if debugging_enabled?
+    end
+
+    def self.output(string)
+      print string
+    end
+
+    def self.enable_debugging
+      @debugging_enabled = true
+    end
+
+    def self.disable_debugging
+      @debugging_enabled = false
+    end
+
+    def self.debugging_enabled? # rubocop:disable Style/TrivialAccessors
+      @debugging_enabled
+    end
+
+    def self.configuration=(config)
+      @config = config
+    end
+
+    def self.configuration
+      @config
+    end
+  end
+end

data/lib/gitrob/github/client_manager.rb

@@ -0,0 +1,46 @@
+module Gitrob
+  module Github
+    class ClientManager
+      USER_AGENT = "Gitrob v#{Gitrob::VERSION}"
+
+      attr_reader :clients
+
+      class NoClientsError < StandardError; end
+
+      def initialize(config)
+        @config  = config
+        @mutex   = Mutex.new
+        @clients = []
+        config[:access_tokens].each do |token|
+          clients << create_client(token)
+        end
+      end
+
+      def sample
+        @mutex.synchronize do
+          fail NoClientsError if clients.count.zero?
+          clients.sample
+        end
+      end
+
+      def remove(client)
+        @mutex.synchronize do
+          clients.delete(client)
+        end
+      end
+
+      private
+
+      def create_client(access_token)
+        ::Github.new(
+          :oauth_token     => access_token,
+          :endpoint        => @config[:endpoint],
+          :site            => @config[:site],
+          :ssl             => @config[:verify_ssl],
+          :user_agent      => USER_AGENT,
+          :auto_pagination => true
+        )
+      end
+    end
+  end
+end

data/lib/gitrob/github/data_manager.rb

@@ -0,0 +1,121 @@
+module Gitrob
+  module Github
+    class DataManager
+      attr_reader :client_manager,
+                  :unknown_logins,
+                  :owners,
+                  :repositories
+
+      def initialize(logins, client_manager)
+        @logins                  = logins
+        @client_manager          = client_manager
+        @unknown_logins          = []
+        @owners                  = []
+        @repositories            = []
+        @repositories_for_owners = {}
+        @mutex                   = Mutex.new
+      end
+
+      def gather_owners(thread_pool)
+        @logins.each do |login|
+          next unless owner = get_owner(login)
+          @owners << owner
+          @repositories_for_owners[owner["login"]] = []
+          next unless owner["type"] == "Organization"
+          get_members(owner, thread_pool) if owner["type"] == "Organization"
+        end
+        @owners = @owners.uniq { |o| o["login"] }
+      end
+
+      def gather_repositories(thread_pool)
+        owners.each do |owner|
+          thread_pool.process do
+            repositories = get_repositories(owner)
+            with_mutex do
+              save_repositories(owner, repositories)
+            end
+            yield owner, repositories if block_given?
+          end
+        end
+      end
+
+      def repositories_for_owner(owner)
+        @repositories_for_owners[owner["login"]]
+      end
+
+      def blobs_for_repository(repository)
+        get_blobs(repository)
+      rescue ::Github::Error::Forbidden => e
+        # Hidden GitHub feature?
+        raise e unless e.message.include?("403 Repository access blocked")
+        []
+      rescue ::Github::Error::NotFound
+        []
+      rescue ::Github::Error::ServiceError => e
+        raise e unless e.message.include?("409 Git Repository is empty")
+        []
+      end
+
+      private
+
+      def get_owner(login)
+        github_client do |client|
+          client.users.get(:user => login)
+        end
+      rescue ::Github::Error::NotFound
+        @unknown_logins << login
+        nil
+      end
+
+      def get_members(org, thread_pool)
+        github_client do |client|
+          client.orgs.members.list(:org_name => org["login"]) do |owner|
+            thread_pool.process do
+              owner = get_owner(owner["login"])
+              with_mutex do
+                @owners << owner
+                @repositories_for_owners[owner["login"]] = []
+              end
+            end
+          end
+        end
+      end
+
+      def get_repositories(owner)
+        github_client do |client|
+          client.repos.list(
+            :user => owner["login"]).reject { |r| r["fork"] }
+        end
+      end
+
+      def get_blobs(repository)
+        github_client do |client|
+          client.get_request(
+            "repos/#{repository[:full_name]}/git/trees/" \
+            "#{repository[:default_branch]}",
+            ::Github::ParamsHash.new(:recursive => 1))["tree"]
+            .reject { |b| b["type"] != "blob" }
+        end
+      end
+
+      def github_client
+        client = @client_manager.sample
+        yield client
+      rescue ::Github::Error::Forbidden => e
+        raise e unless e.message.include?("API rate limit exceeded")
+        @client_manager.remove(client)
+      rescue ::Github::Error::Unauthorized
+        @client_manager.remove(client)
+      end
+
+      def save_repositories(owner, repositories)
+        @repositories += repositories
+        @repositories_for_owners[owner["login"]] = repositories
+      end
+
+      def with_mutex
+        @mutex.synchronize { yield }
+      end
+    end
+  end
+end

data/lib/gitrob/jobs/assessment.rb

@@ -0,0 +1,12 @@
+module Gitrob
+  module Jobs
+    class Assessment
+      include SuckerPunch::Job
+
+      def perform(targets, options)
+        options[:server] = false
+        Gitrob::CLI::Commands::Analyze.new(targets, options)
+      end
+    end
+  end
+end

data/lib/gitrob/jobs/comparison.rb

@@ -0,0 +1,55 @@
+module Gitrob
+  module Jobs
+    class Comparison
+      include SuckerPunch::Job
+
+      def perform(primary_assessment, secondary_assessment)
+        @comparison = Gitrob::Models::Comparison.new
+        @primary_assessment = primary_assessment
+        @secondary_assessment = secondary_assessment
+        @comparison.primary_assessment = primary_assessment
+        @comparison.secondary_assessment = secondary_assessment
+        @comparison.save
+        compare_blobs
+        compare_repositories
+        compare_owners
+        @comparison.finished = true
+        @comparison.save
+      end
+
+      private
+
+      def compare_blobs
+        old_blob_shas = @secondary_assessment.blobs_dataset.select_map(:sha)
+        @primary_assessment.blobs_dataset.eager(:flags).all.each do |blob|
+          next if old_blob_shas.include?(blob.sha)
+          @comparison.add_blob(blob)
+          @comparison.blobs_count += 1
+          @comparison.findings_count += 1 unless blob.flags_count.zero?
+        end
+      end
+
+      def compare_repositories
+        old_repository_github_ids = @secondary_assessment
+                                     .repositories_dataset
+                                     .select_map(:github_id)
+        @primary_assessment.repositories.each do |repository|
+          next if old_repository_github_ids.include?(repository.github_id)
+          @comparison.add_repository(repository)
+          @comparison.repositories_count += 1
+        end
+      end
+
+      def compare_owners
+        old_owner_github_ids = @secondary_assessment
+                                     .owners_dataset
+                                     .select_map(:github_id)
+        @primary_assessment.owners.each do |owner|
+          next if old_owner_github_ids.include?(owner.github_id)
+          @comparison.add_owner(owner)
+          @comparison.owners_count += 1
+        end
+      end
+    end
+  end
+end