gitrob 0.0.6 → 1.0.0

data/bin/gitrob

@@ -1,260 +0,0 @@
-#!/usr/bin/env ruby
-# coding: utf-8
-
-require 'gitrob'
-
-class App
-  include Methadone::Main
-
-  leak_exceptions true
-
-  main do
-    Thread.abort_on_exception = true
-    Paint.mode = 0 if options.include?('no-color')
-
-    puts Paint[Gitrob::banner, :bright, :blue] unless options.include?('no-banner')
-    Gitrob::status("Starting Gitrob version #{Gitrob::VERSION} at #{Time.now.strftime("%Y-%m-%d %I:%S %Z")}")
-
-    if !Gitrob::agreement_accepted?
-      puts Gitrob::agreement
-      print Paint["\n\nDo you agree to the terms of use? [y/n]: ", :bright, :green]
-      choice = $stdin.gets.chomp
-      if %w{y yes}.include?(choice)
-        Gitrob::agreement_accepted
-      else
-        Gitrob::fatal("Exiting Gitrob.")
-      end
-    end
-
-    if !Gitrob::configured? || options.include?('configure')
-      Gitrob::status("Starting Gitrob configuration wizard\n\n")
-
-      hostname = ask("#{Paint[" Enter PostgreSQL hostname:", :bright, :white]} ") { |q| q.default = "localhost" }
-      port     = ask("#{Paint[" Enter PostgreSQL port:", :bright, :white]} ", Integer) { |q| q.default = 5432; q.in = 1..65535 }
-      username = ask("#{Paint[" Enter PostgreSQL username:", :bright, :white]} ")
-      password = ask("#{Paint[" Enter PostgreSQL password for #{username} (masked):", :bright, :white]} ") { |q| q.echo = 'x' }
-      database = ask("#{Paint[" Enter PostgreSQL database name:", :bright, :white]} ") { |q| q.default = 'gitrob' }
-
-      access_tokens = Array.new
-      while access_tokens.uniq.empty?
-        access_tokens = ask(Paint[" Enter GitHub access tokens (blank line to stop):", :bright, :white],
-                          lambda { |ans| ans =~ /[a-f0-9]{40}/ ? ans : nil } ) do |q|
-          q.gather = ""
-        end
-      end
-
-      config = {
-        'sql_connection_uri'   => "postgres://#{username}:#{password}@#{hostname}:#{port}/#{database}",
-        'github_access_tokens' => access_tokens.uniq
-      }
-
-      Gitrob::task("Saving configuration to file...") do
-        Gitrob::save_configuration!(config)
-      end
-
-      exit unless options['organization']
-    end
-
-    Gitrob::task("Loading configuration...") do
-      Gitrob::load_configuration!
-    end
-
-    Gitrob::task("Preparing SQL database...") do
-      Gitrob::prepare_database!
-    end
-
-    if options['reset-db']
-      Gitrob::task("Resetting database tables...") do
-        DataMapper::auto_migrate!
-      end
-      exit
-    end
-
-    if options['delete']
-      Gitrob::delete_organization(options['delete'])
-      exit
-    end
-
-    if options['start-server']
-      Gitrob::status("Starting web application on http://#{options['bind-address']}:#{options['port']}/...")
-      puts "\n\n"
-      Gitrob::WebApp.run!(:port => options['port'].to_i, :bind => options['bind-address'])
-      exit
-    end
-
-    org_name    = options['organization']
-    repo_count  = 0
-    members     = Array.new
-    http_client = Gitrob::Github::HttpClient.new({:access_tokens => Gitrob::configuration['github_access_tokens']})
-    observers   = Gitrob::Observers.constants.collect { |c| Gitrob::Observers::const_get(c) }
-
-    Gitrob::task("Loading file patterns...") do
-      Gitrob::Observers::SensitiveFiles::load_patterns!
-    end
-
-    begin
-      org = Gitrob::Github::Organization.new(org_name, http_client)
-
-      Gitrob::task("Collecting organization repositories...") do
-        repo_count = org.repositories.count
-      end
-    rescue Gitrob::Github::HttpClient::ClientError => e
-      if e.status == 404
-        Gitrob::fatal("Cannot find GitHub organization with that name; exiting.")
-      else
-        raise e
-      end
-    end
-
-    Gitrob::task("Collecting organization members...") do
-      members = org.members
-    end
-
-    progress = Gitrob::ProgressBar.new("Collecting member repositories...",
-      :total => members.count
-    )
-    thread_pool = Thread.pool(options['threads'].to_i)
-
-    members.each do |member|
-      thread_pool.process do
-        if member.repositories.count > 0
-          repo_count += member.repositories.count
-          progress.log("Collected #{Gitrob::Util::pluralize(member.repositories.count, 'repository', 'repositories')} from #{Paint[member.username, :bright, :white]}")
-        end
-        progress.increment
-      end
-    end
-
-    thread_pool.shutdown
-
-    if repo_count.zero?
-      Gitrob::fatal("Organization has no repositories to check; exiting.")
-    end
-
-    progress = Gitrob::ProgressBar.new("Processing repositories...",
-      :total => repo_count
-    )
-
-    db_org = org.save_to_database!
-    thread_pool = Thread.pool(options['threads'].to_i)
-
-    org.repositories.each do |repo|
-      thread_pool.process do
-        begin
-          if repo.contents.count > 0
-            db_repo  = repo.save_to_database!(db_org)
-            findings = 0
-
-            repo.contents.each do |blob|
-              db_blob = blob.to_model(db_org, db_repo)
-
-              observers.each do |observer|
-                observer.observe(db_blob)
-              end
-
-              db_blob.findings.each do |f|
-                db_blob.findings_count += 1
-                findings += 1
-                f.organization = db_org
-                f.repo         = db_repo
-              end
-
-              db_blob.save
-            end
-            progress.log("Processed #{Gitrob::Util::pluralize(repo.contents.count, 'file', 'files')} from #{Paint[repo.full_name, :bright, :white]} with #{findings.zero? ? 'no findings' : Paint[Gitrob::Util.pluralize(findings, 'finding', 'findings'), :yellow]}")
-          end
-          progress.increment
-        rescue Exception => e
-          progress.log_error("Encountered error when processing #{Paint[repo.full_name, :bright, :white]} (#{e.class.name})")
-          progress.increment
-        end
-      end
-    end
-
-    org.members.each do |member|
-      thread_pool.process do
-        begin
-          db_user = member.save_to_database!(db_org)
-
-          member.repositories.each do |repo|
-            if repo.contents.count > 0
-              db_repo  = repo.save_to_database!(db_org, db_user)
-              findings = 0
-
-              repo.contents.each do |blob|
-                db_blob = blob.to_model(db_org, db_repo)
-
-                observers.each do |observer|
-                  observer.observe(db_blob)
-                end
-
-                db_blob.findings.each do |f|
-                  db_blob.findings_count += 1
-                  findings += 1
-                  f.organization = db_org
-                  f.repo         = db_repo
-                  f.user         = db_user
-                end
-
-                db_blob.save
-              end
-              progress.log("Processed #{Gitrob::Util::pluralize(repo.contents.count, 'file', 'files')} from #{Paint[repo.full_name, :bright, :white]} with #{findings.zero? ? 'no findings' : Paint[Gitrob::Util.pluralize(findings, 'finding', 'findings'), :yellow]}")
-            end
-            progress.increment
-          end
-        rescue Exception => e
-          progress.log_error("Encountered error when processing #{Paint[repo.full_name, :bright, :white]} (#{e.class.name})")
-          progress.increment
-        end
-      end
-    end
-
-    thread_pool.shutdown
-
-    if !options.include?('no-server')
-      Gitrob::status("Starting web application on port #{options['port']}...")
-      Gitrob::status("Browse to http://#{options['bind-address']}:#{options['port']}/ to see results!")
-      puts "\n\n"
-      Gitrob::WebApp.run!(:port => options[:port].to_i, :bind => options['bind-address'])
-      exit
-    end
-  end
-
-  version Gitrob::VERSION
-  description "Reconnaissance tool for GitHub organizations."
-
-  options['bind-address'] = '127.0.0.1'
-  options['port']         = 9393
-  options['threads']      = 3
-
-  on('-o', '--organization NAME', "Name of GitHub organization")
-  on('-s', '--start-server', "Start web server, don't run any checks")
-  on('-p', '--port PORT', "Port to bind web server to")
-  on('-b', '--bind-address ADDRESS', "Address to bind web server to")
-  on('-t', '--threads THREADS', "Number of threads to use")
-  on('--delete NAME', "Delete an organization in the database")
-  on('--reset-db', "Resets the database")
-  on('--configure', "Start configuration wizard")
-  on('--no-server', "Don't start the server when finished")
-  on('--no-color', "Don't colorize output")
-  on('--no-banner', "Don't print Gitrob banner")
-
-  begin
-    if ARGV.empty?
-      Gitrob::fatal("No options given; see gitrob --help for options.")
-    end
-
-    go!
-  rescue Gitrob::Github::HttpClient::MissingAccessTokensError
-    Gitrob::fatal("Configuration file does not contain any GitHub access tokens. Run Gitrob with --configure flag to set it up.")
-  rescue Gitrob::Github::HttpClient::AccessTokensDepletedError
-    Gitrob::fatal("All GitHub access tokens under rate limiting. Go have a cup of coffee and try again.")
-  rescue Gitrob::Github::HttpClient::RequestError => e
-    Gitrob::fatal("A request to the GitHub API failed: #{e.message}")
-  rescue Interrupt
-    print "\b\b\n"; # Remove ^C from screen
-    Gitrob::fatal("Caught interrupt; exiting.")
-  rescue => e
-    Gitrob::fatal("An error occurred: #{e.class.name}: #{e.message}")
-  end
-end

data/lib/gitrob/github/blob.rb

@@ -1,41 +0,0 @@
-module Gitrob
-  module Github
-    class Blob
-      attr_reader :path, :size, :repository
-
-      def initialize(path, size, repository)
-        @path, @size, @repository = path, size, repository
-      end
-
-      def extension
-        File.extname(path)[1..-1]
-      end
-
-      def filename
-        File.basename(path)
-      end
-
-      def dirname
-        File.dirname(path)
-      end
-
-      def url
-        "https://github.com/#{URI.escape(repository.owner)}/#{URI.escape(repository.name)}/blob/master/#{URI.escape(path)}"
-      end
-
-      def to_model(organization, repository)
-        repository.blobs.new(
-          :path           => self.path,
-          :filename       => self.filename,
-          :extension      => self.extension,
-          :size           => self.size,
-          :organization   => organization
-        )
-      end
-
-      def save_to_database!(organization, repository)
-        self.to_model(organization, repository).tap { |m| m.save }
-      end
-    end
-  end
-end

data/lib/gitrob/github/http_client.rb

@@ -1,127 +0,0 @@
-module Gitrob
-  module Github
-    class HttpClient
-      include HTTParty
-      base_uri 'https://api.github.com'
-
-      class HttpError < StandardError; end
-      class ConnectionError < HttpError; end
-
-      class RequestError < HttpError
-        attr_reader :status, :body
-        def initialize(method, path, status, body, options)
-          @status = status
-          @body   = body
-          super("#{method} to #{path} returned status #{status} - options: #{options.inspect}")
-        end
-      end
-
-      class ClientError < RequestError; end
-      class ServerError < RequestError; end
-
-      class UnhandledError < StandardError; end
-
-      class AccessTokenError < StandardError; end
-      class MissingAccessTokensError < AccessTokenError; end
-      class AccessTokensDepletedError < AccessTokenError; end
-
-      DEFAULT_TIMEOUT = 0.5 #seconds
-      DEFAULT_RETRIES = 3
-
-      Response = Struct.new(:status, :headers, :body)
-
-      RETRIABLE_EXCEPTIONS = [
-        ServerError,
-        AccessTokenError,
-        Timeout::Error,
-        Errno::ETIMEDOUT,
-        Errno::ECONNRESET,
-        Errno::ECONNREFUSED,
-        Errno::ENETUNREACH,
-        Errno::EHOSTUNREACH,
-        EOFError
-      ]
-
-      def initialize(options)
-        @config = {
-          :timeout => DEFAULT_TIMEOUT,
-          :retries => DEFAULT_RETRIES
-        }.merge(options)
-        raise MissingAccessTokensErrors.new("No access tokens given") unless @config[:access_tokens]
-        default_timeout = @config[:timeout]
-      end
-
-      def do_get(path, params=nil, options={})
-        do_request(:get, path, {:query => params}.merge(options))
-      end
-
-      def do_post(path, params=nil, options={})
-        do_request(:post, path, {:query => params}.merge(options))
-      end
-
-      def do_put(path, params=nil, options={})
-        do_request(:put, path, {:query => params}.merge(options))
-      end
-
-      def do_delete(path, params=nil, options={})
-        do_request(:delete, path, {:query => params}.merge(options))
-      end
-
-    private
-
-      def do_request(method, path, options)
-        with_retries do
-          access_token = get_access_token!
-          response     = self.class.send(method, path, {
-            :headers => {
-              'Authorization' => "token #{access_token}",
-              'User-Agent'    => "Gitrob v#{Gitrob::VERSION}"
-            }
-          }.merge(options))
-          handle_possible_error!(method, path, response, options, access_token)
-          Response.new(response.code, response.headers, response.body)
-        end
-      end
-
-      def with_retries(&block)
-        tries = @config[:retries]
-        yield
-      rescue *RETRIABLE_EXCEPTIONS => ex
-        if (tries -= 1) > 0
-          sleep 0.2
-          retry
-        else
-          raise ex
-        end
-      end
-
-      def handle_possible_error!(method, path, response, options, access_token)
-        if access_token_rate_limited?(response) || access_token_unauthorized?(response)
-          access_tokens.delete(access_token)
-          raise AccessTokenError
-        elsif response.code >= 500
-          raise ServerError.new(method, path, response.code, response.body, options)
-        elsif response.code >= 400
-          raise ClientError.new(method, path, response.code, response.body, options)
-        end
-      end
-
-      def access_token_rate_limited?(response)
-        response.code == 403 && response.headers['X-RateLimit-Remaining'].to_i.zero?
-      end
-
-      def access_token_unauthorized?(response)
-        response.code == 401
-      end
-
-      def get_access_token!
-        raise AccessTokensDepletedError.new("Rate limit on all access tokens has been used up") if access_tokens.count.zero?
-        access_tokens.sample
-      end
-
-      def access_tokens
-        @config[:access_tokens]
-      end
-    end
-  end
-end

data/lib/gitrob/github/organization.rb

@@ -1,99 +0,0 @@
-module Gitrob
-  module Github
-    class Organization
-      attr_reader :name, :http_client
-
-      def initialize(name, http_client)
-        @name, @http_client = name, http_client
-      end
-
-      def display_name
-        info['name'].to_s.empty? ? info['login'] : info['name']
-      end
-
-      def login
-        info['login']
-      end
-
-      def website
-        info['blog']
-      end
-
-      def location
-        info['location']
-      end
-
-      def email
-        info['email']
-      end
-
-      def url
-        "https://github.com/#{name}"
-      end
-
-      def avatar_url
-        info['avatar_url']
-      end
-
-      def repositories
-        @repositories ||= recursive_repositories
-      end
-
-      def members
-        @members ||= recursive_members
-      end
-
-      def to_model
-        Gitrob::Organization.new(
-          :name       => self.display_name,
-          :login      => self.login,
-          :website    => self.website,
-          :location   => self.location,
-          :email      => self.email,
-          :avatar_url => self.avatar_url,
-          :url        => self.url
-        )
-      end
-
-      def save_to_database!
-        self.to_model.tap { |m| m.save }
-      end
-
-    private
-
-      def recursive_members(page = 1)
-        members = Array.new
-        response = http_client.do_get("/orgs/#{name}/members?page=#{page.to_i}")
-        JSON.parse(response.body).each do |member|
-          members << User.new(member['login'], http_client)
-        end
-
-        if response.headers.include?('link') && response.headers['link'].include?('rel="next"')
-          members += recursive_members(page + 1)
-        end
-        members
-      end
-
-      def recursive_repositories(page = 1)
-        repositories = Array.new
-        response = http_client.do_get("/orgs/#{name}/repos?page=#{page.to_i}")
-        JSON.parse(response.body).each do |repo|
-          next if repo['fork']
-          repositories << Repository.new(name, repo['name'], http_client)
-        end
-
-        if response.headers.include?('link') && response.headers['link'].include?('rel="next"')
-          repositories += recursive_repositories(page + 1)
-        end
-        repositories
-      end
-
-      def info
-        if !@info
-          @info = JSON.parse(http_client.do_get("/orgs/#{name}").body)
-        end
-        @info
-      end
-    end
-  end
-end

data/lib/gitrob/github/repository.rb

@@ -1,72 +0,0 @@
-module Gitrob
-  module Github
-    class Repository
-
-      attr_reader :owner, :name, :http_client
-      def initialize(owner, name, http_client)
-        @owner, @name, @http_client = owner, name, http_client
-      end
-
-      def contents
-        if !@contents
-          @contents = []
-          response = JSON.parse(http_client.do_get("/repos/#{owner}/#{name}/git/trees/master?recursive=1").body)
-          response['tree'].each do |object|
-            next unless object['type'] == 'blob'
-            @contents << Blob.new(object['path'], object['size'], self)
-          end
-        end
-        @contents
-      rescue HttpClient::ClientError => ex
-        if ex.status == 409 || ex.status == 404
-          @contents = []
-        else
-          raise ex
-        end
-      end
-
-      def full_name
-        [owner, name].join('/')
-      end
-
-      def url
-        info['html_url']
-      end
-
-      def description
-        info['description']
-      end
-
-      def website
-        info['homepage']
-      end
-
-      def to_model(organization, user = nil)
-        Gitrob::Repo.new(
-          :name         => self.name,
-          :owner_name   => self.owner,
-          :description  => self.description,
-          :website      => self.website,
-          :url          => self.url,
-          :organization => organization,
-          :user         => user
-        )
-      end
-
-      def save_to_database!(organization, user = nil)
-        self.to_model(organization, user).tap { |m| m.save }
-      rescue DataMapper::SaveFailureError => e
-        puts e.resource.errors.inspect
-      end
-
-    private
-
-      def info
-        if !@info
-          @info = JSON.parse(http_client.do_get("/repos/#{owner}/#{name}").body)
-        end
-        @info
-      end
-    end
-  end
-end

data/lib/gitrob/github/user.rb

@@ -1,84 +0,0 @@
-module Gitrob
-  module Github
-    class User
-
-      attr_reader :username, :http_client
-
-      def initialize(username, http_client)
-        @username, @http_client = username, http_client
-      end
-
-      def name
-        info['name'] || username
-      end
-
-      def email
-        info['email']
-      end
-
-      def website
-        info['blog']
-      end
-
-      def location
-        info['location']
-      end
-
-      def bio
-        info['bio']
-      end
-
-      def url
-        info['html_url']
-      end
-
-      def avatar_url
-        info['avatar_url']
-      end
-
-      def repositories
-        @repositories ||= recursive_repositories
-      end
-
-      def to_model(organization)
-        organization.users.new(
-          :username   => self.username,
-          :name       => self.name,
-          :website    => self.website,
-          :location   => self.location,
-          :email      => self.email,
-          :bio        => self.bio,
-          :url        => self.url,
-          :avatar_url => self.avatar_url
-        )
-      end
-
-      def save_to_database!(organization)
-        self.to_model(organization).tap { |m| m.save }
-      end
-
-    private
-
-      def recursive_repositories(page = 1)
-        repositories = Array.new
-        response = http_client.do_get("/users/#{username}/repos?page=#{page.to_i}")
-        JSON.parse(response.body).each do |repo|
-          next if repo['fork']
-          repositories << Repository.new(username, repo['name'], http_client)
-        end
-
-        if response.headers.include?('link') && response.headers['link'].include?('rel="next"')
-          repositories += recursive_repositories(page + 1)
-        end
-        repositories
-      end
-
-      def info
-        if !@info
-          @info = JSON.parse(http_client.do_get("/users/#{username}").body)
-        end
-        @info
-      end
-    end
-  end
-end