gitrob 0.0.6 → 1.0.0

data/db/migrations/002_create_github_access_tokens.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  change do
+    create_table(:github_access_tokens) do
+      primary_key :id
+      foreign_key :assessment_id, :assessments, :on_delete => :cascade, :index => true
+      String :token, :size => 40, :fixed => true
+      DateTime :updated_at
+      DateTime :created_at
+    end
+  end
+end

data/db/migrations/003_create_owners.rb

@@ -0,0 +1,24 @@
+Sequel.migration do
+  change do
+    create_table(:owners) do
+      primary_key :id
+      Integer :github_id, :index => true
+      foreign_key :assessment_id, :assessments, :on_delete => :cascade, :index => true
+      String :login
+      String :type, :size => 12, :index => true
+      String :url
+      String :html_url
+      String :avatar_url
+      String :name
+      String :blog
+      String :location
+      String :email
+      String :bio
+      Integer :repositories_count, :index => true, :default => 0
+      Integer :blobs_count, :index => true, :default => 0
+      Integer :findings_count, :index => true, :default => 0
+      DateTime :updated_at
+      DateTime :created_at
+    end
+  end
+end

data/db/migrations/004_create_repositories.rb

@@ -0,0 +1,23 @@
+Sequel.migration do
+  change do
+    create_table(:repositories) do
+      primary_key :id
+      Integer :github_id, :index => true
+      foreign_key :assessment_id, :assessments, :on_delete => :cascade, :index => true
+      foreign_key :owner_id, :owners, :on_delete => :cascade, :index => true
+      String :name
+      String :full_name
+      String :description
+      Boolean :private
+      String :url
+      String :html_url
+      String :homepage
+      Integer :size
+      String :default_branch
+      Integer :blobs_count, :default => 0
+      Integer :findings_count, :index => true, :default => 0
+      DateTime :updated_at
+      DateTime :created_at
+    end
+  end
+end

data/db/migrations/005_create_blobs.rb

@@ -0,0 +1,16 @@
+Sequel.migration do
+  change do
+    create_table(:blobs) do
+      primary_key :id
+      foreign_key :assessment_id, :assessments, :on_delete => :cascade, :index => true
+      foreign_key :owner_id, :owners, :on_delete => :cascade, :index => true
+      foreign_key :repository_id, :repositories, :on_delete => :cascade, :index => true
+      String :path
+      Integer :size, :index => true
+      String :sha, :size => 40, :fixed => true, :index => true
+      Integer :flags_count, :index => true, :default => 0
+      DateTime :updated_at
+      DateTime :created_at
+    end
+  end
+end

data/db/migrations/006_create_flags.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  change do
+    create_table(:flags) do
+      primary_key :id
+      foreign_key :assessment_id, :assessments, :on_delete => :cascade, :index => true
+      foreign_key :blob_id, :blobs, :on_delete => :cascade, :index => true
+      String :caption
+      String :description
+      DateTime :updated_at
+      DateTime :created_at
+    end
+  end
+end

data/db/migrations/007_create_comparisons.rb

@@ -0,0 +1,17 @@
+Sequel.migration do
+  change do
+    create_table(:comparisons) do
+      primary_key :id
+      foreign_key :primary_assessment_id, :assessments, :on_delete => :cascade, :index => true
+      foreign_key :secondary_assessment_id, :assessments, :on_delete => :cascade, :index => true
+      Integer :blobs_count, :default => 0
+      Integer :repositories_count, :default => 0
+      Integer :owners_count, :default => 0
+      Integer :findings_count, :default => 0
+      Boolean :finished, :default => false, :index => true
+      Boolean :deleted, :default => false, :index => true
+      DateTime :updated_at
+      DateTime :created_at
+    end
+  end
+end

data/db/migrations/008_create_blobs_comparisons.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:blobs_comparisons) do
+      foreign_key :comparison_id, :comparisons, :on_delete => :cascade, :index => true
+      foreign_key :blob_id, :blobs, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/db/migrations/009_create_comparisons_repositories.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:comparisons_repositories) do
+      foreign_key :comparison_id, :comparisons, :on_delete => :cascade, :index => true
+      foreign_key :repository_id, :repositories, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/db/migrations/010_create_comparisons_owners.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:comparisons_owners) do
+      foreign_key :comparison_id, :comparisons, :on_delete => :cascade, :index => true
+      foreign_key :owner_id, :owners, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/exe/gitrob

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# :nocov:
+require "gitrob"
+
+Gitrob::CLI.start(ARGV)
+# :nocov:

data/gitrob.gemspec

@@ -1,36 +1,43 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'gitrob/version'
+require "gitrob/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "gitrob"
   spec.version       = Gitrob::VERSION
   spec.authors       = ["Michael Henriksen"]
   spec.email         = ["michenriksen@neomailbox.ch"]
-  spec.summary       = %q{Reconnaissance tool for GitHub organizations.}
-  spec.description   = %q{Reconnaissance tool for GitHub organizations.}
+
+  spec.summary       = %q{Reconnaissance tool for GitHub organizations}
   spec.homepage      = "https://github.com/michenriksen/gitrob"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "httparty", "~> 0.13"
-  spec.add_dependency "methadone", "~> 1.7"
-  spec.add_dependency "highline", "~> 1.6"
-  spec.add_dependency "paint", "~> 0.8"
-  spec.add_dependency "ruby-progressbar", "~> 1.6"
-  spec.add_dependency "thread", "~> 0.1"
+  spec.add_dependency "thor", "~> 0.19"
+  spec.add_dependency "colorize", "~> 0.7"
+  spec.add_dependency "highline", "~> 1.7"
+  spec.add_dependency "thread", "~> 0.2"
+  spec.add_dependency "ruby-progressbar", "~> 1.7"
   spec.add_dependency "sinatra", "~> 1.4"
   spec.add_dependency "thin", "~> 1.6"
-  spec.add_dependency "datamapper", "~> 1.2"
-  spec.add_dependency "dm-postgres-adapter"
+  spec.add_dependency "pg", "~> 0.18"
+  spec.add_dependency "sequel", "~> 4.27"
+  spec.add_dependency "github_api", "~> 0.12"
+  spec.add_dependency "sucker_punch", "~> 2.0", ">= 2.0.1"
 
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.1"
-  spec.add_development_dependency "webmock", "~> 1.20"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "timecop"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "factory_girl"
+  spec.add_development_dependency "faker"
+  spec.add_development_dependency "awesome_print"
+  spec.add_development_dependency "webmock"
 end

data/lib/gitrob/blob_observer.rb

@@ -0,0 +1,103 @@
+module Gitrob
+  class BlobObserver
+    SIGNATURES_FILE_PATH = File.expand_path(
+      "../../../signatures.json", __FILE__)
+
+    REQUIRED_SIGNATURE_KEYS = %w(part type pattern caption description)
+    ALLOWED_TYPES           = %w(regex match)
+    ALLOWED_PARTS           = %w(path filename extension)
+
+    class Signature < OpenStruct; end
+    class CorruptSignaturesError < StandardError; end
+
+    def self.observe(blob)
+      signatures.each do |signature|
+        if signature.type == "match"
+          observe_with_match_signature(blob, signature)
+        else
+          observe_with_regex_signature(blob, signature)
+        end
+      end
+      blob.flags_count = blob.flags.count
+      blob.save
+      blob.flags_count
+    end
+
+    def self.signatures
+      load_signatures! unless @signatures
+      @signatures
+    end
+
+    def self.load_signatures!
+      @signatures = []
+      JSON.load(File.read(SIGNATURES_FILE_PATH)).each do |signature|
+        @signatures << Signature.new(signature)
+      end
+      validate_signatures!
+    rescue CorruptSignaturesError => e
+      raise e
+    rescue
+      raise CorruptSignaturesError, "Could not parse signature file"
+    end
+
+    def self.validate_signatures!
+      if !signatures.is_a?(Array) || signatures.empty?
+        fail CorruptSignaturesError,
+             "Signature file contains no signatures"
+      end
+      signatures.each do |signature|
+        validate_signature!(signature)
+      end
+    end
+
+    def self.validate_signature!(signature)
+      validate_signature_keys!(signature)
+      validate_signature_type!(signature)
+      validate_signature_part!(signature)
+    end
+
+    def self.validate_signature_keys!(signature)
+      REQUIRED_SIGNATURE_KEYS.each do |key|
+        unless signature.respond_to?(key)
+          fail CorruptSignaturesError,
+               "Missing required signature key: #{key}"
+        end
+      end
+    end
+
+    def self.validate_signature_type!(signature)
+      unless ALLOWED_TYPES.include?(signature.type)
+        fail CorruptSignaturesError,
+             "Invalid signature type: #{signature.type}"
+      end
+    end
+
+    def self.validate_signature_part!(signature)
+      unless ALLOWED_PARTS.include?(signature.part)
+        fail CorruptSignaturesError,
+             "Invalid signature part: #{signature.part}"
+      end
+    end
+
+    def self.observe_with_match_signature(blob, signature)
+      haystack = blob.send(signature.part.to_sym)
+      return unless haystack == signature.pattern
+      blob.add_flag(
+        :caption     => signature.caption,
+        :description => signature.description,
+        :assessment  => blob.assessment
+      )
+    end
+
+    def self.observe_with_regex_signature(blob, signature)
+      haystack = blob.send(signature.part.to_sym)
+      regex    = Regexp.new(signature.pattern, Regexp::IGNORECASE)
+      return if regex.match(haystack).nil?
+      blob.add_flag(
+        :caption     => signature.caption,
+        :description => signature.description,
+        :assessment  => blob.assessment
+      )
+    end
+  end
+end

data/lib/gitrob/cli/command.rb

@@ -0,0 +1,58 @@
+module Gitrob
+  class CLI
+    class Command
+      attr_reader :options
+
+      def self.start(*args)
+        new(*args)
+      end
+
+      def initialize(*args) # rubocop:disable Lint/UnusedMethodArgument
+      end
+
+      def info(*args)
+        Gitrob::CLI.info(*args)
+      end
+
+      def task(message, fatal_error=false, &block)
+        Gitrob::CLI.task(message, fatal_error, &block)
+      end
+
+      def warn(*args)
+        Gitrob::CLI.warn(*args)
+      end
+
+      def error(*args)
+        Gitrob::CLI.error(*args)
+      end
+
+      def fatal(*args)
+        Gitrob::CLI.fatal(*args)
+      end
+
+      def debug(*args)
+        Gitrob::CLI.debug(*args)
+      end
+
+      def debugging_enabled?
+        Gitrob::CLI.debugging_enabled?
+      end
+
+      def output(*args)
+        Gitrob::CLI.output(*args)
+      end
+
+      def thread_pool
+        pool = Thread::Pool.new(options[:threads] || 5)
+        yield pool
+        pool.shutdown
+      end
+
+      def progress_bar(message, options)
+        progress_bar = Gitrob::CLI::ProgressBar.new(message, options)
+        yield progress_bar
+        progress_bar.finish
+      end
+    end
+  end
+end

data/lib/gitrob/cli/commands/accept_terms_of_use.rb

@@ -0,0 +1,61 @@
+module Gitrob
+  class CLI
+    module Commands
+      class AcceptTermsOfUse < Gitrob::CLI::Command
+        AGREEMENT_FILE_PATH = File.expand_path(
+          "../../../../../agreement.txt", __FILE__)
+
+        LICENSE_FILE_PATH = File.expand_path(
+          "../../../../../LICENSE.txt", __FILE__)
+
+        AGREEMENT = "Gitrob is designed for security professionals. " \
+                    "If you use any information\nfound through this " \
+                    "tool for malicious purposes that are not " \
+                    "authorized by\nthe target, you are " \
+                    "violating the terms of use and license of " \
+                    "this\ntool. By typing y/yes, you agree to the " \
+                    "terms of use and that you will use\nthis tool " \
+                    "for lawful purposes only."
+
+        VALID_ANSWERS = %w(y yes n no)
+        POSITIVE_ANSWERS = %w(y yes)
+
+        def initialize(options)
+          @options = options
+          return if terms_of_use_accepted?
+          present_terms_of_use
+          handle_user_input
+        end
+
+        private
+
+        def terms_of_use_accepted?
+          File.exist?(AGREEMENT_FILE_PATH)
+        end
+
+        def present_terms_of_use
+          output "\n#{license}\n\n"
+          output AGREEMENT.light_red
+        end
+
+        def handle_user_input
+          if agree("\n\nDo you agree to the terms of use? (y/n): ".light_green)
+            accept_terms_of_use
+          else
+            fatal("Exiting Gitrob.")
+          end
+        end
+
+        def accept_terms_of_use
+          File.open(AGREEMENT_FILE_PATH, "w") do |file|
+            file.write("user accepted")
+          end
+        end
+
+        def license
+          File.read(LICENSE_FILE_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/gitrob/cli/commands/analyze/analysis.rb

@@ -0,0 +1,75 @@
+module Gitrob
+  class CLI
+    module Commands
+      class Analyze < Gitrob::CLI::Command
+        module Analysis
+          def analyze_repositories
+            repo_progress_bar do |progress|
+              github_data_manager.owners.each do |owner|
+                @db_owner = @db_assessment.save_owner(owner)
+                thread_pool do |pool|
+                  repositories_for_owner(owner).each do |repo|
+                    pool.process do
+                      db_repo = @db_assessment.save_repository(repo, @db_owner)
+                      blobs   = blobs_for_repository(repo)
+                      analyze_blobs(blobs, db_repo, @db_owner, progress)
+                    end
+                  end
+                end
+              end
+            end
+          end
+
+          def analyze_blobs(blobs, db_repo, db_owner, progress)
+            findings = 0
+            blobs.each do |blob|
+              db_blob = @db_assessment.save_blob(blob, db_repo, db_owner)
+              Gitrob::BlobObserver.observe(db_blob)
+              if db_blob.flags.count > 0
+                findings += 1
+                @db_assessment.findings_count += 1
+                db_owner.findings_count += 1
+                db_repo.findings_count += 1
+              end
+            end
+            db_owner.save
+            db_repo.save
+            progress.increment
+            report_findings(findings, db_repo, progress)
+          rescue => e
+            progress.error("#{e.class}: #{e.message}")
+          end
+
+          def report_findings(finding_count, repo, progress)
+            return if finding_count.zero?
+            files = finding_count == 1 ? "1 file" : "#{finding_count} files"
+            progress.info(
+              "Flagged #{files.to_s.light_yellow} " \
+              "in #{repo.full_name.bold}")
+          end
+        end
+
+        def repo_progress_bar
+          progress_bar(
+            "Analyzing repositories...",
+            :total => repository_count) do |progress|
+            yield progress
+          end
+          sleep 0.1
+        end
+
+        def repositories_for_owner(owner)
+          github_data_manager.repositories_for_owner(owner)
+        end
+
+        def blobs_for_repository(repo)
+          github_data_manager.blobs_for_repository(repo)
+        end
+
+        def repository_count
+          @github_data_manager.repositories.count
+        end
+      end
+    end
+  end
+end

data/lib/gitrob/cli/commands/analyze/gathering.rb

@@ -0,0 +1,101 @@
+module Gitrob
+  class CLI
+    module Commands
+      class Analyze < Gitrob::CLI::Command
+        module Gathering
+          def gather_owners
+            task("Gathering targets...") do
+              thread_pool do |pool|
+                github_data_manager.gather_owners(pool)
+              end
+            end
+            fatal("No users or organizations " \
+              "found; exiting") if owner_count.zero?
+          end
+
+          def gather_repositories
+            make_repo_gathering_progress_bar do |progress|
+              thread_pool do |pool|
+                github_data_manager
+                  .gather_repositories(pool) do |owner, repositories|
+                  repository_gathering_update(owner, repositories, progress)
+                end
+              end
+            end
+            fatal("No repositories found; exiting") if repo_count.zero?
+            info "Gathered #{repo_count} repositories"
+          end
+
+          def github_data_manager
+            unless @github_data_manager
+              @github_data_manager = Gitrob::Github::DataManager.new(
+                @targets, github_client_manager
+              )
+            end
+            @github_data_manager
+          end
+
+          def github_client_manager
+            unless @github_client_manager
+              @github_client_manager = Gitrob::Github::ClientManager.new(
+                client_manager_configuration
+              )
+            end
+            @github_client_manager
+          end
+
+          def client_manager_configuration
+            {
+              :endpoint      => @options[:endpoint],
+              :site          => @options[:site],
+              :ssl           => {
+                :verify => @options[:verify_ssl]
+              },
+              :access_tokens => github_access_tokens
+            }
+          end
+
+          def github_access_tokens
+            if @options[:access_tokens]
+              @options[:access_tokens].gsub("\n", ",").split(",").map(&:strip)
+            else
+              Gitrob::CLI.configuration["github_access_tokens"]
+            end
+          end
+
+          def make_repo_gathering_progress_bar
+            total = github_data_manager.owners.count
+            progress_bar(
+              "Gathering repositories from #{total} targets...",
+              :total => total
+            ) do |progress|
+              yield progress
+            end
+            sleep 0.1
+          end
+
+          def repo_count
+            github_data_manager.repositories.count
+          end
+
+          def owner_count
+            github_data_manager.owners.count
+          end
+
+          def repository_gathering_update(owner, repositories, progress_bar)
+            count = repositories.count
+            progress_bar.increment
+            return if count.zero?
+            gathered = Gitrob::Utils.pluralize(
+              count,
+              "repository",
+              "repositories"
+            )
+            progress_bar.info("Gathered #{gathered} " \
+                          "from #{owner['login'].bold}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitrob/cli/commands/analyze.rb

@@ -0,0 +1,63 @@
+require "gitrob/cli/commands/analyze/gathering"
+require "gitrob/cli/commands/analyze/analysis"
+
+module Gitrob
+  class CLI
+    module Commands
+      class Analyze < Gitrob::CLI::Command
+        include Gathering
+        include Analysis
+
+        def initialize(targets, options)
+          Thread.abort_on_exception = true
+          @options = options
+          @targets = targets.split(",").map(&:strip).uniq
+          load_signatures!
+          create_database_assessment
+          gather_owners
+          gather_repositories
+          analyze_repositories
+          @db_assessment.finished = true
+          @db_assessment.save
+          start_web_server
+        end
+
+        private
+
+        def create_database_assessment
+          @db_assessment = Gitrob::Models::Assessment.create(
+            :endpoint   => @options[:endpoint],
+            :site       => @options[:site],
+            :verify_ssl => @options[:verify_ssl],
+            :finished   => false
+          )
+          github_access_tokens.each do |access_token|
+            @db_assessment.save_github_access_token(access_token)
+          end
+        end
+
+        def load_signatures!
+          task("Loading signatures...", true) do
+            Gitrob::BlobObserver.load_signatures!
+          end
+        end
+
+        def start_web_server
+          return unless options[:server]
+          info "Starting web application on port #{options[:port]}..."
+          info "Browse to http://#{options[:bind_address]}:" \
+               "#{options[:port]}/ to see results!"
+
+          if debugging_enabled?
+            Sequel::Model.db.logger = QueryLogger.new(STDOUT)
+          end
+
+          Gitrob::WebApp.run!(
+            :port => options[:port].to_i,
+            :bind => options[:bind_address]
+          )
+        end
+      end
+    end
+  end
+end