gitrob 0.0.1

data/lib/gitrob/github/blob.rb

@@ -0,0 +1,41 @@
+module Gitrob
+  module Github
+    class Blob
+      attr_reader :path, :size, :repository
+
+      def initialize(path, size, repository)
+        @path, @size, @repository = path, size, repository
+      end
+
+      def extension
+        File.extname(path)[1..-1]
+      end
+
+      def filename
+        File.basename(path)
+      end
+
+      def dirname
+        File.dirname(path)
+      end
+
+      def url
+        "https://github.com/#{URI.escape(repository.owner)}/#{URI.escape(repository.name)}/blob/master/#{URI.escape(path)}"
+      end
+
+      def to_model(organization, repository)
+        repository.blobs.new(
+          :path           => self.path,
+          :filename       => self.filename,
+          :extension      => self.extension,
+          :size           => self.size,
+          :organization   => organization
+        )
+      end
+
+      def save_to_database!(organization, repository)
+        self.to_model(organization, repository).tap { |m| m.save }
+      end
+    end
+  end
+end

data/lib/gitrob/github/http_client.rb

@@ -0,0 +1,127 @@
+module Gitrob
+  module Github
+    class HttpClient
+      include HTTParty
+      base_uri 'https://api.github.com'
+
+      class HttpError < StandardError; end
+      class ConnectionError < HttpError; end
+
+      class RequestError < HttpError
+        attr_reader :status, :body
+        def initialize(method, path, status, body, options)
+          @status = status
+          @body   = body
+          super("#{method} to #{path} returned status #{status} - options: #{options.inspect}")
+        end
+      end
+
+      class ClientError < RequestError; end
+      class ServerError < RequestError; end
+
+      class UnhandledError < StandardError; end
+
+      class AccessTokenError < StandardError; end
+      class MissingAccessTokensError < AccessTokenError; end
+      class AccessTokensDepletedError < AccessTokenError; end
+
+      DEFAULT_TIMEOUT = 0.5 #seconds
+      DEFAULT_RETRIES = 3
+
+      Response = Struct.new(:status, :headers, :body)
+
+      RETRIABLE_EXCEPTIONS = [
+        ServerError,
+        AccessTokenError,
+        Timeout::Error,
+        Errno::ETIMEDOUT,
+        Errno::ECONNRESET,
+        Errno::ECONNREFUSED,
+        Errno::ENETUNREACH,
+        Errno::EHOSTUNREACH,
+        EOFError
+      ]
+
+      def initialize(options)
+        @config = {
+          :timeout => DEFAULT_TIMEOUT,
+          :retries => DEFAULT_RETRIES
+        }.merge(options)
+        raise MissingAccessTokensErrors.new("No access tokens given") unless @config[:access_tokens]
+        default_timeout = @config[:timeout]
+      end
+
+      def do_get(path, params=nil, options={})
+        do_request(:get, path, {:query => params}.merge(options))
+      end
+
+      def do_post(path, params=nil, options={})
+        do_request(:post, path, {:query => params}.merge(options))
+      end
+
+      def do_put(path, params=nil, options={})
+        do_request(:put, path, {:query => params}.merge(options))
+      end
+
+      def do_delete(path, params=nil, options={})
+        do_request(:delete, path, {:query => params}.merge(options))
+      end
+
+    private
+
+      def do_request(method, path, options)
+        with_retries do
+          access_token = get_access_token!
+          response     = self.class.send(method, path, {
+            :headers => {
+              'Authorization' => "token #{access_token}",
+              'User-Agent'    => "Gitrob v#{Gitrob::VERSION}"
+            }
+          }.merge(options))
+          handle_possible_error!(method, path, response, options, access_token)
+          Response.new(response.code, response.headers, response.body)
+        end
+      end
+
+      def with_retries(&block)
+        tries = @config[:retries]
+        yield
+      rescue *RETRIABLE_EXCEPTIONS => ex
+        if (tries -= 1) > 0
+          sleep 0.2
+          retry
+        else
+          raise ex
+        end
+      end
+
+      def handle_possible_error!(method, path, response, options, access_token)
+        if access_token_rate_limited?(response) || access_token_unauthorized?(response)
+          access_tokens.delete(access_token)
+          raise AccessTokenError
+        elsif response.code >= 500
+          raise ServerError.new(method, path, response.code, response.body, options)
+        elsif response.code >= 400
+          raise ClientError.new(method, path, response.code, response.body, options)
+        end
+      end
+
+      def access_token_rate_limited?(response)
+        response.code == 403 && response.headers['X-RateLimit-Remaining'].to_i.zero?
+      end
+
+      def access_token_unauthorized?(response)
+        response.code == 401
+      end
+
+      def get_access_token!
+        raise AccessTokensDepletedError.new("Rate limit on all access tokens has been used up") if access_tokens.count.zero?
+        access_tokens.sample
+      end
+
+      def access_tokens
+        @config[:access_tokens]
+      end
+    end
+  end
+end

data/lib/gitrob/github/organization.rb

@@ -0,0 +1,93 @@
+module Gitrob
+  module Github
+    class Organization
+      attr_reader :name, :http_client
+
+      def initialize(name, http_client)
+        @name, @http_client = name, http_client
+      end
+
+      def display_name
+        info['name'].to_s.empty? ? info['login'] : info['name']
+      end
+
+      def login
+        info['login']
+      end
+
+      def website
+        info['blog']
+      end
+
+      def location
+        info['location']
+      end
+
+      def email
+        info['email']
+      end
+
+      def url
+        "https://github.com/#{name}"
+      end
+
+      def avatar_url
+        info['avatar_url']
+      end
+
+      def repositories
+        if !@repositories
+          @repositories = []
+          response = JSON.parse(http_client.do_get("/orgs/#{name}/repos").body)
+          response.each do |repo|
+            next if repo['fork']
+            @repositories << Repository.new(name, repo['name'], http_client)
+          end
+        end
+        @repositories
+      end
+
+      def members()
+        @members ||= recursive_members
+      end
+
+      def to_model
+        Gitrob::Organization.new(
+          :name       => self.display_name,
+          :login      => self.login,
+          :website    => self.website,
+          :location   => self.location,
+          :email      => self.email,
+          :avatar_url => self.avatar_url,
+          :url        => self.url
+        )
+      end
+
+      def save_to_database!
+        self.to_model.tap { |m| m.save }
+      end
+
+    private
+
+      def recursive_members(page = 1)
+        members = Array.new
+        response = http_client.do_get("/orgs/#{name}/members?page=#{page.to_i}")
+        JSON.parse(response.body).each do |member|
+          members << User.new(member['login'], http_client)
+        end
+
+        if response.headers.include?('link') && response.headers['link'].include?('rel="next"')
+          members += recursive_members(page + 1)
+        end
+        members
+      end
+
+      def info
+        if !@info
+          @info = JSON.parse(http_client.do_get("/orgs/#{name}").body)
+        end
+        @info
+      end
+    end
+  end
+end

data/lib/gitrob/github/repository.rb

@@ -0,0 +1,72 @@
+module Gitrob
+  module Github
+    class Repository
+
+      attr_reader :owner, :name, :http_client
+      def initialize(owner, name, http_client)
+        @owner, @name, @http_client = owner, name, http_client
+      end
+
+      def contents
+        if !@contents
+          @contents = []
+          response = JSON.parse(http_client.do_get("/repos/#{owner}/#{name}/git/trees/master?recursive=1").body)
+          response['tree'].each do |object|
+            next unless object['type'] == 'blob'
+            @contents << Blob.new(object['path'], object['size'], self)
+          end
+        end
+        @contents
+      rescue HttpClient::ClientError => ex
+        if ex.status == 409 || ex.status == 404
+          @contents = []
+        else
+          raise ex
+        end
+      end
+
+      def full_name
+        [owner, name].join('/')
+      end
+
+      def url
+        info['html_url']
+      end
+
+      def description
+        info['description']
+      end
+
+      def website
+        info['homepage']
+      end
+
+      def to_model(organization, user = nil)
+        Gitrob::Repo.new(
+          :name         => self.name,
+          :owner_name   => self.owner,
+          :description  => self.description,
+          :website      => self.website,
+          :url          => self.url,
+          :organization => organization,
+          :user         => user
+        )
+      end
+
+      def save_to_database!(organization, user = nil)
+        self.to_model(organization, user).tap { |m| m.save }
+      rescue DataMapper::SaveFailureError => e
+        puts e.resource.errors.inspect
+      end
+
+    private
+
+      def info
+        if !@info
+          @info = JSON.parse(http_client.do_get("/repos/#{owner}/#{name}").body)
+        end
+        @info
+      end
+    end
+  end
+end

data/lib/gitrob/github/user.rb

@@ -0,0 +1,78 @@
+module Gitrob
+  module Github
+    class User
+
+      attr_reader :username, :http_client
+
+      def initialize(username, http_client)
+        @username, @http_client = username, http_client
+      end
+
+      def name
+        info['name'] || username
+      end
+
+      def email
+        info['email']
+      end
+
+      def website
+        info['blog']
+      end
+
+      def location
+        info['location']
+      end
+
+      def bio
+        info['bio']
+      end
+
+      def url
+        info['html_url']
+      end
+
+      def avatar_url
+        info['avatar_url']
+      end
+
+      def repositories
+        if !@repositories
+          @repositories = []
+          response = JSON.parse(http_client.do_get("/users/#{username}/repos").body)
+          response.each do |repo|
+            next if repo['fork']
+            @repositories << Repository.new(username, repo['name'], http_client)
+          end
+        end
+        @repositories
+      end
+
+      def to_model(organization)
+        organization.users.new(
+          :username   => self.username,
+          :name       => self.name,
+          :website    => self.website,
+          :location   => self.location,
+          :email      => self.email,
+          :bio        => self.bio,
+          :url        => self.url,
+          :avatar_url => self.avatar_url
+        )
+      end
+
+      def save_to_database!(organization)
+        self.to_model(organization).tap { |m| m.save }
+      end
+
+    private
+
+      def info
+        if !@info
+          @info = JSON.parse(http_client.do_get("/users/#{username}").body)
+        end
+        @info
+      end
+    end
+  end
+end

data/lib/gitrob/observers/sensitive_files.rb

@@ -0,0 +1,82 @@
+module Gitrob
+  module Observers
+    class SensitiveFiles
+
+      class InvalidPatternFileError < StandardError; end
+      class InvalidPatternError < StandardError; end
+
+      VALID_KEYS  = %w(part type pattern caption description)
+      VALID_PARTS = %w(path filename extension)
+      VALID_TYPES = %w(match regex)
+
+      def self.observe(blob)
+        patterns.each do |pattern|
+          check_blob(blob, pattern)
+        end
+      end
+
+      def self.load_patterns!
+        patterns = read_pattern_file!
+        validate_patterns!(patterns)
+        @patterns = patterns
+      end
+
+      def self.patterns
+        @patterns
+      end
+
+    private
+
+      def self.read_pattern_file!
+        JSON.parse(File.read("#{File.dirname(__FILE__)}/../../../patterns.json"))
+      rescue JSON::ParserError => e
+        raise InvalidPatternFileError.new("Cannot parse pattern file: #{e.message}")
+      end
+
+      def self.validate_patterns!(patterns)
+        if !patterns.is_a?(Array) || patterns.empty?
+          raise InvalidPatternFileError.new("Pattern file contains no patterns")
+        end
+        patterns.each do |pattern|
+          validate_pattern!(pattern)
+        end
+      end
+
+      def self.validate_pattern!(pattern)
+        pattern.keys.each do |key|
+          if !VALID_KEYS.include?(key)
+            raise InvalidPatternError.new("Pattern contains unknown key: #{key}")
+          end
+        end
+
+        if !VALID_PARTS.include?(pattern['part'])
+          raise InvalidPatternError.new("Pattern has unknown part: #{pattern['part']}")
+        end
+
+        if !VALID_TYPES.include?(pattern['type'])
+          raise InvalidPatternError.new("Pattern has unknown type: #{pattern['type']}")
+        end
+      end
+
+      def self.check_blob(blob, pattern)
+        haystack = blob.send(pattern['part'].to_sym)
+        if pattern['type'] == 'match'
+          if haystack == pattern['pattern']
+            blob.findings.new(
+              :caption     => pattern['caption'],
+              :description => pattern['description']
+            )
+          end
+        else
+          regex = Regexp.new(pattern['pattern'], Regexp::IGNORECASE)
+          if !regex.match(haystack).nil?
+            blob.findings.new(
+              :caption     => pattern['caption'],
+              :description => pattern['description']
+            )
+          end
+        end
+      end
+    end
+  end
+end