github_bitbucket_audit 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 892bb28e0de26ae3d42697e84ff19131d6764915
+  data.tar.gz: af6a30b4030441d0c8e5b3b6cc064d87bc573618
+SHA512:
+  metadata.gz: bf452e796efea3b991e457d1f9ff6e1841ef6ee8569b653732bd5275b8f310920fd22dba71754328346816be27e7dbc351b80f7ae6bda3a9f0c32df1299a7f46
+  data.tar.gz: f1a352c1eaa3977cc9a60b0a76e2fb0fbd9557491fb69f3c4fd9716f624d3a0a28f693745de5866d892d4135682480ab7b89ecafedb5307ec70deeb065228452

data/.gitignore

@@ -0,0 +1 @@
+.rvmrc

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    github_bitbucket_audit (0.0.1)
+      github_api (~> 0.11.3)
+      semantic_logger (~> 2.7.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.5)
+    atomic (1.1.16-java)
+    coderay (1.1.0)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.3-java)
+    github_api (0.11.3)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.1)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 1.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.0)
+      oauth2
+    hashie (2.0.5)
+    jwt (0.1.11)
+      multi_json (>= 1.5)
+    method_source (0.8.2)
+    mini_portile (0.5.3)
+    multi_json (1.9.2)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.1-java)
+      mini_portile (~> 0.5.0)
+    oauth2 (0.9.3)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 0.1.8)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    pry (0.9.12.6-java)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+      spoon (~> 0.0)
+    rack (1.5.2)
+    semantic_logger (2.7.0)
+      sync_attr (>= 1.0)
+      thread_safe (>= 0.1.0)
+    slop (3.5.0)
+    spoon (0.0.4)
+      ffi
+    sync_attr (1.0.0)
+    thread_safe (0.3.1-java)
+      atomic (>= 1.1.7, < 2)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  github_bitbucket_audit!
+  pry

data/README.md

@@ -0,0 +1,9 @@
+The gem is used for scanning github accounts and check for certain keywords in there.
+
+It takes a list of username from a CSV file.
+Goes and fetches the public repos for those users.
+Gets the tree for each of those public repos.
+Checks the filenames against the list of keywords.
+For repos that match keywords, it writes them down in an output file.
+
+Usage: gh_bb_audit.rb -u <path_to_user_csv_file> -k <path_tpkeywords_csv_file> -o <path_to_output_file>

data/bin/gh_bb_audit.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/gh_bb_audit'
+require 'optparse'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: gh_bb_audit.rb -u <path_to_user_csv_file> -k <path_tpkeywords_csv_file> -o <path_to_output_file>"
+
+  opts.on("-users", "-u") do |v|
+    options[:user_file_path] = v
+  end
+
+  opts.on("-keywords", "-k") do |v|
+    options[:keywords_file_path] = v
+  end
+
+  opts.on("-output", "-o") do |v|
+    options[:output_file_path] = v
+  end
+
+end.parse!
+
+::GhBbAudit::GithubScanner.new(options[:user_file_path], options[:keywords_file_path], options[:output_file_path]).start_scan

data/gh_bb_audit.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gh_bb_audit/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = "github_bitbucket_audit"
+  spec.version     = "0.0.1"
+  spec.summary     = "Looks for specific keywords in the public repos of github and bitbucket"
+  spec.date        = "2014-04-01"
+  spec.description = "The library takes a list of users and searches their public repos for specific keywords"
+  spec.authors     = ["Ankur Maheshwari"]
+  spec.email       = ["amaheshwari@systango.com"]
+  spec.homepage    = "http://systango.com/"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "github_api", "~> 0.11.3"
+  spec.add_dependency 'semantic_logger', '~> 2.7.0'
+	
+  spec.add_development_dependency "pry"
+end

data/lib/gh_bb_audit.rb

@@ -0,0 +1,16 @@
+require 'semantic_logger' unless defined?(logger)
+require 'github_api'
+
+module GhBbAudit
+  GEM_ROOT = File.join(File.dirname(__FILE__), '..')
+  logger = SemanticLogger['GithubBitbucketAudit']
+end
+
+require_relative './gh_bb_audit/version'
+require_relative './gh_bb_audit/github_user'
+require_relative './gh_bb_audit/github_repo'
+require_relative './gh_bb_audit/github_scanner'
+require_relative './gh_bb_audit/keywords_list'
+require_relative './gh_bb_audit/keyword_matcher'
+require_relative './gh_bb_audit/users_list'
+require_relative './gh_bb_audit/output_writer'

data/lib/gh_bb_audit/github_repo.rb

@@ -0,0 +1,39 @@
+module GhBbAudit
+  class GithubRepo
+    def initialize(user_name,repo_name)
+      @user_name = user_name
+      @repo_name = repo_name
+    end
+
+    def get_all_file_paths
+      return [] if ( !@user_name || !@repo_name )
+      @paths ||= begin
+        get_tree.each.inject([]) do |result, file|
+          result << file.path
+          result
+        end
+      rescue Exception => e
+        []
+      end 
+      @paths
+    end
+
+    private
+    def get_tree
+      repo = ::Github::Repos.new user: @user_name, repo: @repo_name
+      last_commit_sha = repo.commits.all[0].sha rescue nil
+      if !last_commit_sha
+        logger.info("Could not find any commit for Github Repo:: #{@repo_name} for User:: #{@user_name}")
+        return nil
+      end
+      github = ::Github.new
+      @github_tree ||= github.git_data.trees.get @user_name, @repo_name, last_commit_sha , 'recursive' => true rescue nil
+      if !@github_tree
+        logger.info("Could not fetch tree for Repo:: #{@repo_name} for User:: #{@user_name}")
+        return 
+      end
+      @github_tree.tree
+    end
+
+  end
+end

data/lib/gh_bb_audit/github_scanner.rb

@@ -0,0 +1,32 @@
+module GhBbAudit
+	class GithubScanner
+    def initialize(user_csv_path, keyword_csv_path, output_file_path)
+      @user_csv_path = user_csv_path
+      @keyword_csv_path = keyword_csv_path
+      @output_file_path = output_file_path
+    end
+
+    def start_scan
+      output_writer = ::GhBbAudit::OutputWriter.new(@output_file_path)
+      begin
+        matcher = ::GhBbAudit::KeywordMatcher.new(::GhBbAudit::KeywordsList.new(@keyword_csv_path).all_keywords)
+        all_github_user = ::GhBbAudit::UsersList.new(@user_csv_path).all_users
+        
+        all_github_user.each do |user|
+          ::GhBbAudit::GithubUser.new(user).public_repos.each do |public_repo|
+            file_paths = ::GhBbAudit::GithubRepo.new(user,public_repo.name).get_all_file_paths
+            if matcher.repo_contains_keyword?(file_paths)
+              output_writer.write_red_flag_record(user,public_repo.name)
+            end
+          end
+        end
+      rescue StandardError => e
+        logger.error "Error in scanning Github ", error: e.inspect
+      ensure
+        output_writer.close
+      end
+    end
+
+	end
+
+end

data/lib/gh_bb_audit/github_user.rb

@@ -0,0 +1,11 @@
+module GhBbAudit
+  class GithubUser
+  	def initialize(user_name)
+      @user_name = user_name
+    end
+
+    def public_repos
+      @public_repos ||= (::Github.repos.list user: @user_name rescue [])
+    end
+  end
+end

data/lib/gh_bb_audit/keyword_matcher.rb

@@ -0,0 +1,12 @@
+module GhBbAudit
+  class KeywordMatcher
+
+    def initialize(keywords)
+      @keyword_regex = Regexp.union(keywords)
+    end
+
+    def repo_contains_keyword?(repo_file_paths)
+      !((repo_file_paths.select { |files|  @keyword_regex.match(files) }).empty?)
+    end
+  end
+end

data/lib/gh_bb_audit/keywords_list.rb

@@ -0,0 +1,13 @@
+require 'csv'
+module GhBbAudit
+  class KeywordsList
+  	def initialize(path_to_csv_file)
+      @keywords_csv_file_path = path_to_csv_file
+    end
+
+    def all_keywords
+      #Not rescuing here, as we should crash if we can not get keywords
+      @keywords ||= ::CSV.read(@keywords_csv_file_path).flatten.uniq 
+    end
+  end
+end

data/lib/gh_bb_audit/output_writer.rb

@@ -0,0 +1,15 @@
+module GhBbAudit
+  class OutputWriter
+    def initialize(path_to_file)
+    	@fhandle = File.open(path_to_file, 'w+') 
+    end
+
+    def write_red_flag_record(user_name, repo_name)
+      @fhandle.puts("Something fishy in REPO::#{repo_name} for USER:: #{user_name}")
+    end
+
+    def close
+      @fhandle.close
+    end
+  end
+end

data/lib/gh_bb_audit/users_list.rb

@@ -0,0 +1,13 @@
+require 'csv'
+module GhBbAudit
+  class UsersList
+  	def initialize(path_to_csv_file)
+      @user_csv_file_path = path_to_csv_file
+    end
+
+    def all_users
+      #Not rescuing here, as we should crash if we can not get userlist
+      @users ||= ::CSV.read(@user_csv_file_path).flatten.uniq 
+    end
+  end
+end

data/lib/gh_bb_audit/version.rb

@@ -0,0 +1,3 @@
+module GhBbAudit
+	VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: github_bitbucket_audit
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ankur Maheshwari
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: github_api
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: pry
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: The library takes a list of users and searches their public repos for specific keywords
+email:
+- amaheshwari@systango.com
+executables:
+- gh_bb_audit.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- README.md
+- bin/gh_bb_audit.rb
+- gh_bb_audit.gemspec
+- lib/gh_bb_audit.rb
+- lib/gh_bb_audit/github_repo.rb
+- lib/gh_bb_audit/github_scanner.rb
+- lib/gh_bb_audit/github_user.rb
+- lib/gh_bb_audit/keyword_matcher.rb
+- lib/gh_bb_audit/keywords_list.rb
+- lib/gh_bb_audit/output_writer.rb
+- lib/gh_bb_audit/users_list.rb
+- lib/gh_bb_audit/version.rb
+homepage: http://systango.com/
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Looks for specific keywords in the public repos of github and bitbucket
+test_files: []