github_bitbucket_audit 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1e33467e1124a6760aa67b0b204687b674e93e1
-  data.tar.gz: 7d865ce52fbfa86c425ca0712561578333aed3f5
+  metadata.gz: 2558d6a2895876f862a2bce2cb1356279d05bb17
+  data.tar.gz: be6dad5c46676002bba3b5756902aded416902ad
 SHA512:
-  metadata.gz: a89ca8927d17c239892fbfdf4999d46a8ea9293c44f0578dea9f32de0007ec46a40703dccadd4f5eba03dd140dcc29a2013e9a405a3e3367bee9bde221eea2dc
-  data.tar.gz: 8ece21087c45722a2ba070832838f039ea80fb642cf2283bd165e9eadf3e7c379f4b61140b1343efa4afa83f7787c59a48e6fa4666ebd33bece5d7c93f5c65a1
+  metadata.gz: 992ca3fcb436edbc2c10f62570d6f95ed2a36e53b0ec5a582265a5790c56b05901edf198ab051b31197a4b74c074896f5ae4f0bd039c857c644a6d37ea5c3d68
+  data.tar.gz: 768485bc401fc99b7bb7a9761821c642cc498eda8971cd07ce5d032ea858472fa62534911c3cb262ab69cf7140c70e847fcfa7a4e5ac85d8acf6ebc786f197d9

data/Gemfile

@@ -1,3 +1,5 @@
 source "http://rubygems.org"
 
+gem 'bitbucket_rest_api', :git => 'git://github.com/stpl/bitbucket.git'
+
 gemspec

data/Gemfile.lock

@@ -1,20 +1,34 @@
+GIT
+  remote: git://github.com/stpl/bitbucket.git
+  revision: 93289813cbc860031939e4335b00ba082a2afb5e
+  specs:
+    bitbucket_rest_api (0.1.6)
+      faraday (~> 0.8.1)
+      faraday_middleware (~> 0.9.0)
+      hashie (~> 2.0.5)
+      multi_json (~> 1.3)
+      nokogiri (>= 1.5.2)
+      simple_oauth
+
 PATH
   remote: .
   specs:
-    github_bitbucket_audit (0.0.1)
+    github_bitbucket_audit (0.0.3)
+      bitbucket_rest_api (~> 0.1.6)
       github_api (~> 0.11.3)
       semantic_logger (~> 2.7.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    addressable (2.3.5)
-    atomic (1.1.16-java)
+    addressable (2.3.6)
     coderay (1.1.0)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
-    faraday (0.9.0)
-      multipart-post (>= 1.2, < 3)
+    faraday (0.8.9)
+      multipart-post (~> 1.2.0)
+    faraday_middleware (0.9.0)
+      faraday (>= 0.7.4, < 0.9)
     ffi (1.9.3-java)
     github_api (0.11.3)
       addressable (~> 2.3)
@@ -31,7 +45,7 @@ GEM
     mini_portile (0.5.3)
     multi_json (1.9.2)
     multi_xml (0.5.5)
-    multipart-post (2.0.0)
+    multipart-post (1.2.0)
     nokogiri (1.6.1-java)
       mini_portile (~> 0.5.0)
     oauth2 (0.9.3)
@@ -49,16 +63,17 @@ GEM
     semantic_logger (2.7.0)
       sync_attr (>= 1.0)
       thread_safe (>= 0.1.0)
+    simple_oauth (0.2.0)
     slop (3.5.0)
     spoon (0.0.4)
       ffi
     sync_attr (1.0.0)
-    thread_safe (0.3.1-java)
-      atomic (>= 1.1.7, < 2)
+    thread_safe (0.3.2-java)
 
 PLATFORMS
   java
 
 DEPENDENCIES
+  bitbucket_rest_api!
   github_bitbucket_audit!
   pry

data/bin/gh_bb_audit.rb

@@ -28,4 +28,4 @@ OptionParser.new do |opts|
   end
 end.parse!
 
-::GhBbAudit::GithubScanner.new(options[:user_file_path], options[:keywords_file_path], options[:output_file_path], options[:github_user], options[:github_password]).start_scan
+::GhBbAudit::RepoScanner.new(options).start_scan

data/gh_bb_audit.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "github_api", "~> 0.11.3"
+  spec.add_dependency "bitbucket_rest_api", "~> 0.1.6"
   spec.add_dependency 'semantic_logger', '~> 2.7.0'
 	
   spec.add_development_dependency "pry"

data/lib/gh_bb_audit.rb

@@ -1,5 +1,7 @@
 require 'semantic_logger' unless defined?(logger)
 require 'github_api'
+require 'bitbucket_rest_api'
+require 'pry'
 
 SemanticLogger.default_level = :info
 SemanticLogger.add_appender('gh_bb_audit.log')
@@ -8,12 +10,16 @@ module GhBbAudit
   GEM_ROOT = File.join(File.dirname(__FILE__), '..')
 end
 
-require_relative './gh_bb_audit/version'
-require_relative './gh_bb_audit/github_user'
-require_relative './gh_bb_audit/github_repo'
-require_relative './gh_bb_audit/github_scanner'
+require_relative './gh_bb_audit/github_host'
+require_relative './gh_bb_audit/bitbucket_host'
+require_relative './gh_bb_audit/github/github_user'
+require_relative './gh_bb_audit/github/github_repo'
+require_relative './gh_bb_audit/github/github_api'
+require_relative './gh_bb_audit/bitbucket/bitbucket_user'
+require_relative './gh_bb_audit/bitbucket/bitbucket_repo'
+require_relative './gh_bb_audit/repo_scanner'
+require_relative './gh_bb_audit/users_list'
 require_relative './gh_bb_audit/keywords_list'
 require_relative './gh_bb_audit/keyword_matcher'
-require_relative './gh_bb_audit/users_list'
-require_relative './gh_bb_audit/github_api'
 require_relative './gh_bb_audit/output_writer'
+require_relative './gh_bb_audit/version'

data/lib/gh_bb_audit/bitbucket/bitbucket_repo.rb

@@ -0,0 +1,42 @@
+require 'semantic_logger'
+
+module GhBbAudit
+  module Bitbucket
+    class BitbucketRepo
+      include SemanticLogger::Loggable
+
+      def initialize(user_name,repo_name)
+        @user_name = user_name
+        @repo_name = repo_name
+      end
+
+      def get_all_file_paths
+        return [] if ( !@user_name || !@repo_name )
+        begin
+          bb_client = ::BitBucket.new {|config| config.endpoint = 'https://bitbucket.org/api/1.0'} 
+          @paths ||= get_files_in_dir('',bb_client)
+        rescue StandardError => e
+          logger.error "BITBUCKET:: Error in geting files for Bitbucket Repo:: #{@repo_name} for User:: #{@user_name}", error: e.inspect
+        end
+        @paths
+      end
+
+      private
+
+      def get_files_in_dir(path,bb_client)
+        bb_response = bb_client.repos.sources.get @user_name, @repo_name, 'master', path
+        files = bb_response["files"].collect(&:path) rescue []
+        files_in_directory = begin
+          bb_response["directories"].each.inject([]) do |result,dir_name|
+            result << get_files_in_dir(path + '/' + dir_name, bb_client)
+          end
+        rescue StandardError => e
+          logger.error "BITBUCKET:: Error in geting files for Bitbucket Repo:: #{@repo_name} for User:: #{@user_name} and path:: #{path}", error: e.inspect
+          []
+        end
+        return (files + files_in_directory).flatten
+      end
+
+    end
+  end
+end

data/lib/gh_bb_audit/bitbucket/bitbucket_user.rb

@@ -0,0 +1,23 @@
+require 'semantic_logger'
+
+module GhBbAudit
+  module Bitbucket
+    class BitbucketUser
+      include SemanticLogger::Loggable
+
+    	def initialize(user_name)
+    	  @user_name = user_name
+      end
+
+      def public_repos
+        @public_repos ||= begin 
+            response = ::BitBucket.new {|config| config.endpoint   = 'https://bitbucket.org/api/2.0'}.repos.public @user_name 
+            response["values"].collect(&:name)
+    		rescue StandardError => e
+    		  logger.error "BITBUCKET:: Error in fetching repos for Bitbucket User:: #{@user_name} ", error: e.inspect
+    		  []
+    		end
+      end
+    end
+  end
+end

data/lib/gh_bb_audit/bitbucket_host.rb

@@ -0,0 +1,15 @@
+module GhBbAudit
+	class BitbucketHost
+		def self.user(user_name)
+			::GhBbAudit::Bitbucket::BitbucketUser.new(user_name)
+		end
+
+		def self.repo(user_name,repo_name)
+			::GhBbAudit::Bitbucket::BitbucketRepo.new(user_name,repo_name)
+		end
+
+		def self.name
+      "BITBUCKET"
+    end
+  end
+end

data/lib/gh_bb_audit/github/github_api.rb

@@ -0,0 +1,24 @@
+require 'semantic_logger'
+
+module GhBbAudit
+  module Github
+  	class GithubApi
+      include SemanticLogger::Loggable
+
+      def self.set_user_name_pwd(user_name, password)
+        @user_name = user_name
+        @password = password
+      end
+
+      def self.get_api_accesor
+        if !!@user_name && !!@password
+          logger.info("Using Authentication for Github with Username:: #{@user_name}")
+          ::Github.new login: @user_name, password: @password
+        else
+          ::Github.new
+        end
+      end
+      
+  	end
+  end
+end

data/lib/gh_bb_audit/github/github_repo.rb

@@ -0,0 +1,48 @@
+require 'semantic_logger'
+
+module GhBbAudit
+  module Github
+    class GithubRepo
+      include SemanticLogger::Loggable
+      def initialize(user_name,repo_name)
+        @user_name = user_name
+        @repo_name = repo_name
+      end
+
+      def get_all_file_paths
+        return [] if ( !@user_name || !@repo_name )
+        @paths ||= begin
+          get_tree.each.inject([]) do |result, file|
+            result << file.path
+            result
+          end
+        rescue StandardError => e
+          []
+        end 
+        @paths
+      end
+
+      private
+      def get_tree
+        all_commits = begin 
+          GithubApi.get_api_accesor.repos.commits.all  @user_name, @repo_name
+          rescue StandardError => e
+            logger.error "GITHUB:: Error in fetching Commits for Repo::#{@repo_name} for User:: #{@user_name} ", error: e.inspect
+            return nil
+          end
+        last_commit_sha = all_commits[0].sha rescue nil
+        if !last_commit_sha
+          logger.info("GITHUB:: Could not find any commit for Github Repo:: #{@repo_name} for User:: #{@user_name}")
+          return nil
+        end
+        @github_tree ||= GithubApi.get_api_accesor.git_data.trees.get @user_name, @repo_name, last_commit_sha , 'recursive' => true rescue nil
+        if !@github_tree
+          logger.error("GITHUB:: Could not fetch tree for Repo:: #{@repo_name} for User:: #{@user_name}")
+          return 
+        end
+        @github_tree.tree
+      end
+
+    end
+  end
+end

data/lib/gh_bb_audit/github/github_user.rb

@@ -0,0 +1,22 @@
+require 'semantic_logger'
+
+module GhBbAudit
+  module Github
+    class GithubUser
+      include SemanticLogger::Loggable
+
+    	def initialize(user_name)
+    	  @user_name = user_name
+      end
+
+      def public_repos
+        @public_repos ||= begin 
+        	(GithubApi.get_api_accesor.repos.list user: @user_name).collect(&:name)
+  		rescue StandardError => e
+  		  logger.error "GITHUB:: Error in fetching repos for User:: #{@user_name} ", error: e.inspect
+  		  []
+  		end
+      end
+    end
+  end
+end

data/lib/gh_bb_audit/github_host.rb

@@ -0,0 +1,15 @@
+module GhBbAudit
+	class GithubHost
+    def self.user(user_name)
+      ::GhBbAudit::Github::GithubUser.new(user_name)
+    end
+
+    def self.repo(user_name,repo_name)
+      ::GhBbAudit::Github::GithubRepo.new(user_name,repo_name)
+    end
+
+    def self.name
+      "GITHUB"
+    end
+  end
+end

data/lib/gh_bb_audit/output_writer.rb

@@ -4,15 +4,15 @@ module GhBbAudit
     	@fhandle = File.open(path_to_file, 'w+') 
     end
 
-    def repo_name_matched(repo_name,user_name)
+    def repo_name_matched(repo_name,user_name,repo_source)
       @fhandle.puts("")
-      @fhandle.puts("The name of REPO::#{repo_name} for USER::#{user_name} matches keywords")
+      @fhandle.puts("#{repo_source}:: The name of REPO:#{repo_name} for USER:#{user_name} matches keywords")
       @fhandle.puts("")
     end
 
-    def file_paths_matched_in_repo(file_path_array,repo_name,user_name)
+    def file_paths_matched_in_repo(file_path_array,repo_name,user_name,repo_source)
       @fhandle.puts("")
-      @fhandle.puts("For the REPO::#{repo_name} for USER::#{user_name}, the following file paths matched")
+      @fhandle.puts("#{repo_source}:: For the REPO:#{repo_name} for USER:#{user_name}, the following file paths matched")
       file_path_array.each do |file_path|
         @fhandle.puts("---- #{file_path}")
       end

data/lib/gh_bb_audit/repo_scanner.rb

@@ -0,0 +1,46 @@
+require 'semantic_logger'
+module GhBbAudit
+	class RepoScanner
+    include SemanticLogger::Loggable
+
+    def initialize(options)
+      @user_csv_path = options[:user_file_path]
+      @keyword_csv_path = options[:keywords_file_path]
+      @output_file_path = options[:output_file_path]
+      GithubApi.set_user_name_pwd(options[:github_user], options[:github_password]) if options.include?(:github_user) && options.include?(:github_password)
+
+      @matcher = ::GhBbAudit::KeywordMatcher.new(::GhBbAudit::KeywordsList.new(@keyword_csv_path).all_keywords)
+      @users = ::GhBbAudit::UsersList.new(@user_csv_path).all_users
+      @output_writer = ::GhBbAudit::OutputWriter.new(@output_file_path)
+    end
+
+    def start_scan
+      begin
+        [::GhBbAudit::GithubHost, ::GhBbAudit::BitbucketHost].each do |repo_source|
+          @users.each do |user|
+            logger.info("#{repo_source.name}:: Scanning for User: #{user}")
+            repo_source.user(user).public_repos.each do |public_repo|
+              logger.info("#{repo_source.name}:: Scanning Repo: #{public_repo} for User: #{user}")
+              if @matcher.repo_contains_keyword?([public_repo])
+                @output_writer.repo_name_matched(public_repo,user, repo_source.name)
+              end
+              
+              file_paths = repo_source.repo(user,public_repo).get_all_file_paths
+
+              if @matcher.repo_contains_keyword?(file_paths)
+                @output_writer.file_paths_matched_in_repo(@matcher.matched_file_paths(file_paths),public_repo, user, repo_source.name)
+              end
+            end
+          end
+        end
+        
+      rescue StandardError => e
+        logger.error "Error in scanning", error: e.inspect
+      ensure
+        @output_writer.close
+      end
+    end
+
+	end
+
+end

data/lib/gh_bb_audit/version.rb

@@ -1,3 +1,3 @@
 module GhBbAudit
-	VERSION = "0.0.3"
+	VERSION = "0.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: github_bitbucket_audit
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Ankur Maheshwari
@@ -24,6 +24,20 @@ dependencies:
         version: 0.11.3
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: bitbucket_rest_api
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   version_requirements: !ruby/object:Gem::Requirement
@@ -67,13 +81,17 @@ files:
 - bin/gh_bb_audit.rb
 - gh_bb_audit.gemspec
 - lib/gh_bb_audit.rb
-- lib/gh_bb_audit/github_api.rb
-- lib/gh_bb_audit/github_repo.rb
-- lib/gh_bb_audit/github_scanner.rb
-- lib/gh_bb_audit/github_user.rb
+- lib/gh_bb_audit/bitbucket/bitbucket_repo.rb
+- lib/gh_bb_audit/bitbucket/bitbucket_user.rb
+- lib/gh_bb_audit/bitbucket_host.rb
+- lib/gh_bb_audit/github/github_api.rb
+- lib/gh_bb_audit/github/github_repo.rb
+- lib/gh_bb_audit/github/github_user.rb
+- lib/gh_bb_audit/github_host.rb
 - lib/gh_bb_audit/keyword_matcher.rb
 - lib/gh_bb_audit/keywords_list.rb
 - lib/gh_bb_audit/output_writer.rb
+- lib/gh_bb_audit/repo_scanner.rb
 - lib/gh_bb_audit/users_list.rb
 - lib/gh_bb_audit/version.rb
 homepage: http://systango.com/

data/lib/gh_bb_audit/github_api.rb

@@ -1,22 +0,0 @@
-require 'semantic_logger'
-
-module GhBbAudit
-	class GithubApi
-    include SemanticLogger::Loggable
-
-    def self.set_user_name_pwd(user_name, password)
-      @user_name = user_name
-      @password = password
-    end
-
-    def self.get_api_accesor
-      if !!@user_name && !!@password
-        logger.info("Using Authentication for Github with Username:: #{@user_name}")
-        Github.new login: @user_name, password: @password
-      else
-        Github.new
-      end
-    end
-    
-	end
-end

data/lib/gh_bb_audit/github_repo.rb

@@ -1,46 +0,0 @@
-require 'semantic_logger'
-
-module GhBbAudit
-  class GithubRepo
-    include SemanticLogger::Loggable
-    def initialize(user_name,repo_name)
-      @user_name = user_name
-      @repo_name = repo_name
-    end
-
-    def get_all_file_paths
-      return [] if ( !@user_name || !@repo_name )
-      @paths ||= begin
-        get_tree.each.inject([]) do |result, file|
-          result << file.path
-          result
-        end
-      rescue Exception => e
-        []
-      end 
-      @paths
-    end
-
-    private
-    def get_tree
-      all_commits = begin 
-          GithubApi.get_api_accesor.repos.commits.all  @user_name, @repo_name
-        rescue StandardError => e
-          logger.error "Error in fetching Commits for Repo::#{@repo_name} for User:: #{@user_name} ", error: e.inspect
-          return nil
-        end
-      last_commit_sha = all_commits[0].sha rescue nil
-      if !last_commit_sha
-        logger.info("Could not find any commit for Github Repo:: #{@repo_name} for User:: #{@user_name}")
-        return nil
-      end
-      @github_tree ||= GithubApi.get_api_accesor.git_data.trees.get @user_name, @repo_name, last_commit_sha , 'recursive' => true rescue nil
-      if !@github_tree
-        logger.error("Could not fetch tree for Repo:: #{@repo_name} for User:: #{@user_name}")
-        return 
-      end
-      @github_tree.tree
-    end
-
-  end
-end

data/lib/gh_bb_audit/github_scanner.rb

@@ -1,44 +0,0 @@
-require 'semantic_logger'
-module GhBbAudit
-	class GithubScanner
-    include SemanticLogger::Loggable
-
-    def initialize(user_csv_path, keyword_csv_path, output_file_path, github_user =nil, github_password =nil)
-      @user_csv_path = user_csv_path
-      @keyword_csv_path = keyword_csv_path
-      @output_file_path = output_file_path
-      GithubApi.set_user_name_pwd(github_user, github_password) if github_user && github_password
-    end
-
-    def start_scan
-      output_writer = ::GhBbAudit::OutputWriter.new(@output_file_path)
-      begin
-        matcher = ::GhBbAudit::KeywordMatcher.new(::GhBbAudit::KeywordsList.new(@keyword_csv_path).all_keywords)
-        all_github_user = ::GhBbAudit::UsersList.new(@user_csv_path).all_users
-        
-        all_github_user.each do |user|
-          logger.info("Scanning for User:: #{user}")
-          ::GhBbAudit::GithubUser.new(user).public_repos.each do |public_repo|
-            logger.info("Scanning Repo:: #{public_repo.name} for User:: #{user}")
-            if matcher.repo_contains_keyword?([public_repo.name])
-              output_writer.repo_name_matched(public_repo.name,user)
-            end
-            
-            file_paths = ::GhBbAudit::GithubRepo.new(user,public_repo.name).get_all_file_paths
-
-            if matcher.repo_contains_keyword?(file_paths)
-              output_writer.file_paths_matched_in_repo(matcher.matched_file_paths(file_paths),public_repo.name, user)
-            end
-          end
-        end
-      rescue StandardError => e
-        logger.error "Error in scanning Github ", error: e.inspect
-      ensure
-        output_writer.close
-      end
-    end
-
-
-	end
-
-end

data/lib/gh_bb_audit/github_user.rb

@@ -1,20 +0,0 @@
-require 'semantic_logger'
-
-module GhBbAudit
-  class GithubUser
-    include SemanticLogger::Loggable
-
-  	def initialize(user_name)
-  	  @user_name = user_name
-    end
-
-    def public_repos
-      @public_repos ||= begin 
-      	GithubApi.get_api_accesor.repos.list user: @user_name 
-		rescue StandardError => e
-		  logger.error "Error in fetching repos for User:: #{@user_name} ", error: e.inspect
-		  []
-		end
-    end
-  end
-end