github_bitbucket_audit 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a1e33467e1124a6760aa67b0b204687b674e93e1
4
- data.tar.gz: 7d865ce52fbfa86c425ca0712561578333aed3f5
3
+ metadata.gz: 2558d6a2895876f862a2bce2cb1356279d05bb17
4
+ data.tar.gz: be6dad5c46676002bba3b5756902aded416902ad
5
5
  SHA512:
6
- metadata.gz: a89ca8927d17c239892fbfdf4999d46a8ea9293c44f0578dea9f32de0007ec46a40703dccadd4f5eba03dd140dcc29a2013e9a405a3e3367bee9bde221eea2dc
7
- data.tar.gz: 8ece21087c45722a2ba070832838f039ea80fb642cf2283bd165e9eadf3e7c379f4b61140b1343efa4afa83f7787c59a48e6fa4666ebd33bece5d7c93f5c65a1
6
+ metadata.gz: 992ca3fcb436edbc2c10f62570d6f95ed2a36e53b0ec5a582265a5790c56b05901edf198ab051b31197a4b74c074896f5ae4f0bd039c857c644a6d37ea5c3d68
7
+ data.tar.gz: 768485bc401fc99b7bb7a9761821c642cc498eda8971cd07ce5d032ea858472fa62534911c3cb262ab69cf7140c70e847fcfa7a4e5ac85d8acf6ebc786f197d9
data/Gemfile CHANGED
@@ -1,3 +1,5 @@
1
1
  source "http://rubygems.org"
2
2
 
3
+ gem 'bitbucket_rest_api', :git => 'git://github.com/stpl/bitbucket.git'
4
+
3
5
  gemspec
@@ -1,20 +1,34 @@
1
+ GIT
2
+ remote: git://github.com/stpl/bitbucket.git
3
+ revision: 93289813cbc860031939e4335b00ba082a2afb5e
4
+ specs:
5
+ bitbucket_rest_api (0.1.6)
6
+ faraday (~> 0.8.1)
7
+ faraday_middleware (~> 0.9.0)
8
+ hashie (~> 2.0.5)
9
+ multi_json (~> 1.3)
10
+ nokogiri (>= 1.5.2)
11
+ simple_oauth
12
+
1
13
  PATH
2
14
  remote: .
3
15
  specs:
4
- github_bitbucket_audit (0.0.1)
16
+ github_bitbucket_audit (0.0.3)
17
+ bitbucket_rest_api (~> 0.1.6)
5
18
  github_api (~> 0.11.3)
6
19
  semantic_logger (~> 2.7.0)
7
20
 
8
21
  GEM
9
22
  remote: http://rubygems.org/
10
23
  specs:
11
- addressable (2.3.5)
12
- atomic (1.1.16-java)
24
+ addressable (2.3.6)
13
25
  coderay (1.1.0)
14
26
  descendants_tracker (0.0.4)
15
27
  thread_safe (~> 0.3, >= 0.3.1)
16
- faraday (0.9.0)
17
- multipart-post (>= 1.2, < 3)
28
+ faraday (0.8.9)
29
+ multipart-post (~> 1.2.0)
30
+ faraday_middleware (0.9.0)
31
+ faraday (>= 0.7.4, < 0.9)
18
32
  ffi (1.9.3-java)
19
33
  github_api (0.11.3)
20
34
  addressable (~> 2.3)
@@ -31,7 +45,7 @@ GEM
31
45
  mini_portile (0.5.3)
32
46
  multi_json (1.9.2)
33
47
  multi_xml (0.5.5)
34
- multipart-post (2.0.0)
48
+ multipart-post (1.2.0)
35
49
  nokogiri (1.6.1-java)
36
50
  mini_portile (~> 0.5.0)
37
51
  oauth2 (0.9.3)
@@ -49,16 +63,17 @@ GEM
49
63
  semantic_logger (2.7.0)
50
64
  sync_attr (>= 1.0)
51
65
  thread_safe (>= 0.1.0)
66
+ simple_oauth (0.2.0)
52
67
  slop (3.5.0)
53
68
  spoon (0.0.4)
54
69
  ffi
55
70
  sync_attr (1.0.0)
56
- thread_safe (0.3.1-java)
57
- atomic (>= 1.1.7, < 2)
71
+ thread_safe (0.3.2-java)
58
72
 
59
73
  PLATFORMS
60
74
  java
61
75
 
62
76
  DEPENDENCIES
77
+ bitbucket_rest_api!
63
78
  github_bitbucket_audit!
64
79
  pry
@@ -28,4 +28,4 @@ OptionParser.new do |opts|
28
28
  end
29
29
  end.parse!
30
30
 
31
- ::GhBbAudit::GithubScanner.new(options[:user_file_path], options[:keywords_file_path], options[:output_file_path], options[:github_user], options[:github_password]).start_scan
31
+ ::GhBbAudit::RepoScanner.new(options).start_scan
@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
19
19
  spec.require_paths = ["lib"]
20
20
 
21
21
  spec.add_dependency "github_api", "~> 0.11.3"
22
+ spec.add_dependency "bitbucket_rest_api", "~> 0.1.6"
22
23
  spec.add_dependency 'semantic_logger', '~> 2.7.0'
23
24
 
24
25
  spec.add_development_dependency "pry"
@@ -1,5 +1,7 @@
1
1
  require 'semantic_logger' unless defined?(logger)
2
2
  require 'github_api'
3
+ require 'bitbucket_rest_api'
4
+ require 'pry'
3
5
 
4
6
  SemanticLogger.default_level = :info
5
7
  SemanticLogger.add_appender('gh_bb_audit.log')
@@ -8,12 +10,16 @@ module GhBbAudit
8
10
  GEM_ROOT = File.join(File.dirname(__FILE__), '..')
9
11
  end
10
12
 
11
- require_relative './gh_bb_audit/version'
12
- require_relative './gh_bb_audit/github_user'
13
- require_relative './gh_bb_audit/github_repo'
14
- require_relative './gh_bb_audit/github_scanner'
13
+ require_relative './gh_bb_audit/github_host'
14
+ require_relative './gh_bb_audit/bitbucket_host'
15
+ require_relative './gh_bb_audit/github/github_user'
16
+ require_relative './gh_bb_audit/github/github_repo'
17
+ require_relative './gh_bb_audit/github/github_api'
18
+ require_relative './gh_bb_audit/bitbucket/bitbucket_user'
19
+ require_relative './gh_bb_audit/bitbucket/bitbucket_repo'
20
+ require_relative './gh_bb_audit/repo_scanner'
21
+ require_relative './gh_bb_audit/users_list'
15
22
  require_relative './gh_bb_audit/keywords_list'
16
23
  require_relative './gh_bb_audit/keyword_matcher'
17
- require_relative './gh_bb_audit/users_list'
18
- require_relative './gh_bb_audit/github_api'
19
24
  require_relative './gh_bb_audit/output_writer'
25
+ require_relative './gh_bb_audit/version'
@@ -0,0 +1,42 @@
1
+ require 'semantic_logger'
2
+
3
+ module GhBbAudit
4
+ module Bitbucket
5
+ class BitbucketRepo
6
+ include SemanticLogger::Loggable
7
+
8
+ def initialize(user_name,repo_name)
9
+ @user_name = user_name
10
+ @repo_name = repo_name
11
+ end
12
+
13
+ def get_all_file_paths
14
+ return [] if ( !@user_name || !@repo_name )
15
+ begin
16
+ bb_client = ::BitBucket.new {|config| config.endpoint = 'https://bitbucket.org/api/1.0'}
17
+ @paths ||= get_files_in_dir('',bb_client)
18
+ rescue StandardError => e
19
+ logger.error "BITBUCKET:: Error in geting files for Bitbucket Repo:: #{@repo_name} for User:: #{@user_name}", error: e.inspect
20
+ end
21
+ @paths
22
+ end
23
+
24
+ private
25
+
26
+ def get_files_in_dir(path,bb_client)
27
+ bb_response = bb_client.repos.sources.get @user_name, @repo_name, 'master', path
28
+ files = bb_response["files"].collect(&:path) rescue []
29
+ files_in_directory = begin
30
+ bb_response["directories"].each.inject([]) do |result,dir_name|
31
+ result << get_files_in_dir(path + '/' + dir_name, bb_client)
32
+ end
33
+ rescue StandardError => e
34
+ logger.error "BITBUCKET:: Error in geting files for Bitbucket Repo:: #{@repo_name} for User:: #{@user_name} and path:: #{path}", error: e.inspect
35
+ []
36
+ end
37
+ return (files + files_in_directory).flatten
38
+ end
39
+
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,23 @@
1
+ require 'semantic_logger'
2
+
3
+ module GhBbAudit
4
+ module Bitbucket
5
+ class BitbucketUser
6
+ include SemanticLogger::Loggable
7
+
8
+ def initialize(user_name)
9
+ @user_name = user_name
10
+ end
11
+
12
+ def public_repos
13
+ @public_repos ||= begin
14
+ response = ::BitBucket.new {|config| config.endpoint = 'https://bitbucket.org/api/2.0'}.repos.public @user_name
15
+ response["values"].collect(&:name)
16
+ rescue StandardError => e
17
+ logger.error "BITBUCKET:: Error in fetching repos for Bitbucket User:: #{@user_name} ", error: e.inspect
18
+ []
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,15 @@
1
+ module GhBbAudit
2
+ class BitbucketHost
3
+ def self.user(user_name)
4
+ ::GhBbAudit::Bitbucket::BitbucketUser.new(user_name)
5
+ end
6
+
7
+ def self.repo(user_name,repo_name)
8
+ ::GhBbAudit::Bitbucket::BitbucketRepo.new(user_name,repo_name)
9
+ end
10
+
11
+ def self.name
12
+ "BITBUCKET"
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,24 @@
1
+ require 'semantic_logger'
2
+
3
+ module GhBbAudit
4
+ module Github
5
+ class GithubApi
6
+ include SemanticLogger::Loggable
7
+
8
+ def self.set_user_name_pwd(user_name, password)
9
+ @user_name = user_name
10
+ @password = password
11
+ end
12
+
13
+ def self.get_api_accesor
14
+ if !!@user_name && !!@password
15
+ logger.info("Using Authentication for Github with Username:: #{@user_name}")
16
+ ::Github.new login: @user_name, password: @password
17
+ else
18
+ ::Github.new
19
+ end
20
+ end
21
+
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,48 @@
1
+ require 'semantic_logger'
2
+
3
+ module GhBbAudit
4
+ module Github
5
+ class GithubRepo
6
+ include SemanticLogger::Loggable
7
+ def initialize(user_name,repo_name)
8
+ @user_name = user_name
9
+ @repo_name = repo_name
10
+ end
11
+
12
+ def get_all_file_paths
13
+ return [] if ( !@user_name || !@repo_name )
14
+ @paths ||= begin
15
+ get_tree.each.inject([]) do |result, file|
16
+ result << file.path
17
+ result
18
+ end
19
+ rescue StandardError => e
20
+ []
21
+ end
22
+ @paths
23
+ end
24
+
25
+ private
26
+ def get_tree
27
+ all_commits = begin
28
+ GithubApi.get_api_accesor.repos.commits.all @user_name, @repo_name
29
+ rescue StandardError => e
30
+ logger.error "GITHUB:: Error in fetching Commits for Repo::#{@repo_name} for User:: #{@user_name} ", error: e.inspect
31
+ return nil
32
+ end
33
+ last_commit_sha = all_commits[0].sha rescue nil
34
+ if !last_commit_sha
35
+ logger.info("GITHUB:: Could not find any commit for Github Repo:: #{@repo_name} for User:: #{@user_name}")
36
+ return nil
37
+ end
38
+ @github_tree ||= GithubApi.get_api_accesor.git_data.trees.get @user_name, @repo_name, last_commit_sha , 'recursive' => true rescue nil
39
+ if !@github_tree
40
+ logger.error("GITHUB:: Could not fetch tree for Repo:: #{@repo_name} for User:: #{@user_name}")
41
+ return
42
+ end
43
+ @github_tree.tree
44
+ end
45
+
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,22 @@
1
+ require 'semantic_logger'
2
+
3
+ module GhBbAudit
4
+ module Github
5
+ class GithubUser
6
+ include SemanticLogger::Loggable
7
+
8
+ def initialize(user_name)
9
+ @user_name = user_name
10
+ end
11
+
12
+ def public_repos
13
+ @public_repos ||= begin
14
+ (GithubApi.get_api_accesor.repos.list user: @user_name).collect(&:name)
15
+ rescue StandardError => e
16
+ logger.error "GITHUB:: Error in fetching repos for User:: #{@user_name} ", error: e.inspect
17
+ []
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,15 @@
1
+ module GhBbAudit
2
+ class GithubHost
3
+ def self.user(user_name)
4
+ ::GhBbAudit::Github::GithubUser.new(user_name)
5
+ end
6
+
7
+ def self.repo(user_name,repo_name)
8
+ ::GhBbAudit::Github::GithubRepo.new(user_name,repo_name)
9
+ end
10
+
11
+ def self.name
12
+ "GITHUB"
13
+ end
14
+ end
15
+ end
@@ -4,15 +4,15 @@ module GhBbAudit
4
4
  @fhandle = File.open(path_to_file, 'w+')
5
5
  end
6
6
 
7
- def repo_name_matched(repo_name,user_name)
7
+ def repo_name_matched(repo_name,user_name,repo_source)
8
8
  @fhandle.puts("")
9
- @fhandle.puts("The name of REPO::#{repo_name} for USER::#{user_name} matches keywords")
9
+ @fhandle.puts("#{repo_source}:: The name of REPO:#{repo_name} for USER:#{user_name} matches keywords")
10
10
  @fhandle.puts("")
11
11
  end
12
12
 
13
- def file_paths_matched_in_repo(file_path_array,repo_name,user_name)
13
+ def file_paths_matched_in_repo(file_path_array,repo_name,user_name,repo_source)
14
14
  @fhandle.puts("")
15
- @fhandle.puts("For the REPO::#{repo_name} for USER::#{user_name}, the following file paths matched")
15
+ @fhandle.puts("#{repo_source}:: For the REPO:#{repo_name} for USER:#{user_name}, the following file paths matched")
16
16
  file_path_array.each do |file_path|
17
17
  @fhandle.puts("---- #{file_path}")
18
18
  end
@@ -0,0 +1,46 @@
1
+ require 'semantic_logger'
2
+ module GhBbAudit
3
+ class RepoScanner
4
+ include SemanticLogger::Loggable
5
+
6
+ def initialize(options)
7
+ @user_csv_path = options[:user_file_path]
8
+ @keyword_csv_path = options[:keywords_file_path]
9
+ @output_file_path = options[:output_file_path]
10
+ GithubApi.set_user_name_pwd(options[:github_user], options[:github_password]) if options.include?(:github_user) && options.include?(:github_password)
11
+
12
+ @matcher = ::GhBbAudit::KeywordMatcher.new(::GhBbAudit::KeywordsList.new(@keyword_csv_path).all_keywords)
13
+ @users = ::GhBbAudit::UsersList.new(@user_csv_path).all_users
14
+ @output_writer = ::GhBbAudit::OutputWriter.new(@output_file_path)
15
+ end
16
+
17
+ def start_scan
18
+ begin
19
+ [::GhBbAudit::GithubHost, ::GhBbAudit::BitbucketHost].each do |repo_source|
20
+ @users.each do |user|
21
+ logger.info("#{repo_source.name}:: Scanning for User: #{user}")
22
+ repo_source.user(user).public_repos.each do |public_repo|
23
+ logger.info("#{repo_source.name}:: Scanning Repo: #{public_repo} for User: #{user}")
24
+ if @matcher.repo_contains_keyword?([public_repo])
25
+ @output_writer.repo_name_matched(public_repo,user, repo_source.name)
26
+ end
27
+
28
+ file_paths = repo_source.repo(user,public_repo).get_all_file_paths
29
+
30
+ if @matcher.repo_contains_keyword?(file_paths)
31
+ @output_writer.file_paths_matched_in_repo(@matcher.matched_file_paths(file_paths),public_repo, user, repo_source.name)
32
+ end
33
+ end
34
+ end
35
+ end
36
+
37
+ rescue StandardError => e
38
+ logger.error "Error in scanning", error: e.inspect
39
+ ensure
40
+ @output_writer.close
41
+ end
42
+ end
43
+
44
+ end
45
+
46
+ end
@@ -1,3 +1,3 @@
1
1
  module GhBbAudit
2
- VERSION = "0.0.3"
2
+ VERSION = "0.0.4"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: github_bitbucket_audit
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ankur Maheshwari
@@ -24,6 +24,20 @@ dependencies:
24
24
  version: 0.11.3
25
25
  prerelease: false
26
26
  type: :runtime
27
+ - !ruby/object:Gem::Dependency
28
+ name: bitbucket_rest_api
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: 0.1.6
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ~>
37
+ - !ruby/object:Gem::Version
38
+ version: 0.1.6
39
+ prerelease: false
40
+ type: :runtime
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: semantic_logger
29
43
  version_requirements: !ruby/object:Gem::Requirement
@@ -67,13 +81,17 @@ files:
67
81
  - bin/gh_bb_audit.rb
68
82
  - gh_bb_audit.gemspec
69
83
  - lib/gh_bb_audit.rb
70
- - lib/gh_bb_audit/github_api.rb
71
- - lib/gh_bb_audit/github_repo.rb
72
- - lib/gh_bb_audit/github_scanner.rb
73
- - lib/gh_bb_audit/github_user.rb
84
+ - lib/gh_bb_audit/bitbucket/bitbucket_repo.rb
85
+ - lib/gh_bb_audit/bitbucket/bitbucket_user.rb
86
+ - lib/gh_bb_audit/bitbucket_host.rb
87
+ - lib/gh_bb_audit/github/github_api.rb
88
+ - lib/gh_bb_audit/github/github_repo.rb
89
+ - lib/gh_bb_audit/github/github_user.rb
90
+ - lib/gh_bb_audit/github_host.rb
74
91
  - lib/gh_bb_audit/keyword_matcher.rb
75
92
  - lib/gh_bb_audit/keywords_list.rb
76
93
  - lib/gh_bb_audit/output_writer.rb
94
+ - lib/gh_bb_audit/repo_scanner.rb
77
95
  - lib/gh_bb_audit/users_list.rb
78
96
  - lib/gh_bb_audit/version.rb
79
97
  homepage: http://systango.com/
@@ -1,22 +0,0 @@
1
- require 'semantic_logger'
2
-
3
- module GhBbAudit
4
- class GithubApi
5
- include SemanticLogger::Loggable
6
-
7
- def self.set_user_name_pwd(user_name, password)
8
- @user_name = user_name
9
- @password = password
10
- end
11
-
12
- def self.get_api_accesor
13
- if !!@user_name && !!@password
14
- logger.info("Using Authentication for Github with Username:: #{@user_name}")
15
- Github.new login: @user_name, password: @password
16
- else
17
- Github.new
18
- end
19
- end
20
-
21
- end
22
- end
@@ -1,46 +0,0 @@
1
- require 'semantic_logger'
2
-
3
- module GhBbAudit
4
- class GithubRepo
5
- include SemanticLogger::Loggable
6
- def initialize(user_name,repo_name)
7
- @user_name = user_name
8
- @repo_name = repo_name
9
- end
10
-
11
- def get_all_file_paths
12
- return [] if ( !@user_name || !@repo_name )
13
- @paths ||= begin
14
- get_tree.each.inject([]) do |result, file|
15
- result << file.path
16
- result
17
- end
18
- rescue Exception => e
19
- []
20
- end
21
- @paths
22
- end
23
-
24
- private
25
- def get_tree
26
- all_commits = begin
27
- GithubApi.get_api_accesor.repos.commits.all @user_name, @repo_name
28
- rescue StandardError => e
29
- logger.error "Error in fetching Commits for Repo::#{@repo_name} for User:: #{@user_name} ", error: e.inspect
30
- return nil
31
- end
32
- last_commit_sha = all_commits[0].sha rescue nil
33
- if !last_commit_sha
34
- logger.info("Could not find any commit for Github Repo:: #{@repo_name} for User:: #{@user_name}")
35
- return nil
36
- end
37
- @github_tree ||= GithubApi.get_api_accesor.git_data.trees.get @user_name, @repo_name, last_commit_sha , 'recursive' => true rescue nil
38
- if !@github_tree
39
- logger.error("Could not fetch tree for Repo:: #{@repo_name} for User:: #{@user_name}")
40
- return
41
- end
42
- @github_tree.tree
43
- end
44
-
45
- end
46
- end
@@ -1,44 +0,0 @@
1
- require 'semantic_logger'
2
- module GhBbAudit
3
- class GithubScanner
4
- include SemanticLogger::Loggable
5
-
6
- def initialize(user_csv_path, keyword_csv_path, output_file_path, github_user =nil, github_password =nil)
7
- @user_csv_path = user_csv_path
8
- @keyword_csv_path = keyword_csv_path
9
- @output_file_path = output_file_path
10
- GithubApi.set_user_name_pwd(github_user, github_password) if github_user && github_password
11
- end
12
-
13
- def start_scan
14
- output_writer = ::GhBbAudit::OutputWriter.new(@output_file_path)
15
- begin
16
- matcher = ::GhBbAudit::KeywordMatcher.new(::GhBbAudit::KeywordsList.new(@keyword_csv_path).all_keywords)
17
- all_github_user = ::GhBbAudit::UsersList.new(@user_csv_path).all_users
18
-
19
- all_github_user.each do |user|
20
- logger.info("Scanning for User:: #{user}")
21
- ::GhBbAudit::GithubUser.new(user).public_repos.each do |public_repo|
22
- logger.info("Scanning Repo:: #{public_repo.name} for User:: #{user}")
23
- if matcher.repo_contains_keyword?([public_repo.name])
24
- output_writer.repo_name_matched(public_repo.name,user)
25
- end
26
-
27
- file_paths = ::GhBbAudit::GithubRepo.new(user,public_repo.name).get_all_file_paths
28
-
29
- if matcher.repo_contains_keyword?(file_paths)
30
- output_writer.file_paths_matched_in_repo(matcher.matched_file_paths(file_paths),public_repo.name, user)
31
- end
32
- end
33
- end
34
- rescue StandardError => e
35
- logger.error "Error in scanning Github ", error: e.inspect
36
- ensure
37
- output_writer.close
38
- end
39
- end
40
-
41
-
42
- end
43
-
44
- end
@@ -1,20 +0,0 @@
1
- require 'semantic_logger'
2
-
3
- module GhBbAudit
4
- class GithubUser
5
- include SemanticLogger::Loggable
6
-
7
- def initialize(user_name)
8
- @user_name = user_name
9
- end
10
-
11
- def public_repos
12
- @public_repos ||= begin
13
- GithubApi.get_api_accesor.repos.list user: @user_name
14
- rescue StandardError => e
15
- logger.error "Error in fetching repos for User:: #{@user_name} ", error: e.inspect
16
- []
17
- end
18
- end
19
- end
20
- end