scopes_extractor 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/scopes_extractor/http_client.rb +38 -0
- data/lib/scopes_extractor/platforms/bugcrowd/cookie.rb +42 -0
- data/lib/scopes_extractor/platforms/bugcrowd/programs.rb +36 -0
- data/lib/scopes_extractor/platforms/bugcrowd/scopes.rb +45 -0
- data/lib/scopes_extractor/platforms/hackerone/programs.rb +43 -0
- data/lib/scopes_extractor/platforms/hackerone/scopes.rb +50 -0
- data/lib/scopes_extractor/platforms/intigriti/programs.rb +34 -0
- data/lib/scopes_extractor/platforms/intigriti/scopes.rb +46 -0
- data/lib/scopes_extractor/platforms/intigriti/token.rb +54 -0
- data/lib/scopes_extractor/platforms/yeswehack/jwt.rb +35 -0
- data/lib/scopes_extractor/platforms/yeswehack/programs.rb +42 -0
- data/lib/scopes_extractor/platforms/yeswehack/scopes.rb +61 -0
- data/lib/scopes_extractor/utilities.rb +37 -0
- data/lib/scopes_extractor.rb +60 -0
- metadata +171 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 694da3bb3d2fd3ed580db8d5afa0da7bac2c2326037ed4218b57d87eca7ca174
|
4
|
+
data.tar.gz: 100836692c414cd682653231a65a5937dc8b201560bbf10fa95e96e38d9b39a8
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1a3f2040079e8e05cfd2157b259f95b8912813f8f7ec2e867fd0ce9f7802a05a3b46c63027b5493a0d79a2b28f536b67686b9fb3b5d71926e70de31b846764ca
|
7
|
+
data.tar.gz: 53e92c8b6bcf0608c6c14d6e4f4733e29a8c86a930b7a25de1b6d5242c45d8e7eb38df8d0ebbf50a4c4d49d50e2c5c6286d0609d0140fc44d2d72ba21132466a
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# HttpClient Class
|
4
|
+
class HttpClient
|
5
|
+
@request_options = {
|
6
|
+
ssl_verifypeer: false,
|
7
|
+
ssl_verifyhost: 0
|
8
|
+
}
|
9
|
+
|
10
|
+
def self.headers(url, authentication)
|
11
|
+
case
|
12
|
+
when url.include?('yeswehack')
|
13
|
+
{ 'Content-Type' => 'application/json', Authorization: "Bearer #{authentication}" }
|
14
|
+
when url.include?('intigriti')
|
15
|
+
{ Authorization: "Bearer #{authentication}" }
|
16
|
+
when url.include?('bugcrowd')
|
17
|
+
{ 'Cookie' => authentication }
|
18
|
+
when url.include?('hackerone')
|
19
|
+
@request_options[:userpwd] = "#{ENV.fetch('H1_USERNAME', nil)}:#{ENV.fetch('H1_API_KEY', nil)}"
|
20
|
+
{ 'Accept' => 'application/json' }
|
21
|
+
else
|
22
|
+
{ 'Content-Type' => 'application/json' }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.get(url, authentication = nil)
|
27
|
+
@request_options[:headers] = headers(url, authentication)
|
28
|
+
|
29
|
+
Typhoeus.get(url, @request_options)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.post(url, data)
|
33
|
+
@request_options[:headers] = { 'Content-Type' => 'application/json' }
|
34
|
+
@request_options[:body] = data
|
35
|
+
|
36
|
+
Typhoeus.post(url, @request_options)
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'mechanize'
|
4
|
+
|
5
|
+
class Bugcrowd
|
6
|
+
# Bugcrowd Auth Class
|
7
|
+
class Auth
|
8
|
+
def self.cookie
|
9
|
+
# Use Mechanize otherwise the login flow is a hell with Typhoeus
|
10
|
+
mechanize = Mechanize.new
|
11
|
+
|
12
|
+
submit_credentials(mechanize)
|
13
|
+
cookie = dump_cookie(mechanize)
|
14
|
+
return unless cookie
|
15
|
+
|
16
|
+
cookie
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.submit_credentials(mechanize)
|
20
|
+
login_page = mechanize.get('https://bugcrowd.com/user/sign_in')
|
21
|
+
form = login_page.forms.first
|
22
|
+
|
23
|
+
form.field_with(id: 'user_email').value = ENV.fetch('BUGCROWD_EMAIL', nil)
|
24
|
+
form.field_with(id: 'user_password').value = ENV.fetch('BUGCROWD_PASSWORD', nil)
|
25
|
+
form.submit
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.dump_cookie(mechanize)
|
29
|
+
begin
|
30
|
+
page = mechanize.get('https://bugcrowd.com/dashboard')
|
31
|
+
rescue Mechanize::ResponseCodeError
|
32
|
+
return
|
33
|
+
end
|
34
|
+
return unless page
|
35
|
+
|
36
|
+
set_cookie = page.header['Set-Cookie']
|
37
|
+
match = /_crowdcontrol_session=[\w-]+/.match(set_cookie)
|
38
|
+
|
39
|
+
match[0]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class Bugcrowd
|
6
|
+
# Bugcrowd Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, cookie, page_id = 1)
|
9
|
+
response = HttpClient.get("https://bugcrowd.com/programs.json?page[]=#{page_id}&waitlistable[]=false&joinable[]=false", cookie)
|
10
|
+
return unless response&.code == 200
|
11
|
+
|
12
|
+
body = JSON.parse(response.body)
|
13
|
+
parse_programs(body['programs'], options, results, cookie)
|
14
|
+
|
15
|
+
sync(results, options, cookie, page_id + 1) unless page_id == body['meta']['totalPages']
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.parse_programs(programs, options, results, cookie)
|
19
|
+
programs.each do |program|
|
20
|
+
next if program['status'] == 4 # Disabled
|
21
|
+
next if program['min_rewards'].nil? && options[:skip_vdp]
|
22
|
+
|
23
|
+
results[program['name']] = program_info(program)
|
24
|
+
results[program['name']]['scopes'] = Scopes.sync(program_info(program), cookie)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.program_info(program)
|
29
|
+
{
|
30
|
+
slug: program['code'],
|
31
|
+
enabled: true,
|
32
|
+
private: false
|
33
|
+
}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Bugcrowd
|
4
|
+
# Bugcrowd Sync Programs
|
5
|
+
class Scopes
|
6
|
+
def self.sync(program, cookie)
|
7
|
+
scopes = {}
|
8
|
+
response = HttpClient.get("https://bugcrowd.com/#{program[:slug]}.json", cookie)
|
9
|
+
return scopes unless response&.code == 200
|
10
|
+
|
11
|
+
target_group_url = JSON.parse(response.body).dig('program', 'targetGroupsUrl')
|
12
|
+
response = HttpClient.get(File.join('https://bugcrowd.com/', target_group_url), cookie)
|
13
|
+
return scopes unless response&.code == 200
|
14
|
+
|
15
|
+
targets_url = JSON.parse(response.body).dig('groups', 0, 'targets_url')
|
16
|
+
return scopes unless targets_url
|
17
|
+
|
18
|
+
response = HttpClient.get(File.join('https://bugcrowd.com/', targets_url), cookie)
|
19
|
+
return scopes unless response&.code == 200
|
20
|
+
|
21
|
+
in_scopes = JSON.parse(response.body)['targets']
|
22
|
+
scopes['in'] = parse_scopes(in_scopes)
|
23
|
+
|
24
|
+
scopes['out'] = { } # TODO
|
25
|
+
|
26
|
+
scopes
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.parse_scopes(scopes)
|
30
|
+
exclusions = %w[}] # TODO : Try to normalize this
|
31
|
+
scopes_normalized = []
|
32
|
+
|
33
|
+
scopes.each do |scope|
|
34
|
+
next unless scope['category'] == 'website' || scope['category'] == 'api'
|
35
|
+
|
36
|
+
endpoint = scope['name']
|
37
|
+
next if exclusions.any? { |exclusion| endpoint.include?(exclusion) } || !endpoint.include?('.')
|
38
|
+
|
39
|
+
scopes_normalized << endpoint
|
40
|
+
end
|
41
|
+
|
42
|
+
scopes_normalized
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class Hackerone
|
6
|
+
# Hackerone Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, page_id = 1)
|
9
|
+
programs_infos = programs_infos(page_id)
|
10
|
+
return unless programs_infos
|
11
|
+
|
12
|
+
programs_infos[:programs].each do |program|
|
13
|
+
next unless program['attributes']['submission_state'] == 'open' && program['attributes']['offers_bounties']
|
14
|
+
next if options[:skip_vdp] && !program['attributes']['offers_bounties']
|
15
|
+
|
16
|
+
results[program['attributes']['name']] = program_info(program)
|
17
|
+
results[program['attributes']['name']]['scopes'] = Scopes.sync(program_info(program))
|
18
|
+
end
|
19
|
+
|
20
|
+
sync(results, options, page_id + 1) if programs_infos[:next_page]
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.program_info(program)
|
24
|
+
{
|
25
|
+
slug: program['attributes']['handle'],
|
26
|
+
enabled: true,
|
27
|
+
private: !program['attributes']['state'] == 'public_mode'
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.programs_infos(page_id)
|
32
|
+
response = HttpClient.get("https://api.hackerone.com/v1/hackers/programs?page%5Bnumber%5D=#{page_id}")
|
33
|
+
if response&.code == 429
|
34
|
+
sleep 65 # Rate limit
|
35
|
+
programs_infos(page_id)
|
36
|
+
end
|
37
|
+
return unless response.code == 200
|
38
|
+
|
39
|
+
json_body = JSON.parse(response.body)
|
40
|
+
{ next_page: json_body['links']['next'], programs: json_body['data'] }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Hackerone
|
4
|
+
# Hackerone Sync Programs
|
5
|
+
class Scopes
|
6
|
+
def self.sync(program)
|
7
|
+
scopes = {}
|
8
|
+
response = HttpClient.get( "https://api.hackerone.com/v1/hackers/programs/#{program[:slug]}")
|
9
|
+
return scopes unless response&.code == 200
|
10
|
+
|
11
|
+
in_scopes = JSON.parse(response.body)['relationships']['structured_scopes']['data']
|
12
|
+
scopes['in'] = parse_scopes(in_scopes)
|
13
|
+
|
14
|
+
scopes['out'] = { } # TODO
|
15
|
+
|
16
|
+
scopes
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.parse_scopes(scopes)
|
20
|
+
scopes_normalized = []
|
21
|
+
|
22
|
+
scopes.each do |scope|
|
23
|
+
next unless scope['attributes']['asset_type'] == 'URL'
|
24
|
+
|
25
|
+
endpoint = scope['attributes']['asset_identifier']
|
26
|
+
normalized = normalized(endpoint)
|
27
|
+
|
28
|
+
normalized.each do |asset|
|
29
|
+
scopes_normalized << asset
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
scopes_normalized
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.normalized(endpoint)
|
37
|
+
endpoint.sub!(%r{/$}, '')
|
38
|
+
|
39
|
+
normalized = []
|
40
|
+
|
41
|
+
if endpoint.include?(',')
|
42
|
+
endpoint.split(',').each { |asset| normalized << asset }
|
43
|
+
else
|
44
|
+
normalized << endpoint
|
45
|
+
end
|
46
|
+
|
47
|
+
normalized
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class Intigriti
|
6
|
+
# Intigrit Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, token)
|
9
|
+
response = HttpClient.get('https://api.intigriti.com/core/researcher/programs', token)
|
10
|
+
return unless response&.code == 200
|
11
|
+
|
12
|
+
programs = JSON.parse(response.body)
|
13
|
+
parse_programs(programs, options, results, token)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.parse_programs(programs, options, results, token)
|
17
|
+
programs.each do |program|
|
18
|
+
next if options[:skip_vdp] && !program['maxBounty']['value'].positive?
|
19
|
+
next if program['status'] == 4 # Suspended
|
20
|
+
|
21
|
+
results[program['name']] = program_info(program)
|
22
|
+
results[program['name']]['scopes'] = Scopes.sync({ handle: program['handle'], company: program['companyHandle'] }, token)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.program_info(program)
|
27
|
+
{
|
28
|
+
slug: program['handle'],
|
29
|
+
enabled: true,
|
30
|
+
private: program['confidentialityLevel'] != 4 # == public
|
31
|
+
}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'cgi'
|
4
|
+
|
5
|
+
class Intigriti
|
6
|
+
# Intigrit Sync Programs
|
7
|
+
class Scopes
|
8
|
+
def self.sync(program, token)
|
9
|
+
scopes = {}
|
10
|
+
company = CGI.escape(program[:company])
|
11
|
+
handle = CGI.escape(program[:handle])
|
12
|
+
|
13
|
+
response = HttpClient.get("https://api.intigriti.com/core/researcher/programs/#{company}/#{handle}", token)
|
14
|
+
return scopes unless response&.code == 200
|
15
|
+
|
16
|
+
in_scopes = JSON.parse(response.body)['domains']&.last['content']
|
17
|
+
scopes['in'] = parse_scopes(in_scopes)
|
18
|
+
|
19
|
+
out_scopes = JSON.parse(response.body)['outOfScopes'].last['content']['content']
|
20
|
+
scopes['out'] = out_scopes
|
21
|
+
|
22
|
+
scopes
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.parse_scopes(scopes)
|
26
|
+
exclusions = %w[> | \] } Anyrelated] # TODO : Try to normalize this, it only concerns 1 or 2 programs currently
|
27
|
+
scopes_normalized = []
|
28
|
+
|
29
|
+
scopes.each do |scope|
|
30
|
+
next unless scope['type'] == 1 # 1 == Web Application
|
31
|
+
|
32
|
+
endpoint = normalize(scope['endpoint'])
|
33
|
+
next if exclusions.any? { |exclusion| endpoint.include?(exclusion) } || !endpoint.include?('.')
|
34
|
+
|
35
|
+
scopes_normalized << endpoint
|
36
|
+
end
|
37
|
+
|
38
|
+
scopes_normalized
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.normalize(endpoint)
|
42
|
+
endpoint.gsub('/*', '').gsub(' ', '').sub('.*', '.com').sub('.<tld>', '.com')
|
43
|
+
.sub(%r{/$}, '').sub(/\*$/, '')
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'mechanize'
|
4
|
+
|
5
|
+
class Intigriti
|
6
|
+
# Intigriti Auth Class
|
7
|
+
class Auth
|
8
|
+
def self.token
|
9
|
+
# Use Mechanize otherwise the login flow is a hell with Typhoeus
|
10
|
+
mechanize = Mechanize.new
|
11
|
+
|
12
|
+
submit_credentials(mechanize)
|
13
|
+
submit_otp(mechanize)
|
14
|
+
token = dump_token(mechanize)
|
15
|
+
return unless token
|
16
|
+
|
17
|
+
token
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.submit_credentials(mechanize)
|
21
|
+
login_page = mechanize.get('https://login.intigriti.com/account/login')
|
22
|
+
form = login_page.forms.first
|
23
|
+
|
24
|
+
form.field_with(id: 'Input_Email').value = ENV.fetch('INTIGRITI_EMAIL', nil)
|
25
|
+
resp = form.submit
|
26
|
+
form = resp.forms.first
|
27
|
+
|
28
|
+
form.field_with(id: 'Input_Password').value = ENV.fetch('INTIGRITI_PASSWORD', nil)
|
29
|
+
form.submit
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.submit_otp(mechanize)
|
33
|
+
return if ENV['INTIGRITI_OTP']&.empty?
|
34
|
+
|
35
|
+
totp_page = mechanize.get('https://login.intigriti.com/account/loginwith2fa')
|
36
|
+
totp_code = ROTP::TOTP.new(ENV.fetch('INTI_OTP', nil))
|
37
|
+
|
38
|
+
form = totp_page.forms.first
|
39
|
+
form.field_with(id: 'Input_TwoFactorAuthentication_VerificationCode').value = totp_code.now
|
40
|
+
form.submit
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.dump_token(mechanize)
|
44
|
+
begin
|
45
|
+
token_page = mechanize.get('https://app.intigriti.com/auth/token')
|
46
|
+
rescue Mechanize::ResponseCodeError
|
47
|
+
return
|
48
|
+
end
|
49
|
+
return unless token_page&.body
|
50
|
+
|
51
|
+
token_page.body&.undump
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class YesWeHack
|
4
|
+
# YesWeHack Auth Class
|
5
|
+
class Auth
|
6
|
+
def self.jwt
|
7
|
+
totp_token = get_totp_token
|
8
|
+
return unless totp_token
|
9
|
+
|
10
|
+
response = send_totp(totp_token)
|
11
|
+
return unless response
|
12
|
+
|
13
|
+
jwt = JSON.parse(response.body)['token']
|
14
|
+
return unless jwt
|
15
|
+
|
16
|
+
jwt
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.get_totp_token
|
20
|
+
data = { email: ENV.fetch('YWH_EMAIL', nil), password: ENV.fetch('YWH_PASSWORD', nil) }.to_json
|
21
|
+
response = HttpClient.post('https://api.yeswehack.com/login', data)
|
22
|
+
return unless response&.code == 200
|
23
|
+
|
24
|
+
JSON.parse(response.body)['totp_token']
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.send_totp(totp_token)
|
28
|
+
data = { token: totp_token, code: ROTP::TOTP.new(ENV['YWH_OTP']).now }.to_json
|
29
|
+
response = HttpClient.post('https://api.yeswehack.com/account/totp', data)
|
30
|
+
return unless response.code == 200
|
31
|
+
|
32
|
+
response
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class YesWeHack
|
6
|
+
# YesWeHack Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, jwt, page_id = 1)
|
9
|
+
programs_infos = get_programs_infos(page_id, jwt)
|
10
|
+
return unless programs_infos
|
11
|
+
|
12
|
+
parse_programs(programs_infos, options, results, jwt)
|
13
|
+
sync(results, options, jwt, page_id + 1) unless page_id == programs_infos[:nb_pages]
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.get_programs_infos(page_id, jwt)
|
17
|
+
response = HttpClient.get("https://api.yeswehack.com/programs?page=#{page_id}", jwt)
|
18
|
+
return unless response&.code == 200
|
19
|
+
|
20
|
+
json_body = JSON.parse(response.body)
|
21
|
+
{ nb_pages: json_body['pagination']['nb_pages'], programs: json_body['items'] }
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.parse_programs(programs_infos, options, results, jwt)
|
25
|
+
programs_infos[:programs].each do |program|
|
26
|
+
next if program['disabled']
|
27
|
+
next if program['vdp'] && options[:skip_vdp]
|
28
|
+
|
29
|
+
results[program['title']] = program_info(program)
|
30
|
+
results[program['title']]['scopes'] = Scopes.sync(program_info(program), jwt)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.program_info(program)
|
35
|
+
{
|
36
|
+
slug: program['slug'],
|
37
|
+
enabled: true,
|
38
|
+
private: !program['public']
|
39
|
+
}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class YesWeHack
|
4
|
+
# YesWeHack Sync Scopes
|
5
|
+
class Scopes
|
6
|
+
def self.sync(program, jwt)
|
7
|
+
scopes = {}
|
8
|
+
response = HttpClient.get("https://api.yeswehack.com/programs/#{program[:slug]}", jwt)
|
9
|
+
return scopes unless response&.code == 200
|
10
|
+
|
11
|
+
in_scopes = JSON.parse(response.body)['scopes']
|
12
|
+
scopes['in'] = parse_scopes(in_scopes)
|
13
|
+
|
14
|
+
out_scopes = JSON.parse(response.body)&.dig('out_of_scope')
|
15
|
+
scopes['out'] = out_scopes
|
16
|
+
|
17
|
+
scopes
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.parse_scopes(scopes)
|
21
|
+
scopes_normalized = []
|
22
|
+
|
23
|
+
scopes.each do |infos|
|
24
|
+
next unless %w[web-application api].include?(infos['scope_type'])
|
25
|
+
|
26
|
+
normalized = normalize(infos['scope'])
|
27
|
+
normalized.each do |asset|
|
28
|
+
scopes_normalized << asset
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
scopes_normalized
|
33
|
+
end
|
34
|
+
|
35
|
+
# rubocop:disable Metrics/AbcSize
|
36
|
+
# rubocop:disable Metrics/MethodLength
|
37
|
+
# rubocop:disable Metrics/PerceivedComplexity
|
38
|
+
# rubocop:disable Metrics/CyclomaticComplexity
|
39
|
+
def self.normalize(scope)
|
40
|
+
# Remove (+++) & When end with '*'
|
41
|
+
scope = scope.gsub(/\(?\+\)?/, '').sub(/\*$/, '').strip
|
42
|
+
return [] if scope.include?('<') # <yourdomain>-yeswehack.domain.tld
|
43
|
+
|
44
|
+
normalized = []
|
45
|
+
|
46
|
+
match = scope.match(/^(.*)\((.*)\)$/) # Ex: *.lazada.(sg|vn|co.id|co.th|com|com.ph|com.my)
|
47
|
+
if match && match[1] && match[2]
|
48
|
+
tlds = match[2].split('|')
|
49
|
+
tlds.each { |tld| normalized << "#{match[1]}#{tld}" }
|
50
|
+
elsif scope.include?(' ')
|
51
|
+
normalized << scope.split(' ')[0]
|
52
|
+
elsif scope.match?(%r{https?://\*})
|
53
|
+
normalized << scope.sub(%r{https?://}, '')
|
54
|
+
else
|
55
|
+
normalized << scope
|
56
|
+
end
|
57
|
+
|
58
|
+
normalized
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
require 'colorize'
|
5
|
+
|
6
|
+
class ScopesExtractor
|
7
|
+
# Provides helper methods to be used in all the different classes
|
8
|
+
class Utilities
|
9
|
+
# Creates a singleton logger
|
10
|
+
def self.logger
|
11
|
+
@logger ||= Logger.new($stdout)
|
12
|
+
end
|
13
|
+
|
14
|
+
# Set the log level for the previous logger
|
15
|
+
def self.log_level=(level)
|
16
|
+
logger.level = level.downcase.to_sym
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.log_fatal(message)
|
20
|
+
logger.fatal(message.red)
|
21
|
+
|
22
|
+
exit
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.log_info(message)
|
26
|
+
logger.info(message.green)
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.log_control(message)
|
30
|
+
logger.info(message.light_blue)
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.log_warn(message)
|
34
|
+
logger.warn(message.yellow)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'dotenv'
|
4
|
+
require 'json'
|
5
|
+
require 'rotp'
|
6
|
+
require 'typhoeus'
|
7
|
+
|
8
|
+
require_relative 'scopes_extractor/utilities'
|
9
|
+
require_relative 'scopes_extractor/http_client'
|
10
|
+
require_relative 'scopes_extractor/platforms/bugcrowd/cookie'
|
11
|
+
require_relative 'scopes_extractor/platforms/bugcrowd/programs'
|
12
|
+
require_relative 'scopes_extractor/platforms/hackerone/programs'
|
13
|
+
require_relative 'scopes_extractor/platforms/intigriti/token'
|
14
|
+
require_relative 'scopes_extractor/platforms/intigriti/programs'
|
15
|
+
require_relative 'scopes_extractor/platforms/yeswehack/jwt'
|
16
|
+
require_relative 'scopes_extractor/platforms/yeswehack/programs'
|
17
|
+
|
18
|
+
# Class entrypoint to start the extractions and initializes all the objects
|
19
|
+
class ScopesExtractor
|
20
|
+
attr_reader :options
|
21
|
+
attr_accessor :results
|
22
|
+
|
23
|
+
def initialize(options = {})
|
24
|
+
@options = options
|
25
|
+
@results = {}
|
26
|
+
end
|
27
|
+
|
28
|
+
def extract
|
29
|
+
Utilities.log_fatal('[-] The file containing the credentials is mandatory') unless options[:credz_file]
|
30
|
+
Dotenv.load(options[:credz_file])
|
31
|
+
|
32
|
+
if options[:yeswehack]
|
33
|
+
jwt = YesWeHack::Auth.jwt
|
34
|
+
|
35
|
+
results['YesWeHack'] = {}
|
36
|
+
YesWeHack::Programs.sync(results['YesWeHack'], options, jwt)
|
37
|
+
end
|
38
|
+
|
39
|
+
if options[:intigriti]
|
40
|
+
token = Intigriti::Auth.token
|
41
|
+
results['Intigriti'] = {}
|
42
|
+
|
43
|
+
Intigriti::Programs.sync(results['Intigriti'], options, token)
|
44
|
+
end
|
45
|
+
|
46
|
+
if options[:bugcrowd]
|
47
|
+
cookie = Bugcrowd::Auth.cookie
|
48
|
+
results['Bugcrowd'] = {}
|
49
|
+
|
50
|
+
Bugcrowd::Programs.sync(results['Bugcrowd'], options, cookie)
|
51
|
+
end
|
52
|
+
|
53
|
+
if options[:hackerone]
|
54
|
+
results['Hackerone'] = {}
|
55
|
+
Hackerone::Programs.sync(results['Hackerone'], options)
|
56
|
+
end
|
57
|
+
|
58
|
+
p results
|
59
|
+
end
|
60
|
+
end
|
metadata
ADDED
@@ -0,0 +1,171 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: scopes_extractor
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Joshua MARTINELLE
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-05-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: colorize
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.8.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.8.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: dotenv
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2.8'
|
34
|
+
- - ">="
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 2.8.1
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - "~>"
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '2.8'
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 2.8.1
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: logger
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1.5'
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 1.5.3
|
57
|
+
type: :runtime
|
58
|
+
prerelease: false
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - "~>"
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '1.5'
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 1.5.3
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
name: mechanize
|
69
|
+
requirement: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - "~>"
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '2.9'
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: 2.9.1
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '2.9'
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: 2.9.1
|
87
|
+
- !ruby/object:Gem::Dependency
|
88
|
+
name: typhoeus
|
89
|
+
requirement: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - "~>"
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '1.4'
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 1.4.0
|
97
|
+
type: :runtime
|
98
|
+
prerelease: false
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1.4'
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: 1.4.0
|
107
|
+
- !ruby/object:Gem::Dependency
|
108
|
+
name: rotp
|
109
|
+
requirement: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - "~>"
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '6.2'
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 6.2.2
|
117
|
+
type: :runtime
|
118
|
+
prerelease: false
|
119
|
+
version_requirements: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - "~>"
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '6.2'
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: 6.2.2
|
127
|
+
description:
|
128
|
+
email:
|
129
|
+
- contact@jomar.fr
|
130
|
+
executables: []
|
131
|
+
extensions: []
|
132
|
+
extra_rdoc_files: []
|
133
|
+
files:
|
134
|
+
- lib/scopes_extractor.rb
|
135
|
+
- lib/scopes_extractor/http_client.rb
|
136
|
+
- lib/scopes_extractor/platforms/bugcrowd/cookie.rb
|
137
|
+
- lib/scopes_extractor/platforms/bugcrowd/programs.rb
|
138
|
+
- lib/scopes_extractor/platforms/bugcrowd/scopes.rb
|
139
|
+
- lib/scopes_extractor/platforms/hackerone/programs.rb
|
140
|
+
- lib/scopes_extractor/platforms/hackerone/scopes.rb
|
141
|
+
- lib/scopes_extractor/platforms/intigriti/programs.rb
|
142
|
+
- lib/scopes_extractor/platforms/intigriti/scopes.rb
|
143
|
+
- lib/scopes_extractor/platforms/intigriti/token.rb
|
144
|
+
- lib/scopes_extractor/platforms/yeswehack/jwt.rb
|
145
|
+
- lib/scopes_extractor/platforms/yeswehack/programs.rb
|
146
|
+
- lib/scopes_extractor/platforms/yeswehack/scopes.rb
|
147
|
+
- lib/scopes_extractor/utilities.rb
|
148
|
+
homepage: https://rubygems.org/gems/scopes_extractor
|
149
|
+
licenses:
|
150
|
+
- MIT
|
151
|
+
metadata: {}
|
152
|
+
post_install_message:
|
153
|
+
rdoc_options: []
|
154
|
+
require_paths:
|
155
|
+
- lib
|
156
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
157
|
+
requirements:
|
158
|
+
- - ">="
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
version: 2.7.1
|
161
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
162
|
+
requirements:
|
163
|
+
- - ">="
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
requirements: []
|
167
|
+
rubygems_version: 3.1.2
|
168
|
+
signing_key:
|
169
|
+
specification_version: 4
|
170
|
+
summary: BugBounty Scopes Extractor
|
171
|
+
test_files: []
|