scopes_extractor 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/scopes_extractor/http_client.rb +38 -0
- data/lib/scopes_extractor/platforms/bugcrowd/cookie.rb +42 -0
- data/lib/scopes_extractor/platforms/bugcrowd/programs.rb +36 -0
- data/lib/scopes_extractor/platforms/bugcrowd/scopes.rb +45 -0
- data/lib/scopes_extractor/platforms/hackerone/programs.rb +43 -0
- data/lib/scopes_extractor/platforms/hackerone/scopes.rb +50 -0
- data/lib/scopes_extractor/platforms/intigriti/programs.rb +34 -0
- data/lib/scopes_extractor/platforms/intigriti/scopes.rb +46 -0
- data/lib/scopes_extractor/platforms/intigriti/token.rb +54 -0
- data/lib/scopes_extractor/platforms/yeswehack/jwt.rb +35 -0
- data/lib/scopes_extractor/platforms/yeswehack/programs.rb +42 -0
- data/lib/scopes_extractor/platforms/yeswehack/scopes.rb +61 -0
- data/lib/scopes_extractor/utilities.rb +37 -0
- data/lib/scopes_extractor.rb +60 -0
- metadata +171 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 694da3bb3d2fd3ed580db8d5afa0da7bac2c2326037ed4218b57d87eca7ca174
|
4
|
+
data.tar.gz: 100836692c414cd682653231a65a5937dc8b201560bbf10fa95e96e38d9b39a8
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1a3f2040079e8e05cfd2157b259f95b8912813f8f7ec2e867fd0ce9f7802a05a3b46c63027b5493a0d79a2b28f536b67686b9fb3b5d71926e70de31b846764ca
|
7
|
+
data.tar.gz: 53e92c8b6bcf0608c6c14d6e4f4733e29a8c86a930b7a25de1b6d5242c45d8e7eb38df8d0ebbf50a4c4d49d50e2c5c6286d0609d0140fc44d2d72ba21132466a
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# HttpClient Class
|
4
|
+
class HttpClient
|
5
|
+
@request_options = {
|
6
|
+
ssl_verifypeer: false,
|
7
|
+
ssl_verifyhost: 0
|
8
|
+
}
|
9
|
+
|
10
|
+
def self.headers(url, authentication)
|
11
|
+
case
|
12
|
+
when url.include?('yeswehack')
|
13
|
+
{ 'Content-Type' => 'application/json', Authorization: "Bearer #{authentication}" }
|
14
|
+
when url.include?('intigriti')
|
15
|
+
{ Authorization: "Bearer #{authentication}" }
|
16
|
+
when url.include?('bugcrowd')
|
17
|
+
{ 'Cookie' => authentication }
|
18
|
+
when url.include?('hackerone')
|
19
|
+
@request_options[:userpwd] = "#{ENV.fetch('H1_USERNAME', nil)}:#{ENV.fetch('H1_API_KEY', nil)}"
|
20
|
+
{ 'Accept' => 'application/json' }
|
21
|
+
else
|
22
|
+
{ 'Content-Type' => 'application/json' }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.get(url, authentication = nil)
|
27
|
+
@request_options[:headers] = headers(url, authentication)
|
28
|
+
|
29
|
+
Typhoeus.get(url, @request_options)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.post(url, data)
|
33
|
+
@request_options[:headers] = { 'Content-Type' => 'application/json' }
|
34
|
+
@request_options[:body] = data
|
35
|
+
|
36
|
+
Typhoeus.post(url, @request_options)
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'mechanize'
|
4
|
+
|
5
|
+
class Bugcrowd
|
6
|
+
# Bugcrowd Auth Class
|
7
|
+
class Auth
|
8
|
+
def self.cookie
|
9
|
+
# Use Mechanize otherwise the login flow is a hell with Typhoeus
|
10
|
+
mechanize = Mechanize.new
|
11
|
+
|
12
|
+
submit_credentials(mechanize)
|
13
|
+
cookie = dump_cookie(mechanize)
|
14
|
+
return unless cookie
|
15
|
+
|
16
|
+
cookie
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.submit_credentials(mechanize)
|
20
|
+
login_page = mechanize.get('https://bugcrowd.com/user/sign_in')
|
21
|
+
form = login_page.forms.first
|
22
|
+
|
23
|
+
form.field_with(id: 'user_email').value = ENV.fetch('BUGCROWD_EMAIL', nil)
|
24
|
+
form.field_with(id: 'user_password').value = ENV.fetch('BUGCROWD_PASSWORD', nil)
|
25
|
+
form.submit
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.dump_cookie(mechanize)
|
29
|
+
begin
|
30
|
+
page = mechanize.get('https://bugcrowd.com/dashboard')
|
31
|
+
rescue Mechanize::ResponseCodeError
|
32
|
+
return
|
33
|
+
end
|
34
|
+
return unless page
|
35
|
+
|
36
|
+
set_cookie = page.header['Set-Cookie']
|
37
|
+
match = /_crowdcontrol_session=[\w-]+/.match(set_cookie)
|
38
|
+
|
39
|
+
match[0]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class Bugcrowd
|
6
|
+
# Bugcrowd Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, cookie, page_id = 1)
|
9
|
+
response = HttpClient.get("https://bugcrowd.com/programs.json?page[]=#{page_id}&waitlistable[]=false&joinable[]=false", cookie)
|
10
|
+
return unless response&.code == 200
|
11
|
+
|
12
|
+
body = JSON.parse(response.body)
|
13
|
+
parse_programs(body['programs'], options, results, cookie)
|
14
|
+
|
15
|
+
sync(results, options, cookie, page_id + 1) unless page_id == body['meta']['totalPages']
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.parse_programs(programs, options, results, cookie)
|
19
|
+
programs.each do |program|
|
20
|
+
next if program['status'] == 4 # Disabled
|
21
|
+
next if program['min_rewards'].nil? && options[:skip_vdp]
|
22
|
+
|
23
|
+
results[program['name']] = program_info(program)
|
24
|
+
results[program['name']]['scopes'] = Scopes.sync(program_info(program), cookie)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.program_info(program)
|
29
|
+
{
|
30
|
+
slug: program['code'],
|
31
|
+
enabled: true,
|
32
|
+
private: false
|
33
|
+
}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Bugcrowd
|
4
|
+
# Bugcrowd Sync Programs
|
5
|
+
class Scopes
|
6
|
+
def self.sync(program, cookie)
|
7
|
+
scopes = {}
|
8
|
+
response = HttpClient.get("https://bugcrowd.com/#{program[:slug]}.json", cookie)
|
9
|
+
return scopes unless response&.code == 200
|
10
|
+
|
11
|
+
target_group_url = JSON.parse(response.body).dig('program', 'targetGroupsUrl')
|
12
|
+
response = HttpClient.get(File.join('https://bugcrowd.com/', target_group_url), cookie)
|
13
|
+
return scopes unless response&.code == 200
|
14
|
+
|
15
|
+
targets_url = JSON.parse(response.body).dig('groups', 0, 'targets_url')
|
16
|
+
return scopes unless targets_url
|
17
|
+
|
18
|
+
response = HttpClient.get(File.join('https://bugcrowd.com/', targets_url), cookie)
|
19
|
+
return scopes unless response&.code == 200
|
20
|
+
|
21
|
+
in_scopes = JSON.parse(response.body)['targets']
|
22
|
+
scopes['in'] = parse_scopes(in_scopes)
|
23
|
+
|
24
|
+
scopes['out'] = { } # TODO
|
25
|
+
|
26
|
+
scopes
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.parse_scopes(scopes)
|
30
|
+
exclusions = %w[}] # TODO : Try to normalize this
|
31
|
+
scopes_normalized = []
|
32
|
+
|
33
|
+
scopes.each do |scope|
|
34
|
+
next unless scope['category'] == 'website' || scope['category'] == 'api'
|
35
|
+
|
36
|
+
endpoint = scope['name']
|
37
|
+
next if exclusions.any? { |exclusion| endpoint.include?(exclusion) } || !endpoint.include?('.')
|
38
|
+
|
39
|
+
scopes_normalized << endpoint
|
40
|
+
end
|
41
|
+
|
42
|
+
scopes_normalized
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class Hackerone
|
6
|
+
# Hackerone Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, page_id = 1)
|
9
|
+
programs_infos = programs_infos(page_id)
|
10
|
+
return unless programs_infos
|
11
|
+
|
12
|
+
programs_infos[:programs].each do |program|
|
13
|
+
next unless program['attributes']['submission_state'] == 'open' && program['attributes']['offers_bounties']
|
14
|
+
next if options[:skip_vdp] && !program['attributes']['offers_bounties']
|
15
|
+
|
16
|
+
results[program['attributes']['name']] = program_info(program)
|
17
|
+
results[program['attributes']['name']]['scopes'] = Scopes.sync(program_info(program))
|
18
|
+
end
|
19
|
+
|
20
|
+
sync(results, options, page_id + 1) if programs_infos[:next_page]
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.program_info(program)
|
24
|
+
{
|
25
|
+
slug: program['attributes']['handle'],
|
26
|
+
enabled: true,
|
27
|
+
private: !program['attributes']['state'] == 'public_mode'
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.programs_infos(page_id)
|
32
|
+
response = HttpClient.get("https://api.hackerone.com/v1/hackers/programs?page%5Bnumber%5D=#{page_id}")
|
33
|
+
if response&.code == 429
|
34
|
+
sleep 65 # Rate limit
|
35
|
+
programs_infos(page_id)
|
36
|
+
end
|
37
|
+
return unless response.code == 200
|
38
|
+
|
39
|
+
json_body = JSON.parse(response.body)
|
40
|
+
{ next_page: json_body['links']['next'], programs: json_body['data'] }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Hackerone
|
4
|
+
# Hackerone Sync Programs
|
5
|
+
class Scopes
|
6
|
+
def self.sync(program)
|
7
|
+
scopes = {}
|
8
|
+
response = HttpClient.get( "https://api.hackerone.com/v1/hackers/programs/#{program[:slug]}")
|
9
|
+
return scopes unless response&.code == 200
|
10
|
+
|
11
|
+
in_scopes = JSON.parse(response.body)['relationships']['structured_scopes']['data']
|
12
|
+
scopes['in'] = parse_scopes(in_scopes)
|
13
|
+
|
14
|
+
scopes['out'] = { } # TODO
|
15
|
+
|
16
|
+
scopes
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.parse_scopes(scopes)
|
20
|
+
scopes_normalized = []
|
21
|
+
|
22
|
+
scopes.each do |scope|
|
23
|
+
next unless scope['attributes']['asset_type'] == 'URL'
|
24
|
+
|
25
|
+
endpoint = scope['attributes']['asset_identifier']
|
26
|
+
normalized = normalized(endpoint)
|
27
|
+
|
28
|
+
normalized.each do |asset|
|
29
|
+
scopes_normalized << asset
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
scopes_normalized
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.normalized(endpoint)
|
37
|
+
endpoint.sub!(%r{/$}, '')
|
38
|
+
|
39
|
+
normalized = []
|
40
|
+
|
41
|
+
if endpoint.include?(',')
|
42
|
+
endpoint.split(',').each { |asset| normalized << asset }
|
43
|
+
else
|
44
|
+
normalized << endpoint
|
45
|
+
end
|
46
|
+
|
47
|
+
normalized
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class Intigriti
|
6
|
+
# Intigrit Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, token)
|
9
|
+
response = HttpClient.get('https://api.intigriti.com/core/researcher/programs', token)
|
10
|
+
return unless response&.code == 200
|
11
|
+
|
12
|
+
programs = JSON.parse(response.body)
|
13
|
+
parse_programs(programs, options, results, token)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.parse_programs(programs, options, results, token)
|
17
|
+
programs.each do |program|
|
18
|
+
next if options[:skip_vdp] && !program['maxBounty']['value'].positive?
|
19
|
+
next if program['status'] == 4 # Suspended
|
20
|
+
|
21
|
+
results[program['name']] = program_info(program)
|
22
|
+
results[program['name']]['scopes'] = Scopes.sync({ handle: program['handle'], company: program['companyHandle'] }, token)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.program_info(program)
|
27
|
+
{
|
28
|
+
slug: program['handle'],
|
29
|
+
enabled: true,
|
30
|
+
private: program['confidentialityLevel'] != 4 # == public
|
31
|
+
}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'cgi'
|
4
|
+
|
5
|
+
class Intigriti
|
6
|
+
# Intigrit Sync Programs
|
7
|
+
class Scopes
|
8
|
+
def self.sync(program, token)
|
9
|
+
scopes = {}
|
10
|
+
company = CGI.escape(program[:company])
|
11
|
+
handle = CGI.escape(program[:handle])
|
12
|
+
|
13
|
+
response = HttpClient.get("https://api.intigriti.com/core/researcher/programs/#{company}/#{handle}", token)
|
14
|
+
return scopes unless response&.code == 200
|
15
|
+
|
16
|
+
in_scopes = JSON.parse(response.body)['domains']&.last['content']
|
17
|
+
scopes['in'] = parse_scopes(in_scopes)
|
18
|
+
|
19
|
+
out_scopes = JSON.parse(response.body)['outOfScopes'].last['content']['content']
|
20
|
+
scopes['out'] = out_scopes
|
21
|
+
|
22
|
+
scopes
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.parse_scopes(scopes)
|
26
|
+
exclusions = %w[> | \] } Anyrelated] # TODO : Try to normalize this, it only concerns 1 or 2 programs currently
|
27
|
+
scopes_normalized = []
|
28
|
+
|
29
|
+
scopes.each do |scope|
|
30
|
+
next unless scope['type'] == 1 # 1 == Web Application
|
31
|
+
|
32
|
+
endpoint = normalize(scope['endpoint'])
|
33
|
+
next if exclusions.any? { |exclusion| endpoint.include?(exclusion) } || !endpoint.include?('.')
|
34
|
+
|
35
|
+
scopes_normalized << endpoint
|
36
|
+
end
|
37
|
+
|
38
|
+
scopes_normalized
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.normalize(endpoint)
|
42
|
+
endpoint.gsub('/*', '').gsub(' ', '').sub('.*', '.com').sub('.<tld>', '.com')
|
43
|
+
.sub(%r{/$}, '').sub(/\*$/, '')
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'mechanize'
|
4
|
+
|
5
|
+
class Intigriti
|
6
|
+
# Intigriti Auth Class
|
7
|
+
class Auth
|
8
|
+
def self.token
|
9
|
+
# Use Mechanize otherwise the login flow is a hell with Typhoeus
|
10
|
+
mechanize = Mechanize.new
|
11
|
+
|
12
|
+
submit_credentials(mechanize)
|
13
|
+
submit_otp(mechanize)
|
14
|
+
token = dump_token(mechanize)
|
15
|
+
return unless token
|
16
|
+
|
17
|
+
token
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.submit_credentials(mechanize)
|
21
|
+
login_page = mechanize.get('https://login.intigriti.com/account/login')
|
22
|
+
form = login_page.forms.first
|
23
|
+
|
24
|
+
form.field_with(id: 'Input_Email').value = ENV.fetch('INTIGRITI_EMAIL', nil)
|
25
|
+
resp = form.submit
|
26
|
+
form = resp.forms.first
|
27
|
+
|
28
|
+
form.field_with(id: 'Input_Password').value = ENV.fetch('INTIGRITI_PASSWORD', nil)
|
29
|
+
form.submit
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.submit_otp(mechanize)
|
33
|
+
return if ENV['INTIGRITI_OTP']&.empty?
|
34
|
+
|
35
|
+
totp_page = mechanize.get('https://login.intigriti.com/account/loginwith2fa')
|
36
|
+
totp_code = ROTP::TOTP.new(ENV.fetch('INTI_OTP', nil))
|
37
|
+
|
38
|
+
form = totp_page.forms.first
|
39
|
+
form.field_with(id: 'Input_TwoFactorAuthentication_VerificationCode').value = totp_code.now
|
40
|
+
form.submit
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.dump_token(mechanize)
|
44
|
+
begin
|
45
|
+
token_page = mechanize.get('https://app.intigriti.com/auth/token')
|
46
|
+
rescue Mechanize::ResponseCodeError
|
47
|
+
return
|
48
|
+
end
|
49
|
+
return unless token_page&.body
|
50
|
+
|
51
|
+
token_page.body&.undump
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class YesWeHack
|
4
|
+
# YesWeHack Auth Class
|
5
|
+
class Auth
|
6
|
+
def self.jwt
|
7
|
+
totp_token = get_totp_token
|
8
|
+
return unless totp_token
|
9
|
+
|
10
|
+
response = send_totp(totp_token)
|
11
|
+
return unless response
|
12
|
+
|
13
|
+
jwt = JSON.parse(response.body)['token']
|
14
|
+
return unless jwt
|
15
|
+
|
16
|
+
jwt
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.get_totp_token
|
20
|
+
data = { email: ENV.fetch('YWH_EMAIL', nil), password: ENV.fetch('YWH_PASSWORD', nil) }.to_json
|
21
|
+
response = HttpClient.post('https://api.yeswehack.com/login', data)
|
22
|
+
return unless response&.code == 200
|
23
|
+
|
24
|
+
JSON.parse(response.body)['totp_token']
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.send_totp(totp_token)
|
28
|
+
data = { token: totp_token, code: ROTP::TOTP.new(ENV['YWH_OTP']).now }.to_json
|
29
|
+
response = HttpClient.post('https://api.yeswehack.com/account/totp', data)
|
30
|
+
return unless response.code == 200
|
31
|
+
|
32
|
+
response
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'scopes'
|
4
|
+
|
5
|
+
class YesWeHack
|
6
|
+
# YesWeHack Sync Programs
|
7
|
+
class Programs
|
8
|
+
def self.sync(results, options, jwt, page_id = 1)
|
9
|
+
programs_infos = get_programs_infos(page_id, jwt)
|
10
|
+
return unless programs_infos
|
11
|
+
|
12
|
+
parse_programs(programs_infos, options, results, jwt)
|
13
|
+
sync(results, options, jwt, page_id + 1) unless page_id == programs_infos[:nb_pages]
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.get_programs_infos(page_id, jwt)
|
17
|
+
response = HttpClient.get("https://api.yeswehack.com/programs?page=#{page_id}", jwt)
|
18
|
+
return unless response&.code == 200
|
19
|
+
|
20
|
+
json_body = JSON.parse(response.body)
|
21
|
+
{ nb_pages: json_body['pagination']['nb_pages'], programs: json_body['items'] }
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.parse_programs(programs_infos, options, results, jwt)
|
25
|
+
programs_infos[:programs].each do |program|
|
26
|
+
next if program['disabled']
|
27
|
+
next if program['vdp'] && options[:skip_vdp]
|
28
|
+
|
29
|
+
results[program['title']] = program_info(program)
|
30
|
+
results[program['title']]['scopes'] = Scopes.sync(program_info(program), jwt)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.program_info(program)
|
35
|
+
{
|
36
|
+
slug: program['slug'],
|
37
|
+
enabled: true,
|
38
|
+
private: !program['public']
|
39
|
+
}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class YesWeHack
|
4
|
+
# YesWeHack Sync Scopes
|
5
|
+
class Scopes
|
6
|
+
def self.sync(program, jwt)
|
7
|
+
scopes = {}
|
8
|
+
response = HttpClient.get("https://api.yeswehack.com/programs/#{program[:slug]}", jwt)
|
9
|
+
return scopes unless response&.code == 200
|
10
|
+
|
11
|
+
in_scopes = JSON.parse(response.body)['scopes']
|
12
|
+
scopes['in'] = parse_scopes(in_scopes)
|
13
|
+
|
14
|
+
out_scopes = JSON.parse(response.body)&.dig('out_of_scope')
|
15
|
+
scopes['out'] = out_scopes
|
16
|
+
|
17
|
+
scopes
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.parse_scopes(scopes)
|
21
|
+
scopes_normalized = []
|
22
|
+
|
23
|
+
scopes.each do |infos|
|
24
|
+
next unless %w[web-application api].include?(infos['scope_type'])
|
25
|
+
|
26
|
+
normalized = normalize(infos['scope'])
|
27
|
+
normalized.each do |asset|
|
28
|
+
scopes_normalized << asset
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
scopes_normalized
|
33
|
+
end
|
34
|
+
|
35
|
+
# rubocop:disable Metrics/AbcSize
|
36
|
+
# rubocop:disable Metrics/MethodLength
|
37
|
+
# rubocop:disable Metrics/PerceivedComplexity
|
38
|
+
# rubocop:disable Metrics/CyclomaticComplexity
|
39
|
+
def self.normalize(scope)
|
40
|
+
# Remove (+++) & When end with '*'
|
41
|
+
scope = scope.gsub(/\(?\+\)?/, '').sub(/\*$/, '').strip
|
42
|
+
return [] if scope.include?('<') # <yourdomain>-yeswehack.domain.tld
|
43
|
+
|
44
|
+
normalized = []
|
45
|
+
|
46
|
+
match = scope.match(/^(.*)\((.*)\)$/) # Ex: *.lazada.(sg|vn|co.id|co.th|com|com.ph|com.my)
|
47
|
+
if match && match[1] && match[2]
|
48
|
+
tlds = match[2].split('|')
|
49
|
+
tlds.each { |tld| normalized << "#{match[1]}#{tld}" }
|
50
|
+
elsif scope.include?(' ')
|
51
|
+
normalized << scope.split(' ')[0]
|
52
|
+
elsif scope.match?(%r{https?://\*})
|
53
|
+
normalized << scope.sub(%r{https?://}, '')
|
54
|
+
else
|
55
|
+
normalized << scope
|
56
|
+
end
|
57
|
+
|
58
|
+
normalized
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
require 'colorize'
|
5
|
+
|
6
|
+
class ScopesExtractor
|
7
|
+
# Provides helper methods to be used in all the different classes
|
8
|
+
class Utilities
|
9
|
+
# Creates a singleton logger
|
10
|
+
def self.logger
|
11
|
+
@logger ||= Logger.new($stdout)
|
12
|
+
end
|
13
|
+
|
14
|
+
# Set the log level for the previous logger
|
15
|
+
def self.log_level=(level)
|
16
|
+
logger.level = level.downcase.to_sym
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.log_fatal(message)
|
20
|
+
logger.fatal(message.red)
|
21
|
+
|
22
|
+
exit
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.log_info(message)
|
26
|
+
logger.info(message.green)
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.log_control(message)
|
30
|
+
logger.info(message.light_blue)
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.log_warn(message)
|
34
|
+
logger.warn(message.yellow)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'dotenv'
|
4
|
+
require 'json'
|
5
|
+
require 'rotp'
|
6
|
+
require 'typhoeus'
|
7
|
+
|
8
|
+
require_relative 'scopes_extractor/utilities'
|
9
|
+
require_relative 'scopes_extractor/http_client'
|
10
|
+
require_relative 'scopes_extractor/platforms/bugcrowd/cookie'
|
11
|
+
require_relative 'scopes_extractor/platforms/bugcrowd/programs'
|
12
|
+
require_relative 'scopes_extractor/platforms/hackerone/programs'
|
13
|
+
require_relative 'scopes_extractor/platforms/intigriti/token'
|
14
|
+
require_relative 'scopes_extractor/platforms/intigriti/programs'
|
15
|
+
require_relative 'scopes_extractor/platforms/yeswehack/jwt'
|
16
|
+
require_relative 'scopes_extractor/platforms/yeswehack/programs'
|
17
|
+
|
18
|
+
# Class entrypoint to start the extractions and initializes all the objects
|
19
|
+
class ScopesExtractor
|
20
|
+
attr_reader :options
|
21
|
+
attr_accessor :results
|
22
|
+
|
23
|
+
def initialize(options = {})
|
24
|
+
@options = options
|
25
|
+
@results = {}
|
26
|
+
end
|
27
|
+
|
28
|
+
def extract
|
29
|
+
Utilities.log_fatal('[-] The file containing the credentials is mandatory') unless options[:credz_file]
|
30
|
+
Dotenv.load(options[:credz_file])
|
31
|
+
|
32
|
+
if options[:yeswehack]
|
33
|
+
jwt = YesWeHack::Auth.jwt
|
34
|
+
|
35
|
+
results['YesWeHack'] = {}
|
36
|
+
YesWeHack::Programs.sync(results['YesWeHack'], options, jwt)
|
37
|
+
end
|
38
|
+
|
39
|
+
if options[:intigriti]
|
40
|
+
token = Intigriti::Auth.token
|
41
|
+
results['Intigriti'] = {}
|
42
|
+
|
43
|
+
Intigriti::Programs.sync(results['Intigriti'], options, token)
|
44
|
+
end
|
45
|
+
|
46
|
+
if options[:bugcrowd]
|
47
|
+
cookie = Bugcrowd::Auth.cookie
|
48
|
+
results['Bugcrowd'] = {}
|
49
|
+
|
50
|
+
Bugcrowd::Programs.sync(results['Bugcrowd'], options, cookie)
|
51
|
+
end
|
52
|
+
|
53
|
+
if options[:hackerone]
|
54
|
+
results['Hackerone'] = {}
|
55
|
+
Hackerone::Programs.sync(results['Hackerone'], options)
|
56
|
+
end
|
57
|
+
|
58
|
+
p results
|
59
|
+
end
|
60
|
+
end
|
metadata
ADDED
@@ -0,0 +1,171 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: scopes_extractor
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Joshua MARTINELLE
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-05-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: colorize
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.8.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.8.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: dotenv
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2.8'
|
34
|
+
- - ">="
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 2.8.1
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - "~>"
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '2.8'
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 2.8.1
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: logger
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1.5'
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 1.5.3
|
57
|
+
type: :runtime
|
58
|
+
prerelease: false
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - "~>"
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '1.5'
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 1.5.3
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
name: mechanize
|
69
|
+
requirement: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - "~>"
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '2.9'
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: 2.9.1
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '2.9'
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: 2.9.1
|
87
|
+
- !ruby/object:Gem::Dependency
|
88
|
+
name: typhoeus
|
89
|
+
requirement: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - "~>"
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '1.4'
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 1.4.0
|
97
|
+
type: :runtime
|
98
|
+
prerelease: false
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1.4'
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: 1.4.0
|
107
|
+
- !ruby/object:Gem::Dependency
|
108
|
+
name: rotp
|
109
|
+
requirement: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - "~>"
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '6.2'
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 6.2.2
|
117
|
+
type: :runtime
|
118
|
+
prerelease: false
|
119
|
+
version_requirements: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - "~>"
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '6.2'
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: 6.2.2
|
127
|
+
description:
|
128
|
+
email:
|
129
|
+
- contact@jomar.fr
|
130
|
+
executables: []
|
131
|
+
extensions: []
|
132
|
+
extra_rdoc_files: []
|
133
|
+
files:
|
134
|
+
- lib/scopes_extractor.rb
|
135
|
+
- lib/scopes_extractor/http_client.rb
|
136
|
+
- lib/scopes_extractor/platforms/bugcrowd/cookie.rb
|
137
|
+
- lib/scopes_extractor/platforms/bugcrowd/programs.rb
|
138
|
+
- lib/scopes_extractor/platforms/bugcrowd/scopes.rb
|
139
|
+
- lib/scopes_extractor/platforms/hackerone/programs.rb
|
140
|
+
- lib/scopes_extractor/platforms/hackerone/scopes.rb
|
141
|
+
- lib/scopes_extractor/platforms/intigriti/programs.rb
|
142
|
+
- lib/scopes_extractor/platforms/intigriti/scopes.rb
|
143
|
+
- lib/scopes_extractor/platforms/intigriti/token.rb
|
144
|
+
- lib/scopes_extractor/platforms/yeswehack/jwt.rb
|
145
|
+
- lib/scopes_extractor/platforms/yeswehack/programs.rb
|
146
|
+
- lib/scopes_extractor/platforms/yeswehack/scopes.rb
|
147
|
+
- lib/scopes_extractor/utilities.rb
|
148
|
+
homepage: https://rubygems.org/gems/scopes_extractor
|
149
|
+
licenses:
|
150
|
+
- MIT
|
151
|
+
metadata: {}
|
152
|
+
post_install_message:
|
153
|
+
rdoc_options: []
|
154
|
+
require_paths:
|
155
|
+
- lib
|
156
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
157
|
+
requirements:
|
158
|
+
- - ">="
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
version: 2.7.1
|
161
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
162
|
+
requirements:
|
163
|
+
- - ">="
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
requirements: []
|
167
|
+
rubygems_version: 3.1.2
|
168
|
+
signing_key:
|
169
|
+
specification_version: 4
|
170
|
+
summary: BugBounty Scopes Extractor
|
171
|
+
test_files: []
|