scopes_extractor 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 694da3bb3d2fd3ed580db8d5afa0da7bac2c2326037ed4218b57d87eca7ca174
-  data.tar.gz: 100836692c414cd682653231a65a5937dc8b201560bbf10fa95e96e38d9b39a8
+  metadata.gz: 9213285da286cb2ad488374dc8e5971d34bb06987b710c2bbd4dcac12f3d1384
+  data.tar.gz: af84cd2c4e8e8a43ccb40f126bc1996e812bf66b828e2a3b4aea6bd5dcba9531
 SHA512:
-  metadata.gz: 1a3f2040079e8e05cfd2157b259f95b8912813f8f7ec2e867fd0ce9f7802a05a3b46c63027b5493a0d79a2b28f536b67686b9fb3b5d71926e70de31b846764ca
-  data.tar.gz: 53e92c8b6bcf0608c6c14d6e4f4733e29a8c86a930b7a25de1b6d5242c45d8e7eb38df8d0ebbf50a4c4d49d50e2c5c6286d0609d0140fc44d2d72ba21132466a
+  metadata.gz: 20bbb233618c27774be21ab80ac56e8e9ac2b6e749922bdb1768c634b30f7b7b04f23e622f0a958ddc257e6ed5ef1239fb94aec973eafbfe0c20c3300c831d60
+  data.tar.gz: 5612a5f74b44fd9904e0f7b3ba22218104ee68e36ca27b15dbcaf8f437e22e74f680f997a832d64ddf0580f4f0d8421baac3532aee8537a8c587f48c45bf7369

data/lib/scopes_extractor/http_client.rb

@@ -2,37 +2,40 @@
 
 # HttpClient Class
 class HttpClient
-  @request_options = {
-    ssl_verifypeer: false,
-    ssl_verifyhost: 0
-  }
+  def self.request_options
+    {
+      ssl_verifypeer: false,
+      ssl_verifyhost: 0
+    }
+  end
 
   def self.headers(url, authentication)
-    case
-    when url.include?('yeswehack')
+    if url.include?('yeswehack')
       { 'Content-Type' => 'application/json', Authorization: "Bearer #{authentication}" }
-    when url.include?('intigriti')
+    elsif url.include?('intigriti')
       { Authorization: "Bearer #{authentication}" }
-    when url.include?('bugcrowd')
+    elsif url.include?('bugcrowd')
       { 'Cookie' => authentication }
-    when url.include?('hackerone')
-      @request_options[:userpwd] = "#{ENV.fetch('H1_USERNAME', nil)}:#{ENV.fetch('H1_API_KEY', nil)}"
-      { 'Accept' => 'application/json' }
+    elsif url.include?('hackerone')
+      h1_credz = Base64.urlsafe_encode64("#{ENV.fetch('H1_USERNAME', nil)}:#{ENV.fetch('H1_API_KEY', nil)}")
+      { 'Accept' => 'application/json', 'Authorization' => "Basic #{h1_credz}" }
     else
       { 'Content-Type' => 'application/json' }
     end
   end
 
   def self.get(url, authentication = nil)
-    @request_options[:headers] = headers(url, authentication)
+    options = request_options
+    options[:headers] = headers(url, authentication)
 
-    Typhoeus.get(url, @request_options)
+    Typhoeus.get(url, options)
   end
 
   def self.post(url, data)
-    @request_options[:headers] = { 'Content-Type' => 'application/json' }
-    @request_options[:body] = data
+    options = request_options
+    options[:headers] = { 'Content-Type' => 'application/json' }
+    options[:body] = data
 
-    Typhoeus.post(url, @request_options)
+    Typhoeus.post(url, options)
   end
-end
+end

data/lib/scopes_extractor/platforms/bugcrowd/cookie.rb

@@ -39,4 +39,4 @@ class Bugcrowd
       match[0]
     end
   end
-end
+end

data/lib/scopes_extractor/platforms/bugcrowd/programs.rb

@@ -6,7 +6,9 @@ class Bugcrowd
   # Bugcrowd Sync Programs
   class Programs
     def self.sync(results, options, cookie, page_id = 1)
-      response = HttpClient.get("https://bugcrowd.com/programs.json?page[]=#{page_id}&waitlistable[]=false&joinable[]=false", cookie)
+      response = HttpClient.get(
+        "https://bugcrowd.com/programs.json?page[]=#{page_id}&waitlistable[]=false&joinable[]=false", cookie
+      )
       return unless response&.code == 200
 
       body = JSON.parse(response.body)
@@ -33,4 +35,4 @@ class Bugcrowd
       }
     end
   end
-end
+end

data/lib/scopes_extractor/platforms/bugcrowd/scopes.rb

@@ -21,7 +21,7 @@ class Bugcrowd
       in_scopes = JSON.parse(response.body)['targets']
       scopes['in'] = parse_scopes(in_scopes)
 
-      scopes['out'] = { } # TODO
+      scopes['out'] = {} # TODO
 
       scopes
     end

data/lib/scopes_extractor/platforms/hackerone/programs.rb

@@ -40,4 +40,4 @@ class Hackerone
       { next_page: json_body['links']['next'], programs: json_body['data'] }
     end
   end
-end
+end

data/lib/scopes_extractor/platforms/hackerone/scopes.rb

@@ -5,13 +5,13 @@ class Hackerone
   class Scopes
     def self.sync(program)
       scopes = {}
-      response = HttpClient.get( "https://api.hackerone.com/v1/hackers/programs/#{program[:slug]}")
+      response = HttpClient.get("https://api.hackerone.com/v1/hackers/programs/#{program[:slug]}")
       return scopes unless response&.code == 200
 
       in_scopes = JSON.parse(response.body)['relationships']['structured_scopes']['data']
       scopes['in'] = parse_scopes(in_scopes)
 
-      scopes['out'] = { } # TODO
+      scopes['out'] = {} # TODO
 
       scopes
     end
@@ -47,4 +47,4 @@ class Hackerone
       normalized
     end
   end
-end
+end

data/lib/scopes_extractor/platforms/intigriti/programs.rb

@@ -19,7 +19,8 @@ class Intigriti
         next if program['status'] == 4 # Suspended
 
         results[program['name']] = program_info(program)
-        results[program['name']]['scopes'] = Scopes.sync({ handle: program['handle'], company: program['companyHandle'] }, token)
+        results[program['name']]['scopes'] =
+          Scopes.sync({ handle: program['handle'], company: program['companyHandle'] }, token)
       end
     end
 

data/lib/scopes_extractor/platforms/intigriti/token.rb

@@ -30,7 +30,7 @@ class Intigriti
     end
 
     def self.submit_otp(mechanize)
-      return if ENV['INTIGRITI_OTP']&.empty?
+      return if ENV['INTIGRITI_OTP'] && ENV['INTIGRITI_OTP'].empty?
 
       totp_page = mechanize.get('https://login.intigriti.com/account/loginwith2fa')
       totp_code = ROTP::TOTP.new(ENV.fetch('INTI_OTP', nil))
@@ -51,4 +51,4 @@ class Intigriti
       token_page.body&.undump
     end
   end
-end
+end

data/lib/scopes_extractor/platforms/yeswehack/jwt.rb

@@ -4,7 +4,7 @@ class YesWeHack
   # YesWeHack Auth Class
   class Auth
     def self.jwt
-      totp_token = get_totp_token
+      totp_token = extract_totp_token
       return unless totp_token
 
       response = send_totp(totp_token)
@@ -16,7 +16,7 @@ class YesWeHack
       jwt
     end
 
-    def self.get_totp_token
+    def self.extract_totp_token
       data = { email: ENV.fetch('YWH_EMAIL', nil), password: ENV.fetch('YWH_PASSWORD', nil) }.to_json
       response = HttpClient.post('https://api.yeswehack.com/login', data)
       return unless response&.code == 200
@@ -32,4 +32,4 @@ class YesWeHack
       response
     end
   end
-end
+end

data/lib/scopes_extractor/platforms/yeswehack/programs.rb

@@ -39,4 +39,4 @@ class YesWeHack
       }
     end
   end
-end
+end

data/lib/scopes_extractor/platforms/yeswehack/scopes.rb

@@ -25,6 +25,8 @@ class YesWeHack
 
         normalized = normalize(infos['scope'])
         normalized.each do |asset|
+          next unless asset.include?('.')
+
           scopes_normalized << asset
         end
       end
@@ -32,25 +34,25 @@ class YesWeHack
       scopes_normalized
     end
 
-    # rubocop:disable Metrics/AbcSize
-    # rubocop:disable Metrics/MethodLength
-    # rubocop:disable Metrics/PerceivedComplexity
-    # rubocop:disable Metrics/CyclomaticComplexity
     def self.normalize(scope)
       # Remove (+++) & When end with '*'
       scope = scope.gsub(/\(?\+\)?/, '').sub(/\*$/, '').strip
       return [] if scope.include?('<') # <yourdomain>-yeswehack.domain.tld
 
+      scope = scope.split[0] # When spaces
+
       normalized = []
 
-      match = scope.match(/^(.*)\((.*)\)$/) # Ex: *.lazada.(sg|vn|co.id|co.th|com|com.ph|com.my)
-      if match && match[1] && match[2]
-        tlds = match[2].split('|')
-        tlds.each { |tld| normalized << "#{match[1]}#{tld}" }
-      elsif scope.include?(' ')
-        normalized << scope.split(' ')[0]
+      multi_subs = scope.match(/^\((.*)\)(.*)/) # Ex: (online|portal|agents|agentuat|surinameuat|surinameopsuat|suriname|thailandevoa).vfsevisa.com
+      multi_tld = scope.match(/^(.*)\((.*)\)$/) # Ex: *.lazada.(sg|vn|co.id|co.th|com|com.ph|com.my)
+      if multi_tld && multi_tld[1] && multi_tld[2]
+        tlds = multi_tld[2].split('|')
+        tlds.each { |tld| normalized << "#{multi_tld[1]}#{tld}" }
       elsif scope.match?(%r{https?://\*})
         normalized << scope.sub(%r{https?://}, '')
+      elsif multi_subs && multi_subs[1] && multi_subs[2]
+        subs = multi_subs[1].split('|')
+        subs.each { |sub| normalized << "#{sub}#{multi_subs[2]}"}
       else
         normalized << scope
       end
@@ -58,4 +60,4 @@ class YesWeHack
       normalized
     end
   end
-end
+end

data/lib/scopes_extractor.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'base64'
 require 'dotenv'
 require 'json'
 require 'rotp'
@@ -31,8 +32,8 @@ class ScopesExtractor
 
     if options[:yeswehack]
       jwt = YesWeHack::Auth.jwt
-
       results['YesWeHack'] = {}
+
       YesWeHack::Programs.sync(results['YesWeHack'], options, jwt)
     end
 
@@ -52,9 +53,12 @@ class ScopesExtractor
 
     if options[:hackerone]
       results['Hackerone'] = {}
+
       Hackerone::Programs.sync(results['Hackerone'], options)
     end
 
-    p results
+    File.open('extract.json', 'w') { |f| f.write(JSON.pretty_generate(results)) } if options[:save]
+
+    results
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scopes_extractor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Joshua MARTINELLE
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-13 00:00:00.000000000 Z
+date: 2023-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -85,45 +85,45 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 2.9.1
 - !ruby/object:Gem::Dependency
-  name: typhoeus
+  name: rotp
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '6.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 6.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '6.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 6.2.2
 - !ruby/object:Gem::Dependency
-  name: rotp
+  name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '1.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.2.2
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '1.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.2.2
+        version: 1.4.0
 description:
 email:
 - contact@jomar.fr