scopes_extractor 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2720124a1d479958f99a1e40d303f5a8fa1fb45727b7c575197561522245cd3f
-  data.tar.gz: bc579dc76168344ac983876d54db4b55977099e6488fbe26a0971bf85488af35
+  metadata.gz: 95cac573681be20212d7983aa2dbc9d6f7764c5454d3289eaa0b1dc8029183ed
+  data.tar.gz: e05c83b53ae0980ab98f2a2d67b2f92b2ca57ae80acd851ce3de04c07e85bd6e
 SHA512:
-  metadata.gz: 9b5387a228e502be7cd36c5ea4e67151ece0a046406931619c159c17d4345069b0885b5de27683a63190b5ac0a108023dc7524e406e836179eb12cf820c6be41
-  data.tar.gz: 784560b6ed8def4699d7c3e03a34ee02ef5db707183acc7d702e3c058cf0bb0df0c38ceca648609f24cb79941d10dd78bc1f64b8dd4583098bef70dcb19c6f78
+  metadata.gz: a087cc7e23fc0412556587194f25ffbfd1d2c94032a45aa99bf332868cef27e12eda2c9615cd43a8f14c26b0a2e59238ae33d610a2b754fcf5d46681c23b63d5
+  data.tar.gz: 7d3af7abf76821b52340ddd6eb0129c04d370f26aa4f07be8752f7b40868757c3c907191556e6a42c98afd57b70b71d2db952aa262986c86119954d32aa61c22

data/lib/scopes_extractor/http_client.rb

@@ -2,10 +2,12 @@
 
 # HttpClient Class
 class HttpClient
-  @request_options = {
-    ssl_verifypeer: false,
-    ssl_verifyhost: 0
-  }
+  def self.request_options
+    {
+      ssl_verifypeer: false,
+      ssl_verifyhost: 0
+    }
+  end
 
   def self.headers(url, authentication)
     if url.include?('yeswehack')
@@ -15,23 +17,25 @@ class HttpClient
     elsif url.include?('bugcrowd')
       { 'Cookie' => authentication }
     elsif url.include?('hackerone')
-      @request_options[:userpwd] = "#{ENV.fetch('H1_USERNAME', nil)}:#{ENV.fetch('H1_API_KEY', nil)}"
-      { 'Accept' => 'application/json' }
+      h1_credz = Base64.urlsafe_encode64("#{ENV.fetch('H1_USERNAME', nil)}:#{ENV.fetch('H1_API_KEY', nil)}")
+      { 'Accept' => 'application/json', 'Authorization' => "Basic #{h1_credz}" }
     else
       { 'Content-Type' => 'application/json' }
     end
   end
 
   def self.get(url, authentication = nil)
-    @request_options[:headers] = headers(url, authentication)
+    options = request_options
+    options[:headers] = headers(url, authentication)
 
-    Typhoeus.get(url, @request_options)
+    Typhoeus.get(url, options)
   end
 
   def self.post(url, data)
-    @request_options[:headers] = { 'Content-Type' => 'application/json' }
-    @request_options[:body] = data
+    options = request_options
+    options[:headers] = { 'Content-Type' => 'application/json' }
+    options[:body] = data
 
-    Typhoeus.post(url, @request_options)
+    Typhoeus.post(url, options)
   end
 end

data/lib/scopes_extractor/platforms/intigriti/scopes.rb

@@ -27,12 +27,18 @@ class Intigriti
       scopes_normalized = []
 
       scopes.each do |scope|
-        next unless scope['type'] == 1 # 1 == Web Application
-
-        endpoint = normalize(scope['endpoint'])
-        next if exclusions.any? { |exclusion| endpoint.include?(exclusion) } || !endpoint.include?('.')
-
-        scopes_normalized << endpoint
+        next unless scope['type'] == 1 || scope['type'] == 6 # 1 == Web Application || 6 == Other
+
+        if scope['type'] == 1 # Web Application
+          endpoint = normalize(scope['endpoint'])
+          scopes_normalized << endpoint unless exclusions.any? { |exclusion| endpoint.include?(exclusion) } || !endpoint.include?('.')
+        end
+
+        endpoints_description = extract_description(scope['description'])
+        endpoints_description&.each do |endpoint_description|
+          endpoint_description = normalize(endpoint_description)
+          scopes_normalized << endpoint_description unless exclusions.any? { |exclusion| endpoint_description.include?(exclusion) } || !endpoint_description.include?('.')
+        end
       end
 
       scopes_normalized
@@ -42,5 +48,14 @@ class Intigriti
       endpoint.gsub('/*', '').gsub(' ', '').sub('.*', '.com').sub('.<tld>', '.com')
               .sub(%r{/$}, '').sub(/\*$/, '')
     end
+
+    def self.extract_description(description)
+      return [] unless description
+
+      match = description.match(/In Scope(.*)Out of Scope/)
+      return unless match && match[1]
+
+      match[1].scan(/\*\.[\w.-]+/)
+    end
   end
 end

data/lib/scopes_extractor.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'base64'
 require 'dotenv'
 require 'json'
 require 'rotp'
@@ -56,6 +57,8 @@ class ScopesExtractor
       Hackerone::Programs.sync(results['Hackerone'], options)
     end
 
+    File.open('extract.json', 'w') { |f| f.write(JSON.pretty_generate(results)) } if options[:save]
+
     results
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scopes_extractor
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Joshua MARTINELLE
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-16 00:00:00.000000000 Z
+date: 2023-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize