scopes_extractor 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53708b2ee96a050695c28add808c5b8bf8cdfb7e6497aa5d35f0aff99fe7cca3
-  data.tar.gz: 1c598a6d8de2708c3b8431e84ac8456dde7f9e4f468d69e2c29e9e36ea26b4f0
+  metadata.gz: 97fa26671708b0b7dbf28fbcfc431405e3a417022c17b877bd0c060afc654b36
+  data.tar.gz: 70b3d8febc68244cce74ff7ae7275db58071e6bf3ee10daa5d730d080bbbb812
 SHA512:
-  metadata.gz: 74947fc21e47e8373e92598b38f856241f837051f86d9de70cd5a857ee12daf6c1ecc111d69ff3652ca1a459bd66c69deb3989ff7d23f35ff65580075a861c16
-  data.tar.gz: e34117a9518c303601948111c24beafb2c3f4b594891c11eef01240a19bcc76ee94e4ce0670da6323db796553d8c85a0dfdce3bbf6fcd6ae42d17664b88308a6
+  metadata.gz: a58a4210168e3b7d4d85ecba63babef6bf5bef0cd7baee972ba366589f96d393faee0bf0df5168cc96a9c8e394439d831f5fac82baf9eb9c1bf2535e1cf99bfa
+  data.tar.gz: 5dfae355baf7da881c47b56abbf8a0d040bf961c52d8ab7678b0869c7e13c175542219c144f447beaf4009898d8a578804076c446d490d3759d07ea750b50383

data/lib/scopes_extractor/platforms/bugcrowd/programs.rb

@@ -6,9 +6,10 @@ class Bugcrowd
   # Bugcrowd Sync Programs
   class Programs
     def self.sync(results, options, cookie, page_id = 1)
-      response = HttpClient.get(
-        "https://bugcrowd.com/programs.json?page[]=#{page_id}&waitlistable[]=false&joinable[]=false", cookie
-      )
+      url = "https://bugcrowd.com/programs.json?page[]=#{page_id}&waitlistable[]=false&joinable[]=false"
+      url += "&vdp[]=false" if options[:skip_vdp]
+
+      response = HttpClient.get(url, cookie)
       return unless response&.code == 200
 
       body = JSON.parse(response.body)
@@ -20,7 +21,6 @@ class Bugcrowd
     def self.parse_programs(programs, options, results, cookie)
       programs.each do |program|
         next if program['status'] == 4 # Disabled
-        next if program['min_rewards'].nil? && options[:skip_vdp]
 
         results[program['name']] = program_info(program)
         results[program['name']]['scopes'] = Scopes.sync(program_info(program), cookie)

data/lib/scopes_extractor/platforms/bugcrowd/scopes.rb

@@ -37,7 +37,8 @@ class Bugcrowd
         next if exclusions.any? { |exclusion| endpoint.include?(exclusion) } || !endpoint.include?('.')
         next if endpoint.include?('*') && !endpoint.start_with?('*.')
 
-        scopes_normalized << endpoint
+        endpoint.sub!(%r{/$}, '')
+        scopes_normalized << endpoint.sub('/*', '')
       end
 
       scopes_normalized

data/lib/scopes_extractor/platforms/hackerone/programs.rb

@@ -14,7 +14,7 @@ class Hackerone
         next if options[:skip_vdp] && !program['attributes']['offers_bounties']
 
         results[program['attributes']['name']] = program_info(program)
-        results[program['attributes']['name']]['scopes'] = Scopes.sync(program_info(program))
+        results[program['attributes']['name']]['scopes'] = Scopes.sync(program_info(program), options)
       end
 
       sync(results, options, page_id + 1) if programs_infos[:next_page]

data/lib/scopes_extractor/platforms/hackerone/scopes.rb

@@ -3,24 +3,26 @@
 class Hackerone
   # Hackerone Sync Programs
   class Scopes
-    def self.sync(program)
+    def self.sync(program, options)
       scopes = {}
       response = HttpClient.get("https://api.hackerone.com/v1/hackers/programs/#{program[:slug]}")
       return scopes unless response&.code == 200
 
       in_scopes = JSON.parse(response.body)['relationships']['structured_scopes']['data']
-      scopes['in'] = parse_scopes(in_scopes)
+      scopes['in'] = parse_scopes(in_scopes, options)
 
       scopes['out'] = {} # TODO
 
       scopes
     end
 
-    def self.parse_scopes(scopes)
+    def self.parse_scopes(scopes, options)
       scopes_normalized = []
 
       scopes.each do |scope|
-        next unless scope['attributes']['asset_type'] == 'URL'
+        next if scope['attributes']['eligible_for_submission'] == false ||
+          (scope['attributes']['eligible_for_bounty'] == false && options[:skip_vdp])
+        next unless %w[URL WILDCARD].any?(scope['attributes']['asset_type'])
 
         endpoint = scope['attributes']['asset_identifier']
         normalized = normalized(endpoint)
@@ -28,8 +30,9 @@ class Hackerone
         normalized.each do |asset|
           next unless asset.include?('.')
           next if asset.include?('*') && !asset.start_with?('*.')
+          next unless asset.match?(/\w\./)
 
-          scopes_normalized << asset
+          scopes_normalized << asset.sub('/*', '')
         end
       end
 
@@ -42,9 +45,9 @@ class Hackerone
       normalized = []
 
       if endpoint.include?(',')
-        endpoint.split(',').each { |asset| normalized << asset }
+        endpoint.split(',').each { |asset| normalized << asset.sub('/*', '') }
       else
-        normalized << endpoint
+        normalized << endpoint.sub('/*', '')
       end
 
       normalized

data/lib/scopes_extractor/platforms/yeswehack/scopes.rb

@@ -28,7 +28,7 @@ class YesWeHack
           next unless asset.include?('.')
           next if asset.include?('*') && !asset.start_with?('*.')
 
-          scopes_normalized << asset
+          scopes_normalized << asset.sub('/*', '')
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scopes_extractor
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Joshua MARTINELLE
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-23 00:00:00.000000000 Z
+date: 2023-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -164,7 +164,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: BugBounty Scopes Extractor