aquatone 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8ad3da3d450ecd7fd41c64cd0b18f0a3e3ae00b
-  data.tar.gz: 240fbce3dc353f146a32be28566201b2b62fd2c6
+  metadata.gz: baf48eeafa767353c69d4a516f55acf3a8564fe1
+  data.tar.gz: b5abde13dde9cf2cda34c4dca4f8d69184b0b56d
 SHA512:
-  metadata.gz: 244cc756b2cb1c65d36281fbeae199d290a71434f73a12c96e855995eff2a091aae442212d13ee604ac6b34d6d74196813ff3b01bd5b809b16512c86b438c44e
-  data.tar.gz: b109618403ee14536a67a8e75334e88279578f2eb9532edd8ea772c7bf1e443a8747a01038eadfc44bd5a42000ab90571be269432340eef1257161f6d3d40110
+  metadata.gz: d174b9c82b96992bdc75a8f4ffb9e9d74c93a901ec58478487a28c597d769382a1e7b68cc5bd79a7d0bcc0e54175c74be6e61539f56d06e3d848ef002d52d270
+  data.tar.gz: 062654c4512f11b8abf86b621b47dd6183ffa096eb2bb54b06cfe3fe4c3985a74d0aff9314f64583cc0f83f3314c24cf52ccdc2ac02bb459030ce4f1949b7aaf

data/CHANGELOG.md

@@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
+## [0.4.1]
+
+### Added
+
+### Changed
+
+### Fixed
+ - The Google Transparency Report collector broke because of server-side changes.
 
 ## [0.4.0]
 ### Added
@@ -60,7 +68,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
-[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.4.0...HEAD
+[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.4.1...HEAD
+[0.4.1]: https://github.com/michenriksen/aquatone/compare/v0.4.0...v0.4.1
 [0.4.0]: https://github.com/michenriksen/aquatone/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/michenriksen/aquatone/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/michenriksen/aquatone/compare/v0.1.1...v0.2.0

data/lib/aquatone/collectors/gtr.rb

@@ -11,19 +11,19 @@ module Aquatone
         }
       }
 
-      BASE_URI                 = "https://www.google.com/transparencyreport/jsonp/ct/search"
+      BASE_URI                 = "https://www.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch"
       DEFAULT_PAGES_TO_PROCESS = 30.freeze
 
       def run
         token = nil
         pages_to_process.times do
           response = parse_response(request_page(token))
-          response["results"].each do |result|
-            host = result["subject"]
+          hosts    = response.first[1].map { |a| a[1] }.uniq
+          hosts.each do |host|
             add_host(host) if valid_host?(host)
           end
-          break if !response.key?("nextPageToken")
-          token = response["nextPageToken"]
+          _, token, _, current_page, total_pages = response.first.last
+          break if token.nil? || current_page == total_pages
         end
       end
 
@@ -31,23 +31,18 @@ module Aquatone
 
       def request_page(token = nil)
         if token.nil?
-          uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&incl_exp=true&incl_sub=true&c=_callbacks_._#{random_jsonp_callback}"
+          uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true"
         else
-          uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&incl_exp=true&incl_sub=true&token=#{url_escape(token)}&c=_callbacks_._#{random_jsonp_callback}"
+          uri = "#{BASE_URI}/page?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true&p=#{url_escape(token)}"
         end
 
         get_request(uri,
-          { :headers => { "Referer" => "https://www.google.com/transparencyreport/https/ct/?hl=en-US" } }
+          { :format => :plain, :headers => { "Referer" => "https://transparencyreport.google.com/https/certificates" } }
         )
       end
 
-      def random_jsonp_callback
-        "abcdefghijklmnopqrstuvwxyz0123456789".split("").sample(9).join
-      end
-
       def parse_response(body)
-        body = body.split("(", 2).last
-        body.gsub!(");", "")
+        body = body.split("\n", 2).last.strip
         JSON.parse(body)
       end
 

data/lib/aquatone/version.rb

@@ -1,3 +1,3 @@
 module Aquatone
-  VERSION = "0.4.0".freeze
+  VERSION = "0.4.1".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aquatone
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Michael Henriksen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-03 00:00:00.000000000 Z
+date: 2017-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -185,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: A tool for domain flyovers.