aquatone 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -1
- data/lib/aquatone/collectors/gtr.rb +9 -14
- data/lib/aquatone/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: baf48eeafa767353c69d4a516f55acf3a8564fe1
|
4
|
+
data.tar.gz: b5abde13dde9cf2cda34c4dca4f8d69184b0b56d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d174b9c82b96992bdc75a8f4ffb9e9d74c93a901ec58478487a28c597d769382a1e7b68cc5bd79a7d0bcc0e54175c74be6e61539f56d06e3d848ef002d52d270
|
7
|
+
data.tar.gz: 062654c4512f11b8abf86b621b47dd6183ffa096eb2bb54b06cfe3fe4c3985a74d0aff9314f64583cc0f83f3314c24cf52ccdc2ac02bb459030ce4f1949b7aaf
|
data/CHANGELOG.md
CHANGED
@@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|
9
9
|
|
10
10
|
### Changed
|
11
11
|
|
12
|
+
## [0.4.1]
|
13
|
+
|
14
|
+
### Added
|
15
|
+
|
16
|
+
### Changed
|
17
|
+
|
18
|
+
### Fixed
|
19
|
+
- The Google Transparency Report collector broke because of server-side changes.
|
12
20
|
|
13
21
|
## [0.4.0]
|
14
22
|
### Added
|
@@ -60,7 +68,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|
60
68
|
|
61
69
|
### Changed
|
62
70
|
|
63
|
-
[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.4.
|
71
|
+
[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.4.1...HEAD
|
72
|
+
[0.4.1]: https://github.com/michenriksen/aquatone/compare/v0.4.0...v0.4.1
|
64
73
|
[0.4.0]: https://github.com/michenriksen/aquatone/compare/v0.3.0...v0.4.0
|
65
74
|
[0.3.0]: https://github.com/michenriksen/aquatone/compare/v0.2.0...v0.3.0
|
66
75
|
[0.2.0]: https://github.com/michenriksen/aquatone/compare/v0.1.1...v0.2.0
|
@@ -11,19 +11,19 @@ module Aquatone
|
|
11
11
|
}
|
12
12
|
}
|
13
13
|
|
14
|
-
BASE_URI = "https://www.google.com/transparencyreport/
|
14
|
+
BASE_URI = "https://www.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch"
|
15
15
|
DEFAULT_PAGES_TO_PROCESS = 30.freeze
|
16
16
|
|
17
17
|
def run
|
18
18
|
token = nil
|
19
19
|
pages_to_process.times do
|
20
20
|
response = parse_response(request_page(token))
|
21
|
-
response[
|
22
|
-
|
21
|
+
hosts = response.first[1].map { |a| a[1] }.uniq
|
22
|
+
hosts.each do |host|
|
23
23
|
add_host(host) if valid_host?(host)
|
24
24
|
end
|
25
|
-
|
26
|
-
token
|
25
|
+
_, token, _, current_page, total_pages = response.first.last
|
26
|
+
break if token.nil? || current_page == total_pages
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
@@ -31,23 +31,18 @@ module Aquatone
|
|
31
31
|
|
32
32
|
def request_page(token = nil)
|
33
33
|
if token.nil?
|
34
|
-
uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&
|
34
|
+
uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true"
|
35
35
|
else
|
36
|
-
uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&
|
36
|
+
uri = "#{BASE_URI}/page?domain=#{url_escape(domain.name)}&include_expired=true&include_subdomains=true&p=#{url_escape(token)}"
|
37
37
|
end
|
38
38
|
|
39
39
|
get_request(uri,
|
40
|
-
{ :headers => { "Referer" => "https://
|
40
|
+
{ :format => :plain, :headers => { "Referer" => "https://transparencyreport.google.com/https/certificates" } }
|
41
41
|
)
|
42
42
|
end
|
43
43
|
|
44
|
-
def random_jsonp_callback
|
45
|
-
"abcdefghijklmnopqrstuvwxyz0123456789".split("").sample(9).join
|
46
|
-
end
|
47
|
-
|
48
44
|
def parse_response(body)
|
49
|
-
body = body.split("
|
50
|
-
body.gsub!(");", "")
|
45
|
+
body = body.split("\n", 2).last.strip
|
51
46
|
JSON.parse(body)
|
52
47
|
end
|
53
48
|
|
data/lib/aquatone/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aquatone
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Henriksen
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: httparty
|
@@ -185,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
185
185
|
version: '0'
|
186
186
|
requirements: []
|
187
187
|
rubyforge_project:
|
188
|
-
rubygems_version: 2.6.
|
188
|
+
rubygems_version: 2.6.13
|
189
189
|
signing_key:
|
190
190
|
specification_version: 4
|
191
191
|
summary: A tool for domain flyovers.
|