miteru 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -2
- data/README.md +1 -1
- data/lib/miteru.rb +1 -0
- data/lib/miteru/cli.rb +1 -5
- data/lib/miteru/crawler.rb +6 -67
- data/lib/miteru/downloader.rb +3 -0
- data/lib/miteru/feeds.rb +47 -0
- data/lib/miteru/feeds/feed.rb +20 -0
- data/lib/miteru/feeds/openphish.rb +19 -0
- data/lib/miteru/feeds/phishtank.rb +20 -0
- data/lib/miteru/feeds/urlscan.rb +26 -0
- data/lib/miteru/http_client.rb +13 -4
- data/lib/miteru/version.rb +1 -1
- data/miteru.gemspec +3 -3
- metadata +13 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89ed6dea77f4809ef7cacfd7543a35d96f8032807d6cc93fe5e6aa52e752d5d1
|
4
|
+
data.tar.gz: 812d51bdbc1c245c87f7a3fcc7e70577536cc8eacd0cfc9d10b6be3b146b96d9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cb94419c132e650a1e57f3bccd134b91be5847ab9197ceb69c71a08b8bfec3667a5e63f504892ef4d08a74f049f4bc979de4ac7c145a2a30502ed92b91f73c78
|
7
|
+
data.tar.gz: 60b012761f7eced3acf58a7acf4487b2d0466a9ebc2fa1b9fe9b23e3b315b3ed4d5bb798e1786788c904691656c989f7ddda962a52ca307abe4e762d7281ad65
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -42,7 +42,7 @@ Options:
|
|
42
42
|
[--download-to=DOWNLOAD_TO] # Directory to download file(s)
|
43
43
|
# Default: /tmp
|
44
44
|
[--post-to-slack], [--no-post-to-slack] # Post a message to Slack if it detects a phishing kit
|
45
|
-
[--size=N] # Number of urlscan.io's results. (Max:
|
45
|
+
[--size=N] # Number of urlscan.io's results. (Max: 10,000)
|
46
46
|
# Default: 100
|
47
47
|
[--threads=N] # Number of threads to use
|
48
48
|
# Default: 10
|
data/lib/miteru.rb
CHANGED
data/lib/miteru/cli.rb
CHANGED
@@ -1,9 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "colorize"
|
4
|
-
require "digest"
|
5
|
-
require "fileutils"
|
6
|
-
require "http"
|
7
3
|
require "thor"
|
8
4
|
|
9
5
|
module Miteru
|
@@ -12,7 +8,7 @@ module Miteru
|
|
12
8
|
method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
|
13
9
|
method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
|
14
10
|
method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
|
15
|
-
method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max:
|
11
|
+
method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
|
16
12
|
method_option :threads, type: :numeric, default: 10, desc: "Number of threads to use"
|
17
13
|
method_option :verbose, type: :boolean, default: true
|
18
14
|
desc "execute", "Execute the crawler"
|
data/lib/miteru/crawler.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "
|
4
|
-
require "http"
|
5
|
-
require "json"
|
3
|
+
require "colorize"
|
6
4
|
require "parallel"
|
7
5
|
require "uri"
|
8
6
|
|
@@ -11,14 +9,11 @@ module Miteru
|
|
11
9
|
attr_reader :auto_download
|
12
10
|
attr_reader :directory_traveling
|
13
11
|
attr_reader :downloader
|
12
|
+
attr_reader :feeds
|
14
13
|
attr_reader :size
|
15
14
|
attr_reader :threads
|
16
15
|
attr_reader :verbose
|
17
16
|
|
18
|
-
URLSCAN_ENDPOINT = "https://urlscan.io/api/v1"
|
19
|
-
OPENPHISH_ENDPOINT = "https://openphish.com"
|
20
|
-
PHISHTANK_ENDPOINT = "http://data.phishtank.com"
|
21
|
-
|
22
17
|
def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
|
23
18
|
@auto_download = auto_download
|
24
19
|
@directory_traveling = directory_traveling
|
@@ -27,63 +22,14 @@ module Miteru
|
|
27
22
|
@size = size
|
28
23
|
@threads = threads
|
29
24
|
@verbose = verbose
|
30
|
-
raise ArgumentError, "size must be less than 100,000" if size > 100_000
|
31
|
-
end
|
32
|
-
|
33
|
-
def urlscan_feed
|
34
|
-
url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
|
35
|
-
res = JSON.parse(get(url))
|
36
|
-
res["results"].map { |result| result.dig("task", "url") }
|
37
|
-
rescue HTTPResponseError => _
|
38
|
-
[]
|
39
|
-
end
|
40
|
-
|
41
|
-
def openphish_feed
|
42
|
-
res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
|
43
|
-
res.lines.map(&:chomp)
|
44
|
-
rescue HTTPResponseError => _
|
45
|
-
[]
|
46
|
-
end
|
47
|
-
|
48
|
-
def phishtank_feed
|
49
|
-
res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
|
50
|
-
table = CSV.parse(res, headers: true)
|
51
|
-
table.map { |row| row["url"] }
|
52
|
-
rescue HTTPResponseError => _
|
53
|
-
[]
|
54
|
-
end
|
55
|
-
|
56
|
-
def breakdown(url)
|
57
|
-
begin
|
58
|
-
uri = URI.parse(url)
|
59
|
-
rescue URI::InvalidURIError => _
|
60
|
-
return []
|
61
|
-
end
|
62
|
-
|
63
|
-
base = "#{uri.scheme}://#{uri.hostname}"
|
64
|
-
return [base] unless directory_traveling
|
65
|
-
|
66
|
-
segments = uri.path.split("/")
|
67
|
-
return [base] if segments.length.zero?
|
68
25
|
|
69
|
-
|
70
|
-
urls.reject do |breakdowned_url|
|
71
|
-
# Reject a url which ends with specific extension names
|
72
|
-
%w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def suspicious_urls
|
77
|
-
@suspicious_urls ||= [].tap do |arr|
|
78
|
-
urls = (urlscan_feed + openphish_feed + phishtank_feed).select { |url| url.start_with?("http://", "https://") }
|
79
|
-
urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
|
80
|
-
end
|
26
|
+
@feeds = Feeds.new(size, directory_traveling: directory_traveling)
|
81
27
|
end
|
82
28
|
|
83
29
|
def execute
|
84
|
-
puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
|
30
|
+
puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl." if verbose
|
85
31
|
|
86
|
-
Parallel.each(suspicious_urls, in_threads: threads) do |url|
|
32
|
+
Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
|
87
33
|
website = Website.new(url)
|
88
34
|
if website.has_kit?
|
89
35
|
message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
|
@@ -115,7 +61,7 @@ module Miteru
|
|
115
61
|
channel = ENV["SLACK_CHANNEL"] || "#general"
|
116
62
|
|
117
63
|
payload = { text: message, channel: channel }
|
118
|
-
|
64
|
+
HTTPClient.post(webhook_url, json: payload)
|
119
65
|
end
|
120
66
|
|
121
67
|
def post_to_slack?
|
@@ -130,13 +76,6 @@ module Miteru
|
|
130
76
|
ENV["SLACK_WEBHOOK_URL"] != nil
|
131
77
|
end
|
132
78
|
|
133
|
-
private
|
134
79
|
|
135
|
-
def get(url)
|
136
|
-
res = HTTP.follow(max_hops: 3).get(url)
|
137
|
-
raise HTTPResponseError if res.code != 200
|
138
|
-
|
139
|
-
res.body.to_s
|
140
|
-
end
|
141
80
|
end
|
142
81
|
end
|
data/lib/miteru/downloader.rb
CHANGED
data/lib/miteru/feeds.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "./feeds/feed"
|
4
|
+
require_relative "./feeds/openphish"
|
5
|
+
require_relative "./feeds/phishtank"
|
6
|
+
require_relative "./feeds/urlscan"
|
7
|
+
|
8
|
+
module Miteru
|
9
|
+
class Feeds
|
10
|
+
attr_reader :openphish, :phishtank, :urlscan
|
11
|
+
attr_reader :directory_traveling
|
12
|
+
|
13
|
+
def initialize(urlscan_size = 100, directory_traveling: false)
|
14
|
+
@openphish = OpenPhish.new
|
15
|
+
@phishtank = PhishTank.new
|
16
|
+
@urlscan = UrlScan.new(urlscan_size)
|
17
|
+
@directory_traveling = directory_traveling
|
18
|
+
end
|
19
|
+
|
20
|
+
def suspicious_urls
|
21
|
+
@suspicious_urls ||= [].tap do |arr|
|
22
|
+
urls = (openphish.urls + phishtank.urls + urlscan.urls).select { |url| url.start_with?("http://", "https://") }
|
23
|
+
urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def breakdown(url)
|
28
|
+
begin
|
29
|
+
uri = URI.parse(url)
|
30
|
+
rescue URI::InvalidURIError => _
|
31
|
+
return []
|
32
|
+
end
|
33
|
+
|
34
|
+
base = "#{uri.scheme}://#{uri.hostname}"
|
35
|
+
return [base] unless directory_traveling
|
36
|
+
|
37
|
+
segments = uri.path.split("/")
|
38
|
+
return [base] if segments.length.zero?
|
39
|
+
|
40
|
+
urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
|
41
|
+
urls.reject do |breakdowned_url|
|
42
|
+
# Reject a url which ends with specific extension names
|
43
|
+
%w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Miteru
|
4
|
+
class Feeds
|
5
|
+
class Feed
|
6
|
+
def urls
|
7
|
+
raise NotImplementedError, "You must implement #{self.class}##{__method__}"
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def get(url)
|
13
|
+
res = HTTPClient.get(url)
|
14
|
+
raise HTTPResponseError if res.code != 200
|
15
|
+
|
16
|
+
res.body.to_s
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
module Miteru
|
6
|
+
class Feeds
|
7
|
+
class OpenPhish < Feed
|
8
|
+
ENDPOINT = "https://openphish.com"
|
9
|
+
|
10
|
+
def urls
|
11
|
+
res = get("#{ENDPOINT}/feed.txt")
|
12
|
+
res.lines.map(&:chomp)
|
13
|
+
rescue HTTPResponseError => e
|
14
|
+
puts "Failed to load OpenPhish feed (#{e})"
|
15
|
+
[]
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "csv"
|
4
|
+
|
5
|
+
module Miteru
|
6
|
+
class Feeds
|
7
|
+
class PhishTank < Feed
|
8
|
+
ENDPOINT = "http://data.phishtank.com"
|
9
|
+
|
10
|
+
def urls
|
11
|
+
res = get("#{ENDPOINT}/data/online-valid.csv")
|
12
|
+
table = CSV.parse(res, headers: true)
|
13
|
+
table.map { |row| row["url"] }
|
14
|
+
rescue HTTPResponseError => e
|
15
|
+
puts "Failed to load PhishTank feed (#{e})"
|
16
|
+
[]
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
module Miteru
|
6
|
+
class Feeds
|
7
|
+
class UrlScan < Feed
|
8
|
+
ENDPOINT = "https://urlscan.io/api/v1"
|
9
|
+
|
10
|
+
attr_reader :size
|
11
|
+
def initialize(size = 100)
|
12
|
+
@size = size
|
13
|
+
raise ArgumentError, "size must be less than 10,000" if size > 10_000
|
14
|
+
end
|
15
|
+
|
16
|
+
def urls
|
17
|
+
url = "#{ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
|
18
|
+
res = JSON.parse(get(url))
|
19
|
+
res["results"].map { |result| result.dig("task", "url") }
|
20
|
+
rescue HTTPResponseError => e
|
21
|
+
puts "Failed to load urlscan.io feed (#{e})"
|
22
|
+
[]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/miteru/http_client.rb
CHANGED
@@ -25,12 +25,21 @@ module Miteru
|
|
25
25
|
new.download(url, base_dir)
|
26
26
|
end
|
27
27
|
|
28
|
-
def get(url)
|
29
|
-
|
28
|
+
def get(url, options = {})
|
29
|
+
options = options.merge default_options
|
30
|
+
HTTP.follow.timeout(write: 2, connect: 5, read: 10).headers(default_headers).get(url, options)
|
30
31
|
end
|
31
32
|
|
32
|
-
def self.get(url)
|
33
|
-
new.get url
|
33
|
+
def self.get(url, options = {})
|
34
|
+
new.get url, options
|
35
|
+
end
|
36
|
+
|
37
|
+
def post(url, options = {})
|
38
|
+
HTTP.post url, options
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.post(url, options = {})
|
42
|
+
new.post url, options
|
34
43
|
end
|
35
44
|
|
36
45
|
private
|
data/lib/miteru/version.rb
CHANGED
data/miteru.gemspec
CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
25
|
spec.require_paths = ["lib"]
|
26
26
|
|
27
|
-
spec.add_development_dependency "bundler", "~> 1.
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.17"
|
28
28
|
spec.add_development_dependency "coveralls", "~> 0.8"
|
29
29
|
spec.add_development_dependency "glint", "~> 0.1"
|
30
30
|
spec.add_development_dependency "rake", "~> 12.3"
|
@@ -33,8 +33,8 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.add_development_dependency "webmock", "~> 3.4"
|
34
34
|
|
35
35
|
spec.add_dependency "colorize", "~> 0.8"
|
36
|
-
spec.add_dependency "down", "~> 4.
|
37
|
-
spec.add_dependency "http", "~>
|
36
|
+
spec.add_dependency "down", "~> 4.6"
|
37
|
+
spec.add_dependency "http", "~> 4.0"
|
38
38
|
spec.add_dependency "oga", "~> 2.15"
|
39
39
|
spec.add_dependency "parallel", "~> 1.12"
|
40
40
|
spec.add_dependency "thor", "~> 0.19"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miteru
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Niseki
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.17'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.17'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: coveralls
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,28 +128,28 @@ dependencies:
|
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: '4.
|
131
|
+
version: '4.6'
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: '4.
|
138
|
+
version: '4.6'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: http
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
143
|
- - "~>"
|
144
144
|
- !ruby/object:Gem::Version
|
145
|
-
version: '
|
145
|
+
version: '4.0'
|
146
146
|
type: :runtime
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
150
|
- - "~>"
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version: '
|
152
|
+
version: '4.0'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: oga
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -215,6 +215,11 @@ files:
|
|
215
215
|
- lib/miteru/crawler.rb
|
216
216
|
- lib/miteru/downloader.rb
|
217
217
|
- lib/miteru/error.rb
|
218
|
+
- lib/miteru/feeds.rb
|
219
|
+
- lib/miteru/feeds/feed.rb
|
220
|
+
- lib/miteru/feeds/openphish.rb
|
221
|
+
- lib/miteru/feeds/phishtank.rb
|
222
|
+
- lib/miteru/feeds/urlscan.rb
|
218
223
|
- lib/miteru/http_client.rb
|
219
224
|
- lib/miteru/version.rb
|
220
225
|
- lib/miteru/website.rb
|