miteru 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a8370829b3ad8da53b7c73070fd972269b09ea27c6c7dce1a3da5d855f81da27
4
- data.tar.gz: c017feecfeaa27912de5dc898f23b430652270a16852510785109d60ed04891a
3
+ metadata.gz: 89ed6dea77f4809ef7cacfd7543a35d96f8032807d6cc93fe5e6aa52e752d5d1
4
+ data.tar.gz: 812d51bdbc1c245c87f7a3fcc7e70577536cc8eacd0cfc9d10b6be3b146b96d9
5
5
  SHA512:
6
- metadata.gz: '0796be92ea1586ec57f12a4157097d65ea0d49e151eb8876eda3b406ca05908b0bc01ff021f270ab5a1467bda415e281909ef47502f1fb490d5285c69dd1bd9b'
7
- data.tar.gz: edbc8c6cce3d85cbfb8bdac217d7ca0ee9d3dfc353a3b7a3c1c7f7ad132b7d9ee80960bda1bba638b859f53d2805a7634901d47fd9a913630b16b0552d463078
6
+ metadata.gz: cb94419c132e650a1e57f3bccd134b91be5847ab9197ceb69c71a08b8bfec3667a5e63f504892ef4d08a74f049f4bc979de4ac7c145a2a30502ed92b91f73c78
7
+ data.tar.gz: 60b012761f7eced3acf58a7acf4487b2d0466a9ebc2fa1b9fe9b23e3b315b3ed4d5bb798e1786788c904691656c989f7ddda962a52ca307abe4e762d7281ad65
@@ -1,7 +1,6 @@
1
- ---
2
1
  sudo: false
3
2
  language: ruby
4
3
  cache: bundler
5
4
  rvm:
6
5
  - 2.5.1
7
- before_install: gem install bundler -v 1.16.4
6
+ before_install: gem install bundler -v 1.17.1
data/README.md CHANGED
@@ -42,7 +42,7 @@ Options:
42
42
  [--download-to=DOWNLOAD_TO] # Directory to download file(s)
43
43
  # Default: /tmp
44
44
  [--post-to-slack], [--no-post-to-slack] # Post a message to Slack if it detects a phishing kit
45
- [--size=N] # Number of urlscan.io's results. (Max: 100,000)
45
+ [--size=N] # Number of urlscan.io's results. (Max: 10,000)
46
46
  # Default: 100
47
47
  [--threads=N] # Number of threads to use
48
48
  # Default: 10
@@ -4,6 +4,7 @@ require "miteru/error"
4
4
  require "miteru/http_client"
5
5
  require "miteru/website"
6
6
  require "miteru/downloader"
7
+ require "miteru/feeds"
7
8
  require "miteru/crawler"
8
9
  require "miteru/cli"
9
10
  require "miteru/version"
@@ -1,9 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "colorize"
4
- require "digest"
5
- require "fileutils"
6
- require "http"
7
3
  require "thor"
8
4
 
9
5
  module Miteru
@@ -12,7 +8,7 @@ module Miteru
12
8
  method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
13
9
  method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
14
10
  method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
15
- method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 100,000)"
11
+ method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
16
12
  method_option :threads, type: :numeric, default: 10, desc: "Number of threads to use"
17
13
  method_option :verbose, type: :boolean, default: true
18
14
  desc "execute", "Execute the crawler"
@@ -1,8 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "csv"
4
- require "http"
5
- require "json"
3
+ require "colorize"
6
4
  require "parallel"
7
5
  require "uri"
8
6
 
@@ -11,14 +9,11 @@ module Miteru
11
9
  attr_reader :auto_download
12
10
  attr_reader :directory_traveling
13
11
  attr_reader :downloader
12
+ attr_reader :feeds
14
13
  attr_reader :size
15
14
  attr_reader :threads
16
15
  attr_reader :verbose
17
16
 
18
- URLSCAN_ENDPOINT = "https://urlscan.io/api/v1"
19
- OPENPHISH_ENDPOINT = "https://openphish.com"
20
- PHISHTANK_ENDPOINT = "http://data.phishtank.com"
21
-
22
17
  def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
23
18
  @auto_download = auto_download
24
19
  @directory_traveling = directory_traveling
@@ -27,63 +22,14 @@ module Miteru
27
22
  @size = size
28
23
  @threads = threads
29
24
  @verbose = verbose
30
- raise ArgumentError, "size must be less than 100,000" if size > 100_000
31
- end
32
-
33
- def urlscan_feed
34
- url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
35
- res = JSON.parse(get(url))
36
- res["results"].map { |result| result.dig("task", "url") }
37
- rescue HTTPResponseError => _
38
- []
39
- end
40
-
41
- def openphish_feed
42
- res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
43
- res.lines.map(&:chomp)
44
- rescue HTTPResponseError => _
45
- []
46
- end
47
-
48
- def phishtank_feed
49
- res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
50
- table = CSV.parse(res, headers: true)
51
- table.map { |row| row["url"] }
52
- rescue HTTPResponseError => _
53
- []
54
- end
55
-
56
- def breakdown(url)
57
- begin
58
- uri = URI.parse(url)
59
- rescue URI::InvalidURIError => _
60
- return []
61
- end
62
-
63
- base = "#{uri.scheme}://#{uri.hostname}"
64
- return [base] unless directory_traveling
65
-
66
- segments = uri.path.split("/")
67
- return [base] if segments.length.zero?
68
25
 
69
- urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
70
- urls.reject do |breakdowned_url|
71
- # Reject a url which ends with specific extension names
72
- %w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
73
- end
74
- end
75
-
76
- def suspicious_urls
77
- @suspicious_urls ||= [].tap do |arr|
78
- urls = (urlscan_feed + openphish_feed + phishtank_feed).select { |url| url.start_with?("http://", "https://") }
79
- urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
80
- end
26
+ @feeds = Feeds.new(size, directory_traveling: directory_traveling)
81
27
  end
82
28
 
83
29
  def execute
84
- puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
30
+ puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl." if verbose
85
31
 
86
- Parallel.each(suspicious_urls, in_threads: threads) do |url|
32
+ Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
87
33
  website = Website.new(url)
88
34
  if website.has_kit?
89
35
  message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
@@ -115,7 +61,7 @@ module Miteru
115
61
  channel = ENV["SLACK_CHANNEL"] || "#general"
116
62
 
117
63
  payload = { text: message, channel: channel }
118
- HTTP.post(webhook_url, json: payload)
64
+ HTTPClient.post(webhook_url, json: payload)
119
65
  end
120
66
 
121
67
  def post_to_slack?
@@ -130,13 +76,6 @@ module Miteru
130
76
  ENV["SLACK_WEBHOOK_URL"] != nil
131
77
  end
132
78
 
133
- private
134
79
 
135
- def get(url)
136
- res = HTTP.follow(max_hops: 3).get(url)
137
- raise HTTPResponseError if res.code != 200
138
-
139
- res.body.to_s
140
- end
141
80
  end
142
81
  end
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "digest"
4
+ require "fileutils"
5
+
3
6
  module Miteru
4
7
  class Downloader
5
8
  attr_reader :base_dir
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "./feeds/feed"
4
+ require_relative "./feeds/openphish"
5
+ require_relative "./feeds/phishtank"
6
+ require_relative "./feeds/urlscan"
7
+
8
+ module Miteru
9
+ class Feeds
10
+ attr_reader :openphish, :phishtank, :urlscan
11
+ attr_reader :directory_traveling
12
+
13
+ def initialize(urlscan_size = 100, directory_traveling: false)
14
+ @openphish = OpenPhish.new
15
+ @phishtank = PhishTank.new
16
+ @urlscan = UrlScan.new(urlscan_size)
17
+ @directory_traveling = directory_traveling
18
+ end
19
+
20
+ def suspicious_urls
21
+ @suspicious_urls ||= [].tap do |arr|
22
+ urls = (openphish.urls + phishtank.urls + urlscan.urls).select { |url| url.start_with?("http://", "https://") }
23
+ urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
24
+ end
25
+ end
26
+
27
+ def breakdown(url)
28
+ begin
29
+ uri = URI.parse(url)
30
+ rescue URI::InvalidURIError => _
31
+ return []
32
+ end
33
+
34
+ base = "#{uri.scheme}://#{uri.hostname}"
35
+ return [base] unless directory_traveling
36
+
37
+ segments = uri.path.split("/")
38
+ return [base] if segments.length.zero?
39
+
40
+ urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
41
+ urls.reject do |breakdowned_url|
42
+ # Reject a url which ends with specific extension names
43
+ %w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ class Feeds
5
+ class Feed
6
+ def urls
7
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
8
+ end
9
+
10
+ private
11
+
12
+ def get(url)
13
+ res = HTTPClient.get(url)
14
+ raise HTTPResponseError if res.code != 200
15
+
16
+ res.body.to_s
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Miteru
6
+ class Feeds
7
+ class OpenPhish < Feed
8
+ ENDPOINT = "https://openphish.com"
9
+
10
+ def urls
11
+ res = get("#{ENDPOINT}/feed.txt")
12
+ res.lines.map(&:chomp)
13
+ rescue HTTPResponseError => e
14
+ puts "Failed to load OpenPhish feed (#{e})"
15
+ []
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "csv"
4
+
5
+ module Miteru
6
+ class Feeds
7
+ class PhishTank < Feed
8
+ ENDPOINT = "http://data.phishtank.com"
9
+
10
+ def urls
11
+ res = get("#{ENDPOINT}/data/online-valid.csv")
12
+ table = CSV.parse(res, headers: true)
13
+ table.map { |row| row["url"] }
14
+ rescue HTTPResponseError => e
15
+ puts "Failed to load PhishTank feed (#{e})"
16
+ []
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Miteru
6
+ class Feeds
7
+ class UrlScan < Feed
8
+ ENDPOINT = "https://urlscan.io/api/v1"
9
+
10
+ attr_reader :size
11
+ def initialize(size = 100)
12
+ @size = size
13
+ raise ArgumentError, "size must be less than 10,000" if size > 10_000
14
+ end
15
+
16
+ def urls
17
+ url = "#{ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
18
+ res = JSON.parse(get(url))
19
+ res["results"].map { |result| result.dig("task", "url") }
20
+ rescue HTTPResponseError => e
21
+ puts "Failed to load urlscan.io feed (#{e})"
22
+ []
23
+ end
24
+ end
25
+ end
26
+ end
@@ -25,12 +25,21 @@ module Miteru
25
25
  new.download(url, base_dir)
26
26
  end
27
27
 
28
- def get(url)
29
- HTTP.timeout(write: 2, connect: 5, read: 10).headers(default_headers).get(url, default_options)
28
+ def get(url, options = {})
29
+ options = options.merge default_options
30
+ HTTP.follow.timeout(write: 2, connect: 5, read: 10).headers(default_headers).get(url, options)
30
31
  end
31
32
 
32
- def self.get(url)
33
- new.get url
33
+ def self.get(url, options = {})
34
+ new.get url, options
35
+ end
36
+
37
+ def post(url, options = {})
38
+ HTTP.post url, options
39
+ end
40
+
41
+ def self.post(url, options = {})
42
+ new.post url, options
34
43
  end
35
44
 
36
45
  private
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.9.2"
4
+ VERSION = "0.9.3"
5
5
  end
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
24
24
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "bundler", "~> 1.16"
27
+ spec.add_development_dependency "bundler", "~> 1.17"
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
29
  spec.add_development_dependency "glint", "~> 0.1"
30
30
  spec.add_development_dependency "rake", "~> 12.3"
@@ -33,8 +33,8 @@ Gem::Specification.new do |spec|
33
33
  spec.add_development_dependency "webmock", "~> 3.4"
34
34
 
35
35
  spec.add_dependency "colorize", "~> 0.8"
36
- spec.add_dependency "down", "~> 4.5"
37
- spec.add_dependency "http", "~> 3.3"
36
+ spec.add_dependency "down", "~> 4.6"
37
+ spec.add_dependency "http", "~> 4.0"
38
38
  spec.add_dependency "oga", "~> 2.15"
39
39
  spec.add_dependency "parallel", "~> 1.12"
40
40
  spec.add_dependency "thor", "~> 0.19"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.9.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-10-21 00:00:00.000000000 Z
11
+ date: 2018-10-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.16'
19
+ version: '1.17'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.16'
26
+ version: '1.17'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: coveralls
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -128,28 +128,28 @@ dependencies:
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '4.5'
131
+ version: '4.6'
132
132
  type: :runtime
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '4.5'
138
+ version: '4.6'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: http
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '3.3'
145
+ version: '4.0'
146
146
  type: :runtime
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '3.3'
152
+ version: '4.0'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: oga
155
155
  requirement: !ruby/object:Gem::Requirement
@@ -215,6 +215,11 @@ files:
215
215
  - lib/miteru/crawler.rb
216
216
  - lib/miteru/downloader.rb
217
217
  - lib/miteru/error.rb
218
+ - lib/miteru/feeds.rb
219
+ - lib/miteru/feeds/feed.rb
220
+ - lib/miteru/feeds/openphish.rb
221
+ - lib/miteru/feeds/phishtank.rb
222
+ - lib/miteru/feeds/urlscan.rb
218
223
  - lib/miteru/http_client.rb
219
224
  - lib/miteru/version.rb
220
225
  - lib/miteru/website.rb