miteru 0.12.13 → 0.14.3

This diff has not been reviewed by any users.
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: feb9c62c511e26e85a86755680ed77ddb34aca15c6d17d8c57dd2c60aa84becf
4
- data.tar.gz: d9e86a727e12e171b65accb829b07602847986ba81ea6457400ad8527b862324
3
+ metadata.gz: 95bccae4b2e34a97963be17cea1b90180ab18278b07be56db5bbc9e1755f673a
4
+ data.tar.gz: 0cf9b0015a178f037c802cd88da68828d6db7618cd35a4fc4e9a6dd9f5030703
5
5
  SHA512:
6
- metadata.gz: 8402163c1decbcee1c352a2ca9004f52b8c0262c5adc18d10866abe190151c8bf56d1e7ed03018f33539d1f108a1ccfc265d289dcaeb167606a10217ec112d6a
7
- data.tar.gz: 6ecde10d8a4d1842995a2281e1b30663cb25c3fdd9edefa5c8f883c94f2da3b973372a84adfa70bd8537c126ab12b870b2bf6046f3021e329129428974fc96c0
6
+ metadata.gz: 65ebef82c94e969c4886a34bab6241aea8c6a495a041a10a8df7af7429d77a0eda03f233df8ae56b904542d649c8a9bb549e6ceeb4007b5f19ad2616407b20ff
7
+ data.tar.gz: 2d145f705bd18c0219695ae83aeff01bccfeb3fc34dc04a0a9dcf9374a9b839bd43762d76501430509baac8598895df088e35dcd24a5c846e5962072ce00ec40
@@ -3,6 +3,5 @@ language: ruby
3
3
  cache: bundler
4
4
  rvm:
5
5
  - 2.6
6
- before_install:
7
- - gem update --system
8
- - gem install bundler
6
+ - 2.7
7
+ before_install: gem install bundler -v 2.1
data/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
4
4
  [![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
5
- ![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)
5
+ [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
6
6
  [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
7
7
  [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
8
8
 
@@ -15,7 +15,10 @@ Miteru is an experimental phishing kit detection tool.
15
15
  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
16
16
  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
17
17
  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
18
+ - urlscan.io phish feed (available for Pro users)
18
19
  - [Ayashige feed](https://github.com/ninoseki/ayashige)
20
+ - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
21
+ - [PhishStats feed](https://phishstats.info/)
19
22
  - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
20
23
  - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
21
24
 
@@ -85,6 +88,10 @@ For using `--post-to-slack` feature, you should set the following environment va
85
88
  - `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
86
89
  - `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
87
90
 
91
+ If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
92
+
93
+ It enables you to subscribe the urlscan.io phish feed.
94
+
88
95
  ## Examples
89
96
 
90
97
  ### Aasciinema cast
@@ -28,6 +28,8 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ attr_reader :valid_extensions
32
+
31
33
  def initialize
32
34
  @auto_download = false
33
35
  @ayashige = false
@@ -37,6 +39,8 @@ module Miteru
37
39
  @size = 100
38
40
  @threads = Parallel.processor_count
39
41
  @verbose = false
42
+
43
+ @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
40
44
  end
41
45
 
42
46
  def auto_download?
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "./feeds/feed"
4
+ require_relative "./feeds/phishing_database"
5
+ require_relative "./feeds/phishstats"
4
6
  require_relative "./feeds/ayashige"
5
7
  require_relative "./feeds/urlscan"
8
+ require_relative "./feeds/urlscan_pro"
6
9
 
7
10
  module Miteru
8
11
  class Feeds
@@ -10,7 +13,10 @@ module Miteru
10
13
 
11
14
  def initialize
12
15
  @feeds = [
16
+ PhishingDatabase.new,
17
+ PhishStats.new,
13
18
  UrlScan.new(Miteru.configuration.size),
19
+ UrlScanPro.new,
14
20
  Miteru.configuration.ayashige? ? Ayashige.new : nil
15
21
  ].compact
16
22
  end
@@ -43,6 +49,7 @@ module Miteru
43
49
  return [base] if segments.length.zero?
44
50
 
45
51
  urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
52
+
46
53
  urls.reject do |breakdowned_url|
47
54
  # Reject a url which ends with specific extension names
48
55
  invalid_extension? breakdowned_url
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "uri"
5
+
6
+ module Miteru
7
+ class Feeds
8
+ class PhishingDatabase < Feed
9
+ URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
10
+
11
+ def urls
12
+ body = get(URL)
13
+ body.to_s.lines.map(&:chomp)
14
+ rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
15
+ puts "Failed to load phishing database feed (#{e})"
16
+ []
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "uri"
5
+
6
+ module Miteru
7
+ class Feeds
8
+ class PhishStats < Feed
9
+ URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
10
+
11
+ def urls
12
+ json = JSON.parse(get(URL))
13
+ json.map do |entry|
14
+ entry.dig("url")
15
+ end
16
+ rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
17
+ puts "Failed to load PhishStats feed (#{e})"
18
+ []
19
+ end
20
+
21
+ private
22
+
23
+ def url_for(path)
24
+ URI(URL + path)
25
+ end
26
+ end
27
+ end
28
+ end
@@ -1,39 +1,35 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "json"
4
- require "uri"
3
+ require "urlscan"
5
4
 
6
5
  module Miteru
7
6
  class Feeds
8
7
  class UrlScan < Feed
9
- HOST = "urlscan.io"
10
- VERSION = 1
11
- URL = "https://#{HOST}/api/v#{VERSION}"
12
-
13
8
  attr_reader :size
9
+
14
10
  def initialize(size = 100)
15
11
  @size = size
16
12
  raise ArgumentError, "size must be less than 10,000" if size > 10_000
17
13
  end
18
14
 
15
+ def api
16
+ @api ||= ::UrlScan::API.new
17
+ end
18
+
19
19
  def urls
20
- url = url_for("/search/")
21
- url.query = URI.encode_www_form(
22
- q: "task.method:automatic",
23
- size: size
24
- )
25
-
26
- res = JSON.parse(get(url))
27
- res["results"].map { |result| result.dig("task", "url") }
28
- rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
20
+ urls_from_community_feed
21
+ rescue ::UrlScan::ResponseError => e
29
22
  puts "Failed to load urlscan.io feed (#{e})"
30
23
  []
31
24
  end
32
25
 
33
26
  private
34
27
 
35
- def url_for(path)
36
- URI(URL + path)
28
+ def urls_from_community_feed
29
+ res = api.search("task.method:automatic", size: size)
30
+
31
+ results = res["results"] || []
32
+ results.map { |result| result.dig("task", "url") }
37
33
  end
38
34
  end
39
35
  end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "urlscan"
4
+
5
+ module Miteru
6
+ class Feeds
7
+ class UrlScanPro < Feed
8
+ def api
9
+ @api ||= ::UrlScan::API.new
10
+ end
11
+
12
+ def urls
13
+ urls_from_pro_feed
14
+ rescue ::UrlScan::ResponseError => e
15
+ puts "Failed to load urlscan.io pro feed (#{e})"
16
+ []
17
+ end
18
+
19
+ private
20
+
21
+ def api_key?
22
+ ENV.key? "URLSCAN_API_KEY"
23
+ end
24
+
25
+ def urls_from_pro_feed
26
+ return [] unless api_key?
27
+
28
+ res = api.pro.phishfeed
29
+ results = res["results"] || []
30
+ results.map { |result| result.dig("page_url") }
31
+ rescue ArgumentError => _e
32
+ []
33
+ end
34
+ end
35
+ end
36
+ end
@@ -18,11 +18,20 @@ module Miteru
18
18
  end
19
19
 
20
20
  def download(url, destination)
21
- down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
21
+ down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
22
  down.download(url, destination: destination)
23
23
  destination
24
24
  end
25
25
 
26
+ def head(url, options = {})
27
+ options = options.merge default_options
28
+
29
+ HTTP.follow
30
+ .timeout(3)
31
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
+ .head(url, options)
33
+ end
34
+
26
35
  def get(url, options = {})
27
36
  options = options.merge default_options
28
37
 
@@ -48,6 +57,10 @@ module Miteru
48
57
  def post(url, options = {})
49
58
  new.post url, options
50
59
  end
60
+
61
+ def head(url, options = {})
62
+ new.head url, options
63
+ end
51
64
  end
52
65
 
53
66
  private
@@ -5,37 +5,32 @@ require "securerandom"
5
5
 
6
6
  module Miteru
7
7
  class Kit
8
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
9
 
10
- attr_reader :base_url, :link
10
+ attr_reader :url
11
11
 
12
- def initialize(base_url:, link:)
13
- @base_url = base_url
14
- @link = link.start_with?("/") ? link[1..-1] : link
12
+ def initialize(url)
13
+ @url = url
15
14
  end
16
15
 
17
- def valid?
18
- VALID_EXTENSIONS.include? extname
16
+ def valid?;
17
+ valid_ext? && reachable?
19
18
  end
20
19
 
21
20
  def extname
22
- return ".tar.gz" if link.end_with?("tar.gz")
21
+ return ".tar.gz" if url.end_with?("tar.gz")
23
22
 
24
- File.extname(link)
23
+ File.extname(url)
25
24
  end
26
25
 
27
26
  def basename
28
- File.basename(link)
27
+ File.basename(url)
29
28
  end
30
29
 
31
30
  def filename
32
31
  CGI.unescape basename
33
32
  end
34
33
 
35
- def url
36
- "#{base_url}/#{basename}"
37
- end
38
-
39
34
  def download_filepath
40
35
  "#{base_dir}/#{download_filename}"
41
36
  end
@@ -59,7 +54,7 @@ module Miteru
59
54
  end
60
55
 
61
56
  def hostname
62
- URI(base_url).hostname
57
+ URI(url).hostname
63
58
  end
64
59
 
65
60
  def download_filename
@@ -69,5 +64,16 @@ module Miteru
69
64
  def base_dir
70
65
  @base_dir ||= Miteru.configuration.download_to
71
66
  end
67
+
68
+ def valid_ext?
69
+ VALID_EXTENSIONS.include? extname
70
+ end
71
+
72
+ def reachable?
73
+ res = HTTPClient.head(url)
74
+ res.status.success?
75
+ rescue StandardError
76
+ false
77
+ end
72
78
  end
73
79
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.12.13"
4
+ VERSION = "0.14.3"
5
5
  end
@@ -4,7 +4,10 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
+
7
9
  attr_reader :url
10
+
8
11
  def initialize(url)
9
12
  @url = url
10
13
  end
@@ -15,7 +18,7 @@ module Miteru
15
18
 
16
19
  def kits
17
20
  @kits ||= links.map do |link|
18
- kit = Kit.new(base_url: url, link: link.to_s)
21
+ kit = Kit.new(link)
19
22
  kit.valid? ? kit : nil
20
23
  end.compact
21
24
  end
@@ -33,7 +36,7 @@ module Miteru
33
36
  end
34
37
 
35
38
  def has_kits?
36
- ok? && index? && kits?
39
+ kits?
37
40
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
38
41
  false
39
42
  end
@@ -46,6 +49,10 @@ module Miteru
46
49
  "It might contain #{noun}: #{filename_with_sizes}."
47
50
  end
48
51
 
52
+ def links
53
+ (href_links + possible_file_links).compact.uniq
54
+ end
55
+
49
56
  private
50
57
 
51
58
  def response
@@ -66,12 +73,31 @@ module Miteru
66
73
  nil
67
74
  end
68
75
 
69
- def links
70
- if doc
71
- doc.css("a").map { |a| a.get("href") }.compact
76
+ def href_links
77
+ if doc && ok? && index?
78
+ doc.css("a").map { |a| a.get("href") }.compact.map do |href|
79
+ href = href.start_with?("/") ? href : "/#{href}"
80
+ url + href
81
+ end
72
82
  else
73
83
  []
74
84
  end
85
+ rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
86
+ []
87
+ end
88
+
89
+ def possible_file_links
90
+ uri = URI.parse(url)
91
+
92
+ segments = uri.path.split("/")
93
+ return [] if segments.length.zero?
94
+
95
+ last = segments.last
96
+ VALID_EXTENSIONS.map do |ext|
97
+ new_segments = segments[0..-2] + ["#{last}#{ext}"]
98
+ uri.path = new_segments.join("/")
99
+ uri.to_s
100
+ end
75
101
  end
76
102
  end
77
103
  end
@@ -24,19 +24,20 @@ Gem::Specification.new do |spec|
24
24
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "bundler", "~> 2.0"
27
+ spec.add_development_dependency "bundler", "~> 2.1"
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
29
  spec.add_development_dependency "glint", "~> 0.1"
30
30
  spec.add_development_dependency "rake", "~> 13.0"
31
31
  spec.add_development_dependency "rspec", "~> 3.9"
32
- spec.add_development_dependency "vcr", "~> 5.0"
33
- spec.add_development_dependency "webmock", "~> 3.7"
32
+ spec.add_development_dependency "vcr", "~> 6.0"
33
+ spec.add_development_dependency "webmock", "~> 3.8"
34
34
 
35
35
  spec.add_dependency "colorize", "~> 0.8"
36
- spec.add_dependency "down", "~> 5.0"
37
- spec.add_dependency "http", "~> 4.2"
38
- spec.add_dependency "oga", "~> 2.15"
36
+ spec.add_dependency "down", "~> 5.1"
37
+ spec.add_dependency "http", "~> 4.4"
38
+ spec.add_dependency "oga", "~> 3.2"
39
39
  spec.add_dependency "parallel", "~> 1.19"
40
40
  spec.add_dependency "slack-notifier", "~> 2.3"
41
- spec.add_dependency "thor", "~> 0.20"
41
+ spec.add_dependency "thor", "~> 1.0"
42
+ spec.add_dependency "urlscan", "~> 0.5"
42
43
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.13
4
+ version: 0.14.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-11-27 00:00:00.000000000 Z
11
+ date: 2020-06-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '2.0'
19
+ version: '2.1'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '2.0'
26
+ version: '2.1'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: coveralls
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -86,28 +86,28 @@ dependencies:
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '5.0'
89
+ version: '6.0'
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '5.0'
96
+ version: '6.0'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: webmock
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: '3.7'
103
+ version: '3.8'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: '3.7'
110
+ version: '3.8'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: colorize
113
113
  requirement: !ruby/object:Gem::Requirement
@@ -128,42 +128,42 @@ dependencies:
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '5.0'
131
+ version: '5.1'
132
132
  type: :runtime
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '5.0'
138
+ version: '5.1'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: http
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '4.2'
145
+ version: '4.4'
146
146
  type: :runtime
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '4.2'
152
+ version: '4.4'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: oga
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
157
  - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '2.15'
159
+ version: '3.2'
160
160
  type: :runtime
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
164
  - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '2.15'
166
+ version: '3.2'
167
167
  - !ruby/object:Gem::Dependency
168
168
  name: parallel
169
169
  requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,28 @@ dependencies:
198
198
  requirements:
199
199
  - - "~>"
200
200
  - !ruby/object:Gem::Version
201
- version: '0.20'
201
+ version: '1.0'
202
202
  type: :runtime
203
203
  prerelease: false
204
204
  version_requirements: !ruby/object:Gem::Requirement
205
205
  requirements:
206
206
  - - "~>"
207
207
  - !ruby/object:Gem::Version
208
- version: '0.20'
208
+ version: '1.0'
209
+ - !ruby/object:Gem::Dependency
210
+ name: urlscan
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '0.5'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '0.5'
209
223
  description: An experimental phishing kit detector
210
224
  email:
211
225
  - manabu.niseki@gmail.com
@@ -235,7 +249,10 @@ files:
235
249
  - lib/miteru/feeds.rb
236
250
  - lib/miteru/feeds/ayashige.rb
237
251
  - lib/miteru/feeds/feed.rb
252
+ - lib/miteru/feeds/phishing_database.rb
253
+ - lib/miteru/feeds/phishstats.rb
238
254
  - lib/miteru/feeds/urlscan.rb
255
+ - lib/miteru/feeds/urlscan_pro.rb
239
256
  - lib/miteru/http_client.rb
240
257
  - lib/miteru/kit.rb
241
258
  - lib/miteru/notifier.rb
@@ -247,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
247
264
  licenses:
248
265
  - MIT
249
266
  metadata: {}
250
- post_install_message:
267
+ post_install_message:
251
268
  rdoc_options: []
252
269
  require_paths:
253
270
  - lib
@@ -262,8 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
262
279
  - !ruby/object:Gem::Version
263
280
  version: '0'
264
281
  requirements: []
265
- rubygems_version: 3.0.3
266
- signing_key:
282
+ rubygems_version: 3.1.2
283
+ signing_key:
267
284
  specification_version: 4
268
285
  summary: An experimental phishing kit detector
269
286
  test_files: []