miteru 0.14.5 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +68 -0
- data/.gitignore +3 -0
- data/.overcommit.yml +12 -0
- data/.standard.yml +4 -0
- data/README.md +19 -88
- data/docker/Dockerfile +5 -2
- data/exe/miteru +2 -1
- data/lib/miteru/attachement.rb +3 -2
- data/lib/miteru/cli.rb +12 -4
- data/lib/miteru/configuration.rb +19 -0
- data/lib/miteru/crawler.rb +1 -2
- data/lib/miteru/database.rb +65 -0
- data/lib/miteru/downloader.rb +20 -23
- data/lib/miteru/error.rb +1 -0
- data/lib/miteru/feeds/ayashige.rb +1 -1
- data/lib/miteru/feeds/phishing_database.rb +1 -1
- data/lib/miteru/feeds/phishstats.rb +1 -1
- data/lib/miteru/feeds/urlscan_pro.rb +1 -1
- data/lib/miteru/feeds.rb +2 -2
- data/lib/miteru/http_client.rb +7 -7
- data/lib/miteru/kit.rb +44 -25
- data/lib/miteru/notifier.rb +5 -17
- data/lib/miteru/record.rb +48 -0
- data/lib/miteru/version.rb +1 -1
- data/lib/miteru/website.rb +5 -5
- data/lib/miteru.rb +5 -3
- data/miteru.gemspec +30 -22
- data/renovate.json +5 -0
- metadata +144 -27
- data/.travis.yml +0 -7
data/lib/miteru/error.rb
CHANGED
@@ -6,7 +6,7 @@ require "uri"
|
|
6
6
|
module Miteru
|
7
7
|
class Feeds
|
8
8
|
class PhishingDatabase < Feed
|
9
|
-
URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-
|
9
|
+
URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt"
|
10
10
|
|
11
11
|
def urls
|
12
12
|
body = get(URL)
|
data/lib/miteru/feeds.rb
CHANGED
@@ -9,7 +9,7 @@ require_relative "./feeds/urlscan_pro"
|
|
9
9
|
|
10
10
|
module Miteru
|
11
11
|
class Feeds
|
12
|
-
IGNORE_EXTENSIONS = %w
|
12
|
+
IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
|
13
13
|
|
14
14
|
def initialize
|
15
15
|
@feeds = [
|
@@ -48,7 +48,7 @@ module Miteru
|
|
48
48
|
segments = uri.path.split("/")
|
49
49
|
return [base] if segments.length.zero?
|
50
50
|
|
51
|
-
urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join(
|
51
|
+
urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
|
52
52
|
|
53
53
|
urls.reject do |breakdowned_url|
|
54
54
|
# Reject a url which ends with specific extension names
|
data/lib/miteru/http_client.rb
CHANGED
@@ -7,7 +7,7 @@ require "uri"
|
|
7
7
|
module Miteru
|
8
8
|
class HTTPClient
|
9
9
|
DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
|
10
|
-
URLSCAN_UA = "miteru/#{Miteru::VERSION}"
|
10
|
+
URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
|
11
11
|
|
12
12
|
attr_reader :ssl_context
|
13
13
|
|
@@ -27,18 +27,18 @@ module Miteru
|
|
27
27
|
options = options.merge default_options
|
28
28
|
|
29
29
|
HTTP.follow
|
30
|
-
|
31
|
-
|
32
|
-
|
30
|
+
.timeout(3)
|
31
|
+
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
|
32
|
+
.head(url, options)
|
33
33
|
end
|
34
34
|
|
35
35
|
def get(url, options = {})
|
36
36
|
options = options.merge default_options
|
37
37
|
|
38
38
|
HTTP.follow
|
39
|
-
|
40
|
-
|
41
|
-
|
39
|
+
.timeout(write: 2, connect: 5, read: 10)
|
40
|
+
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
|
41
|
+
.get(url, options)
|
42
42
|
end
|
43
43
|
|
44
44
|
def post(url, options = {})
|
data/lib/miteru/kit.rb
CHANGED
@@ -1,21 +1,30 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "cgi"
|
4
|
-
require "
|
4
|
+
require "uuidtools"
|
5
|
+
require "uri"
|
5
6
|
|
6
7
|
module Miteru
|
7
8
|
class Kit
|
8
9
|
VALID_EXTENSIONS = Miteru.configuration.valid_extensions
|
9
10
|
VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
|
10
11
|
|
11
|
-
attr_reader :url
|
12
|
+
attr_reader :url, :status, :content_length, :mime_type, :headers
|
12
13
|
|
13
14
|
def initialize(url)
|
14
15
|
@url = url
|
16
|
+
|
17
|
+
@content_length = nil
|
18
|
+
@mime_type = nil
|
19
|
+
@status = nil
|
20
|
+
@headers = nil
|
15
21
|
end
|
16
22
|
|
17
|
-
def valid
|
18
|
-
|
23
|
+
def valid?
|
24
|
+
# make a HEAD request for the validation
|
25
|
+
before_validation
|
26
|
+
|
27
|
+
valid_ext? && reachable? && valid_mime_type? && valid_content_length?
|
19
28
|
end
|
20
29
|
|
21
30
|
def extname
|
@@ -25,21 +34,21 @@ module Miteru
|
|
25
34
|
end
|
26
35
|
|
27
36
|
def basename
|
28
|
-
File.basename(url)
|
37
|
+
@basename ||= File.basename(url)
|
29
38
|
end
|
30
39
|
|
31
40
|
def filename
|
32
|
-
CGI.unescape
|
41
|
+
@filename ||= CGI.unescape(basename)
|
33
42
|
end
|
34
43
|
|
35
|
-
def
|
36
|
-
"#{base_dir}/#{
|
44
|
+
def filepath_to_download
|
45
|
+
"#{base_dir}/#{filename_to_download}"
|
37
46
|
end
|
38
47
|
|
39
48
|
def filesize
|
40
|
-
return nil unless File.exist?(
|
49
|
+
return nil unless File.exist?(filepath_to_download)
|
41
50
|
|
42
|
-
File.size
|
51
|
+
File.size filepath_to_download
|
43
52
|
end
|
44
53
|
|
45
54
|
def filename_with_size
|
@@ -48,18 +57,22 @@ module Miteru
|
|
48
57
|
"#{filename}(#{filesize / 1024}KB)"
|
49
58
|
end
|
50
59
|
|
51
|
-
private
|
52
|
-
|
53
60
|
def id
|
54
|
-
@id ||=
|
61
|
+
@id ||= UUIDTools::UUID.random_create.to_s
|
55
62
|
end
|
56
63
|
|
57
64
|
def hostname
|
58
|
-
URI(url).hostname
|
65
|
+
@hostname ||= URI(url).hostname
|
59
66
|
end
|
60
67
|
|
61
|
-
def
|
62
|
-
|
68
|
+
def decoded_url
|
69
|
+
@decoded_url ||= URI.decode_www_form_component(url)
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
def filename_to_download
|
75
|
+
"#{id}#{extname}"
|
63
76
|
end
|
64
77
|
|
65
78
|
def base_dir
|
@@ -70,20 +83,26 @@ module Miteru
|
|
70
83
|
VALID_EXTENSIONS.include? extname
|
71
84
|
end
|
72
85
|
|
73
|
-
def
|
74
|
-
|
86
|
+
def before_validation
|
87
|
+
res = HTTPClient.head(url)
|
88
|
+
@content_length = res.content_length
|
89
|
+
@mime_type = res.content_type.mime_type.to_s
|
90
|
+
@status = res.status
|
91
|
+
@headers = res.headers.to_h
|
92
|
+
rescue StandardError
|
93
|
+
# do nothing
|
75
94
|
end
|
76
95
|
|
77
|
-
def
|
78
|
-
|
96
|
+
def reachable?
|
97
|
+
status&.success?
|
98
|
+
end
|
99
|
+
|
100
|
+
def valid_mime_type?
|
79
101
|
VALID_MIME_TYPES.include? mime_type
|
80
102
|
end
|
81
103
|
|
82
|
-
def
|
83
|
-
|
84
|
-
reachable?(res) && valid_mime_type?(res)
|
85
|
-
rescue StandardError
|
86
|
-
false
|
104
|
+
def valid_content_length?
|
105
|
+
content_length.to_i > 0
|
87
106
|
end
|
88
107
|
end
|
89
108
|
end
|
data/lib/miteru/notifier.rb
CHANGED
@@ -9,29 +9,17 @@ module Miteru
|
|
9
9
|
attachement = Attachement.new(url)
|
10
10
|
kits = kits.select(&:filesize)
|
11
11
|
|
12
|
-
if
|
13
|
-
notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
|
14
|
-
notifier.post(text: message, attachments: attachement.to_a)
|
12
|
+
if notifiable? && kits.any?
|
13
|
+
notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
|
14
|
+
notifier.post(text: message.capitalize, attachments: attachement.to_a)
|
15
15
|
end
|
16
16
|
|
17
17
|
message = message.colorize(:light_red) if kits.any?
|
18
18
|
puts "#{url}: #{message}"
|
19
19
|
end
|
20
20
|
|
21
|
-
def
|
22
|
-
slack_webhook_url? && Miteru.configuration.post_to_slack?
|
23
|
-
end
|
24
|
-
|
25
|
-
def slack_webhook_url
|
26
|
-
ENV.fetch "SLACK_WEBHOOK_URL"
|
27
|
-
end
|
28
|
-
|
29
|
-
def slack_channel
|
30
|
-
ENV.fetch "SLACK_CHANNEL", "#general"
|
31
|
-
end
|
32
|
-
|
33
|
-
def slack_webhook_url?
|
34
|
-
ENV.key? "SLACK_WEBHOOK_URL"
|
21
|
+
def notifiable?
|
22
|
+
Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
|
35
23
|
end
|
36
24
|
end
|
37
25
|
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record"
|
4
|
+
|
5
|
+
module Miteru
|
6
|
+
class Record < ActiveRecord::Base
|
7
|
+
class << self
|
8
|
+
#
|
9
|
+
# Check uniqueness of a record by a hash
|
10
|
+
#
|
11
|
+
# @param [String] hash
|
12
|
+
#
|
13
|
+
# @return [Boolean] true if it is unique. Otherwise false.
|
14
|
+
#
|
15
|
+
def unique_hash?(hash)
|
16
|
+
record = find_by(hash: hash)
|
17
|
+
return true if record.nil?
|
18
|
+
|
19
|
+
false
|
20
|
+
end
|
21
|
+
|
22
|
+
#
|
23
|
+
# Create a new record based on a kit
|
24
|
+
#
|
25
|
+
# @param [Miteru::Kit] kit
|
26
|
+
# @param [String] hash
|
27
|
+
#
|
28
|
+
# @return [Miteru::Record]
|
29
|
+
#
|
30
|
+
def create_by_kit_and_hash(kit, hash)
|
31
|
+
record = new(
|
32
|
+
hash: hash,
|
33
|
+
hostname: kit.hostname,
|
34
|
+
url: kit.decoded_url,
|
35
|
+
headers: kit.headers,
|
36
|
+
filename: kit.filename,
|
37
|
+
filesize: kit.filesize,
|
38
|
+
mime_type: kit.mime_type,
|
39
|
+
downloaded_as: kit.filepath_to_download
|
40
|
+
)
|
41
|
+
record.save
|
42
|
+
record
|
43
|
+
rescue TypeError, ActiveRecord::RecordNotUnique => _e
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/miteru/version.rb
CHANGED
data/lib/miteru/website.rb
CHANGED
@@ -17,10 +17,10 @@ module Miteru
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def kits
|
20
|
-
@kits ||= links.
|
20
|
+
@kits ||= links.filter_map do |link|
|
21
21
|
kit = Kit.new(link)
|
22
22
|
kit.valid? ? kit : nil
|
23
|
-
end
|
23
|
+
end
|
24
24
|
end
|
25
25
|
|
26
26
|
def ok?
|
@@ -42,11 +42,11 @@ module Miteru
|
|
42
42
|
end
|
43
43
|
|
44
44
|
def message
|
45
|
-
return "
|
45
|
+
return "it doesn't contain a phishing kit." unless kits?
|
46
46
|
|
47
47
|
filename_with_sizes = kits.map(&:filename_with_size).join(", ")
|
48
48
|
noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
|
49
|
-
"
|
49
|
+
"it might contain #{noun}: #{filename_with_sizes}."
|
50
50
|
end
|
51
51
|
|
52
52
|
def links
|
@@ -75,7 +75,7 @@ module Miteru
|
|
75
75
|
|
76
76
|
def href_links
|
77
77
|
if doc && ok? && index?
|
78
|
-
doc.css("a").
|
78
|
+
doc.css("a").filter_map { |a| a.get("href") }.map do |href|
|
79
79
|
href = href.start_with?("/") ? href : "/#{href}"
|
80
80
|
url + href
|
81
81
|
end
|
data/lib/miteru.rb
CHANGED
@@ -3,6 +3,10 @@
|
|
3
3
|
require "miteru/version"
|
4
4
|
|
5
5
|
require "miteru/configuration"
|
6
|
+
require "miteru/database"
|
7
|
+
|
8
|
+
require "miteru/record"
|
9
|
+
|
6
10
|
require "miteru/error"
|
7
11
|
require "miteru/http_client"
|
8
12
|
require "miteru/kit"
|
@@ -14,6 +18,4 @@ require "miteru/notifier"
|
|
14
18
|
require "miteru/crawler"
|
15
19
|
require "miteru/cli"
|
16
20
|
|
17
|
-
module Miteru
|
18
|
-
# Your code goes here...
|
19
|
-
end
|
21
|
+
module Miteru; end
|
data/miteru.gemspec
CHANGED
@@ -1,43 +1,51 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
lib = File.expand_path(
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
5
|
require "miteru/version"
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
|
-
spec.name
|
9
|
-
spec.version
|
10
|
-
spec.authors
|
11
|
-
spec.email
|
8
|
+
spec.name = "miteru"
|
9
|
+
spec.version = Miteru::VERSION
|
10
|
+
spec.authors = ["Manabu Niseki"]
|
11
|
+
spec.email = ["manabu.niseki@gmail.com"]
|
12
12
|
|
13
|
-
spec.summary
|
14
|
-
spec.description
|
15
|
-
spec.homepage
|
16
|
-
spec.license
|
13
|
+
spec.summary = "An experimental phishing kit detector"
|
14
|
+
spec.description = "An experimental phishing kit detector"
|
15
|
+
spec.homepage = "https://github.com/ninoseki/miteru"
|
16
|
+
spec.license = "MIT"
|
17
17
|
|
18
18
|
# Specify which files should be added to the gem when it is released.
|
19
19
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
20
20
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
21
21
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
22
22
|
end
|
23
|
-
spec.bindir
|
24
|
-
spec.executables
|
23
|
+
spec.bindir = "exe"
|
24
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
25
|
spec.require_paths = ["lib"]
|
26
26
|
|
27
|
-
spec.add_development_dependency "bundler", "~> 2.
|
28
|
-
spec.add_development_dependency "
|
27
|
+
spec.add_development_dependency "bundler", "~> 2.2"
|
28
|
+
spec.add_development_dependency "coveralls_reborn", "~> 0.22"
|
29
29
|
spec.add_development_dependency "glint", "~> 0.1"
|
30
|
+
spec.add_development_dependency "mysql2", "~> 0.5"
|
31
|
+
spec.add_development_dependency "overcommit", "~> 0.58"
|
32
|
+
spec.add_development_dependency "pg", "~> 1.2"
|
30
33
|
spec.add_development_dependency "rake", "~> 13.0"
|
31
|
-
spec.add_development_dependency "rspec", "~> 3.
|
34
|
+
spec.add_development_dependency "rspec", "~> 3.10"
|
35
|
+
spec.add_development_dependency "standard", "~> 1.3"
|
32
36
|
spec.add_development_dependency "vcr", "~> 6.0"
|
33
|
-
spec.add_development_dependency "webmock", "~> 3.
|
37
|
+
spec.add_development_dependency "webmock", "~> 3.14"
|
38
|
+
spec.add_development_dependency "webrick", "~> 1.7.0"
|
34
39
|
|
40
|
+
spec.add_dependency "activerecord", "~> 6.1"
|
35
41
|
spec.add_dependency "colorize", "~> 0.8"
|
36
|
-
spec.add_dependency "down", "~> 5.
|
37
|
-
spec.add_dependency "http", "~>
|
38
|
-
spec.add_dependency "oga", "~> 3.
|
39
|
-
spec.add_dependency "parallel", "~> 1.
|
40
|
-
spec.add_dependency "slack-notifier", "~> 2.
|
41
|
-
spec.add_dependency "
|
42
|
-
spec.add_dependency "
|
42
|
+
spec.add_dependency "down", "~> 5.2"
|
43
|
+
spec.add_dependency "http", "~> 5.0"
|
44
|
+
spec.add_dependency "oga", "~> 3.3"
|
45
|
+
spec.add_dependency "parallel", "~> 1.20"
|
46
|
+
spec.add_dependency "slack-notifier", "~> 2.4"
|
47
|
+
spec.add_dependency "sqlite3", "~> 1.4"
|
48
|
+
spec.add_dependency "thor", "~> 1.1"
|
49
|
+
spec.add_dependency "urlscan", "~> 0.7"
|
50
|
+
spec.add_dependency "uuidtools", "~> 2.2"
|
43
51
|
end
|