miteru 0.14.5 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/miteru/error.rb CHANGED
@@ -2,5 +2,6 @@
2
2
 
3
3
  module Miteru
4
4
  class HTTPResponseError < StandardError; end
5
+
5
6
  class DownloadError < StandardError; end
6
7
  end
@@ -7,7 +7,7 @@ module Miteru
7
7
  class Feeds
8
8
  class Ayashige < Feed
9
9
  HOST = "ayashige.herokuapp.com"
10
- URL = "https://#{HOST}"
10
+ URL = "https://#{HOST}".freeze
11
11
 
12
12
  def urls
13
13
  url = url_for("/feed")
@@ -6,7 +6,7 @@ require "uri"
6
6
  module Miteru
7
7
  class Feeds
8
8
  class PhishingDatabase < Feed
9
- URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
9
+ URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt"
10
10
 
11
11
  def urls
12
12
  body = get(URL)
@@ -11,7 +11,7 @@ module Miteru
11
11
  def urls
12
12
  json = JSON.parse(get(URL))
13
13
  json.map do |entry|
14
- entry.dig("url")
14
+ entry["url"]
15
15
  end
16
16
  rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
17
17
  puts "Failed to load PhishStats feed (#{e})"
@@ -27,7 +27,7 @@ module Miteru
27
27
 
28
28
  res = api.pro.phishfeed
29
29
  results = res["results"] || []
30
- results.map { |result| result.dig("page_url") }
30
+ results.map { |result| result["page_url"] }
31
31
  rescue ArgumentError => _e
32
32
  []
33
33
  end
data/lib/miteru/feeds.rb CHANGED
@@ -9,7 +9,7 @@ require_relative "./feeds/urlscan_pro"
9
9
 
10
10
  module Miteru
11
11
  class Feeds
12
- IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
12
+ IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
13
13
 
14
14
  def initialize
15
15
  @feeds = [
@@ -48,7 +48,7 @@ module Miteru
48
48
  segments = uri.path.split("/")
49
49
  return [base] if segments.length.zero?
50
50
 
51
- urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
51
+ urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
52
52
 
53
53
  urls.reject do |breakdowned_url|
54
54
  # Reject a url which ends with specific extension names
@@ -7,7 +7,7 @@ require "uri"
7
7
  module Miteru
8
8
  class HTTPClient
9
9
  DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
10
- URLSCAN_UA = "miteru/#{Miteru::VERSION}"
10
+ URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
11
11
 
12
12
  attr_reader :ssl_context
13
13
 
@@ -27,18 +27,18 @@ module Miteru
27
27
  options = options.merge default_options
28
28
 
29
29
  HTTP.follow
30
- .timeout(3)
31
- .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
- .head(url, options)
30
+ .timeout(3)
31
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
+ .head(url, options)
33
33
  end
34
34
 
35
35
  def get(url, options = {})
36
36
  options = options.merge default_options
37
37
 
38
38
  HTTP.follow
39
- .timeout(write: 2, connect: 5, read: 10)
40
- .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
41
- .get(url, options)
39
+ .timeout(write: 2, connect: 5, read: 10)
40
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
41
+ .get(url, options)
42
42
  end
43
43
 
44
44
  def post(url, options = {})
data/lib/miteru/kit.rb CHANGED
@@ -1,21 +1,30 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "cgi"
4
- require "securerandom"
4
+ require "uuidtools"
5
+ require "uri"
5
6
 
6
7
  module Miteru
7
8
  class Kit
8
9
  VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
10
  VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
10
11
 
11
- attr_reader :url
12
+ attr_reader :url, :status, :content_length, :mime_type, :headers
12
13
 
13
14
  def initialize(url)
14
15
  @url = url
16
+
17
+ @content_length = nil
18
+ @mime_type = nil
19
+ @status = nil
20
+ @headers = nil
15
21
  end
16
22
 
17
- def valid?;
18
- valid_ext? && reachable_and_valid_mime_type?
23
+ def valid?
24
+ # make a HEAD request for the validation
25
+ before_validation
26
+
27
+ valid_ext? && reachable? && valid_mime_type? && valid_content_length?
19
28
  end
20
29
 
21
30
  def extname
@@ -25,21 +34,21 @@ module Miteru
25
34
  end
26
35
 
27
36
  def basename
28
- File.basename(url)
37
+ @basename ||= File.basename(url)
29
38
  end
30
39
 
31
40
  def filename
32
- CGI.unescape basename
41
+ @filename ||= CGI.unescape(basename)
33
42
  end
34
43
 
35
- def download_filepath
36
- "#{base_dir}/#{download_filename}"
44
+ def filepath_to_download
45
+ "#{base_dir}/#{filename_to_download}"
37
46
  end
38
47
 
39
48
  def filesize
40
- return nil unless File.exist?(download_filepath)
49
+ return nil unless File.exist?(filepath_to_download)
41
50
 
42
- File.size download_filepath
51
+ File.size filepath_to_download
43
52
  end
44
53
 
45
54
  def filename_with_size
@@ -48,18 +57,22 @@ module Miteru
48
57
  "#{filename}(#{filesize / 1024}KB)"
49
58
  end
50
59
 
51
- private
52
-
53
60
  def id
54
- @id ||= SecureRandom.hex(10)
61
+ @id ||= UUIDTools::UUID.random_create.to_s
55
62
  end
56
63
 
57
64
  def hostname
58
- URI(url).hostname
65
+ @hostname ||= URI(url).hostname
59
66
  end
60
67
 
61
- def download_filename
62
- "#{hostname}_#{filename}_#{id}#{extname}"
68
+ def decoded_url
69
+ @decoded_url ||= URI.decode_www_form_component(url)
70
+ end
71
+
72
+ private
73
+
74
+ def filename_to_download
75
+ "#{id}#{extname}"
63
76
  end
64
77
 
65
78
  def base_dir
@@ -70,20 +83,26 @@ module Miteru
70
83
  VALID_EXTENSIONS.include? extname
71
84
  end
72
85
 
73
- def reachable?(response)
74
- response.status.success?
86
+ def before_validation
87
+ res = HTTPClient.head(url)
88
+ @content_length = res.content_length
89
+ @mime_type = res.content_type.mime_type.to_s
90
+ @status = res.status
91
+ @headers = res.headers.to_h
92
+ rescue StandardError
93
+ # do nothing
75
94
  end
76
95
 
77
- def valid_mime_type?(response)
78
- mime_type = response.content_type.mime_type.to_s
96
+ def reachable?
97
+ status&.success?
98
+ end
99
+
100
+ def valid_mime_type?
79
101
  VALID_MIME_TYPES.include? mime_type
80
102
  end
81
103
 
82
- def reachable_and_valid_mime_type?
83
- res = HTTPClient.head(url)
84
- reachable?(res) && valid_mime_type?(res)
85
- rescue StandardError
86
- false
104
+ def valid_content_length?
105
+ content_length.to_i > 0
87
106
  end
88
107
  end
89
108
  end
@@ -9,29 +9,17 @@ module Miteru
9
9
  attachement = Attachement.new(url)
10
10
  kits = kits.select(&:filesize)
11
11
 
12
- if post_to_slack? && kits.any?
13
- notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
14
- notifier.post(text: message, attachments: attachement.to_a)
12
+ if notifiable? && kits.any?
13
+ notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
14
+ notifier.post(text: message.capitalize, attachments: attachement.to_a)
15
15
  end
16
16
 
17
17
  message = message.colorize(:light_red) if kits.any?
18
18
  puts "#{url}: #{message}"
19
19
  end
20
20
 
21
- def post_to_slack?
22
- slack_webhook_url? && Miteru.configuration.post_to_slack?
23
- end
24
-
25
- def slack_webhook_url
26
- ENV.fetch "SLACK_WEBHOOK_URL"
27
- end
28
-
29
- def slack_channel
30
- ENV.fetch "SLACK_CHANNEL", "#general"
31
- end
32
-
33
- def slack_webhook_url?
34
- ENV.key? "SLACK_WEBHOOK_URL"
21
+ def notifiable?
22
+ Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
35
23
  end
36
24
  end
37
25
  end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Miteru
6
+ class Record < ActiveRecord::Base
7
+ class << self
8
+ #
9
+ # Check uniqueness of a record by a hash
10
+ #
11
+ # @param [String] hash
12
+ #
13
+ # @return [Boolean] true if it is unique. Otherwise false.
14
+ #
15
+ def unique_hash?(hash)
16
+ record = find_by(hash: hash)
17
+ return true if record.nil?
18
+
19
+ false
20
+ end
21
+
22
+ #
23
+ # Create a new record based on a kit
24
+ #
25
+ # @param [Miteru::Kit] kit
26
+ # @param [String] hash
27
+ #
28
+ # @return [Miteru::Record]
29
+ #
30
+ def create_by_kit_and_hash(kit, hash)
31
+ record = new(
32
+ hash: hash,
33
+ hostname: kit.hostname,
34
+ url: kit.decoded_url,
35
+ headers: kit.headers,
36
+ filename: kit.filename,
37
+ filesize: kit.filesize,
38
+ mime_type: kit.mime_type,
39
+ downloaded_as: kit.filepath_to_download
40
+ )
41
+ record.save
42
+ record
43
+ rescue TypeError, ActiveRecord::RecordNotUnique => _e
44
+ nil
45
+ end
46
+ end
47
+ end
48
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.14.5"
4
+ VERSION = "1.0.1"
5
5
  end
@@ -17,10 +17,10 @@ module Miteru
17
17
  end
18
18
 
19
19
  def kits
20
- @kits ||= links.map do |link|
20
+ @kits ||= links.filter_map do |link|
21
21
  kit = Kit.new(link)
22
22
  kit.valid? ? kit : nil
23
- end.compact
23
+ end
24
24
  end
25
25
 
26
26
  def ok?
@@ -42,11 +42,11 @@ module Miteru
42
42
  end
43
43
 
44
44
  def message
45
- return "It doesn't contain a phishing kit." unless kits?
45
+ return "it doesn't contain a phishing kit." unless kits?
46
46
 
47
47
  filename_with_sizes = kits.map(&:filename_with_size).join(", ")
48
48
  noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
49
- "It might contain #{noun}: #{filename_with_sizes}."
49
+ "it might contain #{noun}: #{filename_with_sizes}."
50
50
  end
51
51
 
52
52
  def links
@@ -75,7 +75,7 @@ module Miteru
75
75
 
76
76
  def href_links
77
77
  if doc && ok? && index?
78
- doc.css("a").map { |a| a.get("href") }.compact.map do |href|
78
+ doc.css("a").filter_map { |a| a.get("href") }.map do |href|
79
79
  href = href.start_with?("/") ? href : "/#{href}"
80
80
  url + href
81
81
  end
data/lib/miteru.rb CHANGED
@@ -3,6 +3,10 @@
3
3
  require "miteru/version"
4
4
 
5
5
  require "miteru/configuration"
6
+ require "miteru/database"
7
+
8
+ require "miteru/record"
9
+
6
10
  require "miteru/error"
7
11
  require "miteru/http_client"
8
12
  require "miteru/kit"
@@ -14,6 +18,4 @@ require "miteru/notifier"
14
18
  require "miteru/crawler"
15
19
  require "miteru/cli"
16
20
 
17
- module Miteru
18
- # Your code goes here...
19
- end
21
+ module Miteru; end
data/miteru.gemspec CHANGED
@@ -1,43 +1,51 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- lib = File.expand_path('lib', __dir__)
3
+ lib = File.expand_path("lib", __dir__)
4
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
  require "miteru/version"
6
6
 
7
7
  Gem::Specification.new do |spec|
8
- spec.name = "miteru"
9
- spec.version = Miteru::VERSION
10
- spec.authors = ["Manabu Niseki"]
11
- spec.email = ["manabu.niseki@gmail.com"]
8
+ spec.name = "miteru"
9
+ spec.version = Miteru::VERSION
10
+ spec.authors = ["Manabu Niseki"]
11
+ spec.email = ["manabu.niseki@gmail.com"]
12
12
 
13
- spec.summary = "An experimental phishing kit detector"
14
- spec.description = "An experimental phishing kit detector"
15
- spec.homepage = "https://github.com/ninoseki/miteru"
16
- spec.license = "MIT"
13
+ spec.summary = "An experimental phishing kit detector"
14
+ spec.description = "An experimental phishing kit detector"
15
+ spec.homepage = "https://github.com/ninoseki/miteru"
16
+ spec.license = "MIT"
17
17
 
18
18
  # Specify which files should be added to the gem when it is released.
19
19
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
20
20
  spec.files = Dir.chdir(File.expand_path(__dir__)) do
21
21
  `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
22
22
  end
23
- spec.bindir = "exe"
24
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
23
+ spec.bindir = "exe"
24
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "bundler", "~> 2.1"
28
- spec.add_development_dependency "coveralls", "~> 0.8"
27
+ spec.add_development_dependency "bundler", "~> 2.2"
28
+ spec.add_development_dependency "coveralls_reborn", "~> 0.22"
29
29
  spec.add_development_dependency "glint", "~> 0.1"
30
+ spec.add_development_dependency "mysql2", "~> 0.5"
31
+ spec.add_development_dependency "overcommit", "~> 0.58"
32
+ spec.add_development_dependency "pg", "~> 1.2"
30
33
  spec.add_development_dependency "rake", "~> 13.0"
31
- spec.add_development_dependency "rspec", "~> 3.9"
34
+ spec.add_development_dependency "rspec", "~> 3.10"
35
+ spec.add_development_dependency "standard", "~> 1.3"
32
36
  spec.add_development_dependency "vcr", "~> 6.0"
33
- spec.add_development_dependency "webmock", "~> 3.8"
37
+ spec.add_development_dependency "webmock", "~> 3.14"
38
+ spec.add_development_dependency "webrick", "~> 1.7.0"
34
39
 
40
+ spec.add_dependency "activerecord", "~> 6.1"
35
41
  spec.add_dependency "colorize", "~> 0.8"
36
- spec.add_dependency "down", "~> 5.1"
37
- spec.add_dependency "http", "~> 4.4"
38
- spec.add_dependency "oga", "~> 3.2"
39
- spec.add_dependency "parallel", "~> 1.19"
40
- spec.add_dependency "slack-notifier", "~> 2.3"
41
- spec.add_dependency "thor", "~> 1.0"
42
- spec.add_dependency "urlscan", "~> 0.5"
42
+ spec.add_dependency "down", "~> 5.2"
43
+ spec.add_dependency "http", "~> 5.0"
44
+ spec.add_dependency "oga", "~> 3.3"
45
+ spec.add_dependency "parallel", "~> 1.20"
46
+ spec.add_dependency "slack-notifier", "~> 2.4"
47
+ spec.add_dependency "sqlite3", "~> 1.4"
48
+ spec.add_dependency "thor", "~> 1.1"
49
+ spec.add_dependency "urlscan", "~> 0.7"
50
+ spec.add_dependency "uuidtools", "~> 2.2"
43
51
  end