miteru 0.14.7 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,13 +6,12 @@ require "uri"
6
6
 
7
7
  module Miteru
8
8
  class Downloader
9
- attr_reader :base_dir
10
- attr_reader :memo
9
+ attr_reader :base_dir, :memo
11
10
 
12
11
  def initialize(base_dir = "/tmp")
13
12
  @base_dir = base_dir
14
13
  @memo = {}
15
- raise ArgumentError, "#{base_dir} is not exist." unless Dir.exist?(base_dir)
14
+ raise ArgumentError, "#{base_dir} doesn't exist." unless Dir.exist?(base_dir)
16
15
  end
17
16
 
18
17
  def download_kits(kits)
@@ -22,23 +21,29 @@ module Miteru
22
21
  private
23
22
 
24
23
  def download_kit(kit)
25
- destination = kit.download_filepath
24
+ destination = kit.filepath_to_download
25
+
26
26
  begin
27
- downloaded_filepath = HTTPClient.download(kit.url, destination)
28
- hash = sha256(downloaded_filepath)
29
- if duplicated?(hash)
30
- puts "Do not download #{kit.url} because there is a duplicate file in the directory (SHA256: #{hash})."
31
- FileUtils.rm downloaded_filepath
32
- else
33
- puts "Download #{kit.url} as #{downloaded_filepath}"
34
- end
27
+ downloaded_as = HTTPClient.download(kit.url, destination)
35
28
  rescue Down::Error => e
36
29
  puts "Failed to download: #{kit.url} (#{e})"
30
+ return
37
31
  end
38
- end
39
32
 
40
- def filepath_to_download(filename)
41
- "#{base_dir}/#{filename}"
33
+ hash = sha256(downloaded_as)
34
+
35
+ ActiveRecord::Base.connection_pool.with_connection do
36
+ # Remove a downloaded file if it is not unique
37
+ unless Record.unique_hash?(hash)
38
+ puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
39
+ FileUtils.rm downloaded_as
40
+ return
41
+ end
42
+
43
+ # Record a kit in DB
44
+ Record.create_by_kit_and_hash(kit, hash)
45
+ puts "Download #{kit.url} as #{downloaded_as}"
46
+ end
42
47
  end
43
48
 
44
49
  def sha256(path)
@@ -49,13 +54,5 @@ module Miteru
49
54
  memo[path] = hash
50
55
  hash
51
56
  end
52
-
53
- def sha256s
54
- Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
55
- end
56
-
57
- def duplicated?(hash)
58
- sha256s.count(hash) > 1
59
- end
60
57
  end
61
58
  end
data/lib/miteru/error.rb CHANGED
@@ -2,5 +2,6 @@
2
2
 
3
3
  module Miteru
4
4
  class HTTPResponseError < StandardError; end
5
+
5
6
  class DownloadError < StandardError; end
6
7
  end
@@ -10,10 +10,10 @@ module Miteru
10
10
  URL = "https://#{HOST}"
11
11
 
12
12
  def urls
13
- url = url_for("/feed")
13
+ url = url_for("/api/v1/domains/")
14
14
  res = JSON.parse(get(url))
15
15
 
16
- domains = res.map { |item| item["domain"] }
16
+ domains = res.map { |item| item["fqdn"] }
17
17
  domains.map do |domain|
18
18
  [
19
19
  "https://#{domain}",
@@ -3,10 +3,43 @@
3
3
  module Miteru
4
4
  class Feeds
5
5
  class Feed
6
+ include Mixins::URL
7
+
8
+ def source
9
+ @source ||= self.class.to_s.split("::").last
10
+ end
11
+
12
+ #
13
+ # Return URLs
14
+ #
15
+ # @return [Array<String>] URLs
16
+ #
6
17
  def urls
7
18
  raise NotImplementedError, "You must implement #{self.class}##{__method__}"
8
19
  end
9
20
 
21
+ #
22
+ # Return entries
23
+ #
24
+ # @return [Array<Miteru::Entry>]
25
+ #
26
+ def entries
27
+ breakdowend_urls.map do |url|
28
+ Entry.new(url, source)
29
+ end
30
+ end
31
+
32
+ #
33
+ # Return breakdowned URLs
34
+ #
35
+ # @return [Array<String>] Breakdowned URLs
36
+ #
37
+ def breakdowend_urls
38
+ urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
39
+ breakdown(url, Miteru.configuration.directory_traveling?)
40
+ end.flatten.uniq
41
+ end
42
+
10
43
  private
11
44
 
12
45
  def get(url)
@@ -11,7 +11,7 @@ module Miteru
11
11
  def urls
12
12
  json = JSON.parse(get(URL))
13
13
  json.map do |entry|
14
- entry.dig("url")
14
+ entry["url"]
15
15
  end
16
16
  rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
17
17
  puts "Failed to load PhishStats feed (#{e})"
@@ -27,7 +27,7 @@ module Miteru
27
27
 
28
28
  res = api.pro.phishfeed
29
29
  results = res["results"] || []
30
- results.map { |result| result.dig("page_url") }
30
+ results.map { |result| result["page_url"] }
31
31
  rescue ArgumentError => _e
32
32
  []
33
33
  end
data/lib/miteru/feeds.rb CHANGED
@@ -8,8 +8,20 @@ require_relative "./feeds/urlscan"
8
8
  require_relative "./feeds/urlscan_pro"
9
9
 
10
10
  module Miteru
11
+ class Entry
12
+ # @return [String]
13
+ attr_reader :url
14
+ # @return [String]
15
+ attr_reader :source
16
+
17
+ def initialize(url, source)
18
+ @url = url
19
+ @source = source
20
+ end
21
+ end
22
+
11
23
  class Feeds
12
- IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
24
+ IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
13
25
 
14
26
  def initialize
15
27
  @feeds = [
@@ -21,43 +33,13 @@ module Miteru
21
33
  ].compact
22
34
  end
23
35
 
24
- def directory_traveling?
25
- Miteru.configuration.directory_traveling?
26
- end
27
-
28
- def suspicious_urls
29
- @suspicious_urls ||= [].tap do |arr|
30
- urls = @feeds.map do |feed|
31
- feed.urls.select { |url| url.start_with?("http://", "https://") }
32
- end.flatten.uniq
33
-
34
- urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
35
- end
36
- end
37
-
38
- def breakdown(url)
39
- begin
40
- uri = URI.parse(url)
41
- rescue URI::InvalidURIError => _e
42
- return []
43
- end
44
-
45
- base = "#{uri.scheme}://#{uri.hostname}"
46
- return [base] unless directory_traveling?
47
-
48
- segments = uri.path.split("/")
49
- return [base] if segments.length.zero?
50
-
51
- urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
52
-
53
- urls.reject do |breakdowned_url|
54
- # Reject a url which ends with specific extension names
55
- invalid_extension? breakdowned_url
56
- end
57
- end
58
-
59
- def invalid_extension?(url)
60
- IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
36
+ #
37
+ # Returns a list of suspicious entries
38
+ #
39
+ # @return [Array<Entry>]
40
+ #
41
+ def suspicious_entries
42
+ @suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
61
43
  end
62
44
  end
63
45
  end
@@ -7,7 +7,7 @@ require "uri"
7
7
  module Miteru
8
8
  class HTTPClient
9
9
  DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
10
- URLSCAN_UA = "miteru/#{Miteru::VERSION}"
10
+ URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
11
11
 
12
12
  attr_reader :ssl_context
13
13
 
@@ -27,18 +27,18 @@ module Miteru
27
27
  options = options.merge default_options
28
28
 
29
29
  HTTP.follow
30
- .timeout(3)
31
- .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
- .head(url, options)
30
+ .timeout(3)
31
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
+ .head(url, options)
33
33
  end
34
34
 
35
35
  def get(url, options = {})
36
36
  options = options.merge default_options
37
37
 
38
38
  HTTP.follow
39
- .timeout(write: 2, connect: 5, read: 10)
40
- .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
41
- .get(url, options)
39
+ .timeout(write: 2, connect: 5, read: 10)
40
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
41
+ .get(url, options)
42
42
  end
43
43
 
44
44
  def post(url, options = {})
data/lib/miteru/kit.rb CHANGED
@@ -1,28 +1,43 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "cgi"
4
- require "securerandom"
4
+ require "uuidtools"
5
+ require "uri"
5
6
 
6
7
  module Miteru
7
8
  class Kit
8
9
  VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
10
  VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
10
11
 
12
+ # @return [String]
11
13
  attr_reader :url
12
14
 
15
+ # @return [String]
16
+ attr_reader :source
17
+
18
+ # @return [Integer, nil]
13
19
  attr_reader :status
20
+
21
+ # @return [Integer, nil]
14
22
  attr_reader :content_length
23
+
24
+ # @return [String, nil]
15
25
  attr_reader :mime_type
16
26
 
17
- def initialize(url)
27
+ # @return [Hash, nil]
28
+ attr_reader :headers
29
+
30
+ def initialize(url, source)
18
31
  @url = url
32
+ @source = source
19
33
 
20
34
  @content_length = nil
21
35
  @mime_type = nil
22
36
  @status = nil
37
+ @headers = nil
23
38
  end
24
39
 
25
- def valid?;
40
+ def valid?
26
41
  # make a HEAD request for the validation
27
42
  before_validation
28
43
 
@@ -36,41 +51,50 @@ module Miteru
36
51
  end
37
52
 
38
53
  def basename
39
- File.basename(url)
54
+ @basename ||= File.basename(url)
40
55
  end
41
56
 
42
57
  def filename
43
- CGI.unescape basename
58
+ @filename ||= CGI.unescape(basename)
59
+ end
60
+
61
+ def filepath_to_download
62
+ "#{base_dir}/#{filename_to_download}"
44
63
  end
45
64
 
46
- def download_filepath
47
- "#{base_dir}/#{download_filename}"
65
+ def downloaded?
66
+ File.exist?(filepath_to_download)
48
67
  end
49
68
 
50
69
  def filesize
51
- return nil unless File.exist?(download_filepath)
70
+ return nil unless downloaded?
52
71
 
53
- File.size download_filepath
72
+ File.size filepath_to_download
54
73
  end
55
74
 
56
75
  def filename_with_size
57
76
  return filename unless filesize
58
77
 
59
- "#{filename}(#{filesize / 1024}KB)"
78
+ kb = (filesize.to_f / 1024.0).ceil
79
+ "#{filename}(#{kb}KB)"
60
80
  end
61
81
 
62
- private
63
-
64
82
  def id
65
- @id ||= SecureRandom.hex(10)
83
+ @id ||= UUIDTools::UUID.random_create.to_s
66
84
  end
67
85
 
68
86
  def hostname
69
- URI(url).hostname
87
+ @hostname ||= URI(url).hostname
88
+ end
89
+
90
+ def decoded_url
91
+ @decoded_url ||= URI.decode_www_form_component(url)
70
92
  end
71
93
 
72
- def download_filename
73
- "#{hostname}_#{filename}_#{id}#{extname}"
94
+ private
95
+
96
+ def filename_to_download
97
+ "#{id}#{extname}"
74
98
  end
75
99
 
76
100
  def base_dir
@@ -86,6 +110,7 @@ module Miteru
86
110
  @content_length = res.content_length
87
111
  @mime_type = res.content_type.mime_type.to_s
88
112
  @status = res.status
113
+ @headers = res.headers.to_h
89
114
  rescue StandardError
90
115
  # do nothing
91
116
  end
@@ -0,0 +1,47 @@
1
+ module Miteru
2
+ module Mixins
3
+ module URL
4
+ IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
5
+
6
+ #
7
+ # Validate extension of a URL
8
+ #
9
+ # @param [String] url
10
+ #
11
+ # @return [Boolean]
12
+ #
13
+ def invalid_extension?(url)
14
+ IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
15
+ end
16
+
17
+ #
18
+ # Breakdown a URL into URLs
19
+ #
20
+ # @param [String] url
21
+ # @param [Boolean] enable_directory_traveling
22
+ #
23
+ # @return [Array<String>]
24
+ #
25
+ def breakdown(url, enable_directory_traveling)
26
+ begin
27
+ uri = URI.parse(url)
28
+ rescue URI::InvalidURIError => _e
29
+ return []
30
+ end
31
+
32
+ base = "#{uri.scheme}://#{uri.hostname}"
33
+ return [base] unless enable_directory_traveling
34
+
35
+ segments = uri.path.split("/")
36
+ return [base] if segments.length.zero?
37
+
38
+ urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
39
+
40
+ urls.reject do |breakdowned_url|
41
+ # Reject a url which ends with specific extension names
42
+ invalid_extension? breakdowned_url
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -7,31 +7,19 @@ module Miteru
7
7
  class Notifier
8
8
  def notify(url:, kits:, message:)
9
9
  attachement = Attachement.new(url)
10
- kits = kits.select(&:filesize)
10
+ kits = kits.select(&:downloaded?)
11
11
 
12
- if post_to_slack? && kits.any?
13
- notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
14
- notifier.post(text: message, attachments: attachement.to_a)
12
+ if notifiable? && kits.any?
13
+ notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
14
+ notifier.post(text: message.capitalize, attachments: attachement.to_a)
15
15
  end
16
16
 
17
17
  message = message.colorize(:light_red) if kits.any?
18
18
  puts "#{url}: #{message}"
19
19
  end
20
20
 
21
- def post_to_slack?
22
- slack_webhook_url? && Miteru.configuration.post_to_slack?
23
- end
24
-
25
- def slack_webhook_url
26
- ENV.fetch "SLACK_WEBHOOK_URL"
27
- end
28
-
29
- def slack_channel
30
- ENV.fetch "SLACK_CHANNEL", "#general"
31
- end
32
-
33
- def slack_webhook_url?
34
- ENV.key? "SLACK_WEBHOOK_URL"
21
+ def notifiable?
22
+ Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
35
23
  end
36
24
  end
37
25
  end