miteru 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: efab1503dd6c96a4376d6baed09eadccde40d9a43440604e45c2926bdb301209
4
- data.tar.gz: fda7418c3101bd3e691bd6cb0af7b3b2a6b3c0cd58da30d7a2171868251468c4
3
+ metadata.gz: 48d47d401250750c0ffa68ccf5f58602a1e0fa4918b0b812aebcd0c60cdfe484
4
+ data.tar.gz: 9acbfd9d4465c25ddb20186fe57a2d0a84ba74ba24fa2e55b2b2103b5a4d12f9
5
5
  SHA512:
6
- metadata.gz: 0b10ef50e80806ca90386c4da6ae9057e044af58a1975d3c8b4c03942966b09e23386a64d233916610066ded7c7ec59700ddb4b149107cee71acb6a52459d139
7
- data.tar.gz: a977b070175607fed8a2ec30a331278a6f0cd1f6438c1a3b4accffd2be18f8963f92afac01d0144f0567aabefab19c3ed4af4dd4f83c338a3af9bb22022ea542
6
+ metadata.gz: 7690a2497203522b7c9469656a5e1ea9fd2e72671555548a1d0833841d8b3bce34a60c22b2e9597b94add35342dbae0b2cb3fd58fdb3ec08b9e16375f1ca6044
7
+ data.tar.gz: 27d5aef80aa2bae48091bfaf398fae956e7226384414bac4505af3752090a722c1be1b421d3260773dd40d7d7c2dc65281b5555bbade267f83554d267371cd85
data/README.md CHANGED
@@ -13,7 +13,8 @@ Miteru is an experimental phishing kit detection tool.
13
13
  - [urlscan.io](https://urlscan.io/search/#certstream-suspicious)
14
14
  - [OpenPhish community feed](https://openphish.com/feed.txt)
15
15
  - [PhishTank feed](http://data.phishtank.com/data/online-valid.csv)
16
- - It checks a suspicious URL whether it has a directory listing and contains a phishing kit (`*.zip` file) or not.
16
+ - It checks a suspicious URL whether it has a directory listing and contains a phishing kit (compressed file) or not.
17
+ - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
17
18
 
18
19
  ## Installation
19
20
 
@@ -8,7 +8,7 @@ require "thor"
8
8
 
9
9
  module Miteru
10
10
  class CLI < Thor
11
- method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of *.zip file(s)"
11
+ method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of compressed file(s)"
12
12
  method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
13
13
  method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
14
14
  method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
@@ -26,16 +26,16 @@ module Miteru
26
26
  websites.each do |website|
27
27
  next unless website.has_kit?
28
28
 
29
- message = "#{website.url}: it might contain phishing kit(s) (#{website.zip_files.join(', ')})."
29
+ message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
30
30
  puts message.colorize(:light_red)
31
- post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
32
- download_zip_files(website.url, website.zip_files, options[:download_to]) if options[:auto_download]
31
+ post_to_slack(website.message) if options[:post_to_slack] && valid_slack_setting?
32
+ download_compressed_files(website.url, website.compressed_files, options[:download_to]) if options[:auto_download]
33
33
  end
34
34
  end
35
35
 
36
36
  no_commands do
37
- def download_zip_files(url, zip_files, base_dir)
38
- zip_files.each do |path|
37
+ def download_compressed_files(url, compressed_files, base_dir)
38
+ compressed_files.each do |path|
39
39
  target_url = "#{url}/#{path}"
40
40
  begin
41
41
  download_file_path = HTTPClient.download(target_url, base_dir)
@@ -29,17 +29,23 @@ module Miteru
29
29
  url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
30
30
  res = JSON.parse(get(url))
31
31
  res["results"].map { |result| result.dig("task", "url") }
32
+ rescue HTTPResponseError => _
33
+ []
32
34
  end
33
35
 
34
36
  def openphish_feed
35
37
  res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
36
38
  res.lines.map(&:chomp)
39
+ rescue HTTPResponseError => _
40
+ []
37
41
  end
38
42
 
39
43
  def phishtank_feed
40
44
  res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
41
45
  table = CSV.parse(res, headers: true)
42
46
  table.map { |row| row["url"] }
47
+ rescue HTTPResponseError => _
48
+ []
43
49
  end
44
50
 
45
51
  def breakdown(url)
@@ -62,11 +68,15 @@ module Miteru
62
68
  end
63
69
 
64
70
  def suspicious_urls
65
- urls = (urlscan_feed + openphish_feed + phishtank_feed)
66
- urls.map { |url| breakdown(url) }.flatten.uniq.sort
71
+ @suspicious_urls ||= [].tap do |arr|
72
+ urls = (urlscan_feed + openphish_feed + phishtank_feed)
73
+ urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
74
+ end
67
75
  end
68
76
 
69
77
  def execute
78
+ puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
79
+
70
80
  pool = Thread.pool(threads)
71
81
  websites = []
72
82
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.8.0"
4
+ VERSION = "0.9.0"
5
5
  end
@@ -14,10 +14,10 @@ module Miteru
14
14
  doc.at_css("title")&.text
15
15
  end
16
16
 
17
- def zip_files
18
- @zip_files ||= doc.css("a").map do |a|
17
+ def compressed_files
18
+ @compressed_files ||= doc.css("a").map do |a|
19
19
  href = a.get("href")
20
- href&.end_with?(".zip") ? href : nil
20
+ [".zip", ".rar", ".7z", ".tar", ".gz"].any? { |ext| href&.end_with? ext } ? href : nil
21
21
  end.compact.map do |href|
22
22
  href.start_with?("/") ? href[1..-1] : href
23
23
  end
@@ -31,12 +31,12 @@ module Miteru
31
31
  title == "Index of /"
32
32
  end
33
33
 
34
- def zip_files?
35
- !zip_files.empty?
34
+ def compressed_files?
35
+ !compressed_files.empty?
36
36
  end
37
37
 
38
38
  def has_kit?
39
- @has_kit ||= ok? && index? && zip_files?
39
+ @has_kit ||= ok? && index? && compressed_files?
40
40
  end
41
41
 
42
42
  def build
@@ -46,7 +46,7 @@ module Miteru
46
46
  def unbuild
47
47
  @doc = nil
48
48
  @response = nil
49
- @zip_files = nil
49
+ @compressed_files = nil
50
50
  end
51
51
 
52
52
  private
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-10-06 00:00:00.000000000 Z
11
+ date: 2018-10-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler