miteru 1.0.1 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 58464f5c1b0b17549201ea547edbcf22cd83bcbbd7ef1a1233ab2d04eeb4c13c
4
- data.tar.gz: 6a8048507e00ded2fe11ef1a71b05536a267ee54b6e64b7662cf5f238ace84fd
3
+ metadata.gz: c858dd689d566a4a12270d7474b9174d60deeabb23f61e1a62440040558f0a3c
4
+ data.tar.gz: 0f53873e358ab18712c0e9d7c5fc6b6cd863464b467d78e82983ca2386834115
5
5
  SHA512:
6
- metadata.gz: efb6fcb505fff1451ab5cbff5a0bbc1d3e480aa7cc4f51005b84065c5c174b4c12e9609f07217f7cbf0bdb844dd8efe7430e99e2f3bb7ea3fa7207d9ff531ae4
7
- data.tar.gz: 5ab2d22a2c9c647665a3feefd9b579ac336e434a84320db6df4ba3b7e1ea4d44a14b1db7a91f8e63add7e7a0d544d0d791cd07f1d4258065642c2fae35c0002b
6
+ metadata.gz: 76359bf3d5883acdfdef815045d634a286be8985f21408f9f91349a7e048077309ab468d095d4b49e9e26b19018b4d280b2290beb8620b5d7729e1b9fc5d9586
7
+ data.tar.gz: 9bb8d92dbb325a74c38a8f05a4ed5967831a70601597d6d45235d9a1571228dcfcc543e255628b763b977bf2a649b3976779a2765c8ba19dd174dc988d275320
@@ -42,18 +42,17 @@ jobs:
42
42
  ruby: [2.7, "3.0"]
43
43
 
44
44
  steps:
45
- - uses: actions/checkout@v2
45
+ - uses: actions/checkout@v3
46
46
  - name: Set up Ruby
47
47
  uses: ruby/setup-ruby@v1
48
48
  with:
49
49
  ruby-version: ${{ matrix.ruby }}
50
+ bundler: latest
50
51
  bundler-cache: true
51
52
 
52
53
  - name: Install dependencies
53
54
  run: |
54
55
  sudo apt-get -yqq install libpq-dev libmysqlclient-dev
55
- gem install bundler
56
- bundle install
57
56
 
58
57
  - name: Test with PostgreSQL
59
58
  env:
@@ -47,7 +47,7 @@ module Miteru
47
47
  end
48
48
 
49
49
  def defanged_url
50
- @defanged_url ||= url.to_s.gsub /\./, "[.]"
50
+ @defanged_url ||= url.to_s.gsub(/\./, "[.]")
51
51
  end
52
52
 
53
53
  def domain
@@ -37,12 +37,21 @@ module Miteru
37
37
  # @return [String]
38
38
  attr_accessor :slack_channel
39
39
 
40
+ # @return [String]
41
+ attr_accessor :urlscan_api_key
42
+
43
+ # @return [String]
44
+ attr_accessor :urlscan_submit_visibility
45
+
40
46
  # @return [Array<String>]
41
47
  attr_reader :valid_extensions
42
48
 
43
49
  # @return [Array<String>]
44
50
  attr_reader :valid_mime_types
45
51
 
52
+ # @return [Integer]
53
+ attr_reader :file_maxsize
54
+
46
55
  def initialize
47
56
  @auto_download = false
48
57
  @ayashige = false
@@ -52,10 +61,14 @@ module Miteru
52
61
  @size = 100
53
62
  @threads = Parallel.processor_count
54
63
  @verbose = false
55
- @database = ENV["MITERU_DATABASE"] || "miteru.db"
64
+ @database = ENV.fetch("MITERU_DATABASE", "miteru.db")
65
+ @file_maxsize = ENV.fetch("FILE_MAXSIZE", 1024 * 1024 * 100).to_i
66
+
67
+ @slack_webhook_url = ENV.fetch("SLACK_WEBHOOK_URL", nil)
68
+ @slack_channel = ENV.fetch("SLACK_CHANNEL", "#general")
56
69
 
57
- @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
58
- @slack_channel = ENV["SLACK_CHANNEL"] || "#general"
70
+ @urlscan_api_key = ENV.fetch("URLSCAN_API_KEY", nil)
71
+ @urlscan_submit_visibility = ENV.fetch("URLSCAN_SUBMIT_VISIBILITY", "public")
59
72
 
60
73
  @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
61
74
  @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
@@ -82,7 +95,11 @@ module Miteru
82
95
  end
83
96
 
84
97
  def slack_webhook_url?
85
- @slack_webhook_url
98
+ !@slack_webhook_url.nil?
99
+ end
100
+
101
+ def urlscan_api_key?
102
+ !@urlscan_api_key.nil?
86
103
  end
87
104
  end
88
105
 
@@ -11,11 +11,10 @@ module Miteru
11
11
  def initialize
12
12
  @downloader = Downloader.new(Miteru.configuration.download_to)
13
13
  @feeds = Feeds.new
14
- @notifier = Notifier.new
15
14
  end
16
15
 
17
- def crawl(url)
18
- website = Website.new(url)
16
+ def crawl(entry)
17
+ website = Website.new(entry.url, entry.source)
19
18
  downloader.download_kits(website.kits) if website.has_kits? && auto_download?
20
19
  notify(website) if website.has_kits? || verbose?
21
20
  rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
@@ -23,11 +22,11 @@ module Miteru
23
22
  end
24
23
 
25
24
  def execute
26
- suspicious_urls = feeds.suspicious_urls
27
- puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
25
+ suspicious_entries = feeds.suspicious_entries
26
+ Miteru.logger.info "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
28
27
 
29
- Parallel.each(suspicious_urls, in_threads: threads) do |url|
30
- crawl url
28
+ Parallel.each(suspicious_entries, in_threads: threads) do |entry|
29
+ crawl entry
31
30
  end
32
31
  end
33
32
 
@@ -36,7 +35,9 @@ module Miteru
36
35
  end
37
36
 
38
37
  def notify(website)
39
- @notifier.notify(url: website.url, kits: website.kits, message: website.message)
38
+ Parallel.each(notifiers) do |notifier|
39
+ notifier.notify website
40
+ end
40
41
  end
41
42
 
42
43
  def auto_download?
@@ -47,6 +48,12 @@ module Miteru
47
48
  Miteru.configuration.verbose?
48
49
  end
49
50
 
51
+ private
52
+
53
+ def notifiers
54
+ @notifiers ||= [Notifiers::Slack.new, Notifiers::UrlScan.new].select(&:notifiable?)
55
+ end
56
+
50
57
  class << self
51
58
  def execute
52
59
  new.execute
@@ -2,7 +2,7 @@
2
2
 
3
3
  require "active_record"
4
4
 
5
- class InitialSchema < ActiveRecord::Migration[6.1]
5
+ class InitialSchema < ActiveRecord::Migration[7.0]
6
6
  def change
7
7
  create_table :records, if_not_exists: true do |t|
8
8
  t.string :hash, null: false, index: { unique: true }
@@ -19,6 +19,12 @@ class InitialSchema < ActiveRecord::Migration[6.1]
19
19
  end
20
20
  end
21
21
 
22
+ class V11Schema < ActiveRecord::Migration[7.0]
23
+ def change
24
+ add_column :records, :source, :string, if_not_exists: true
25
+ end
26
+ end
27
+
22
28
  def adapter
23
29
  return "postgresql" if Miteru.configuration.database.start_with?("postgresql://", "postgres://")
24
30
  return "mysql2" if Miteru.configuration.database.start_with?("mysql2://")
@@ -44,6 +50,7 @@ module Miteru
44
50
  ActiveRecord::Migration.verbose = false
45
51
 
46
52
  InitialSchema.migrate(:up)
53
+ V11Schema.migrate(:up)
47
54
  rescue StandardError => _e
48
55
  # Do nothing
49
56
  end
@@ -57,6 +64,7 @@ module Miteru
57
64
  return unless ActiveRecord::Base.connected?
58
65
 
59
66
  InitialSchema.migrate(:down)
67
+ V11Schema.migrate(:down)
60
68
  end
61
69
  end
62
70
  end
@@ -26,7 +26,15 @@ module Miteru
26
26
  begin
27
27
  downloaded_as = HTTPClient.download(kit.url, destination)
28
28
  rescue Down::Error => e
29
- puts "Failed to download: #{kit.url} (#{e})"
29
+ Miteru.logger.error "Failed to download: #{kit.url} (#{e})"
30
+ return
31
+ end
32
+
33
+ # check filesize
34
+ size = File.size downloaded_as
35
+ if size > Miteru.configuration.file_maxsize
36
+ Miteru.logger.info "#{kit.url}'s filesize exceeds the limit: #{size}"
37
+ FileUtils.rm downloaded_as
30
38
  return
31
39
  end
32
40
 
@@ -35,14 +43,14 @@ module Miteru
35
43
  ActiveRecord::Base.connection_pool.with_connection do
36
44
  # Remove a downloaded file if it is not unique
37
45
  unless Record.unique_hash?(hash)
38
- puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
46
+ Miteru.logger.info "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
39
47
  FileUtils.rm downloaded_as
40
48
  return
41
49
  end
42
50
 
43
51
  # Record a kit in DB
44
52
  Record.create_by_kit_and_hash(kit, hash)
45
- puts "Download #{kit.url} as #{downloaded_as}"
53
+ Miteru.logger.info "Download #{kit.url} as #{downloaded_as}"
46
54
  end
47
55
  end
48
56
 
@@ -7,13 +7,13 @@ module Miteru
7
7
  class Feeds
8
8
  class Ayashige < Feed
9
9
  HOST = "ayashige.herokuapp.com"
10
- URL = "https://#{HOST}".freeze
10
+ URL = "https://#{HOST}"
11
11
 
12
12
  def urls
13
- url = url_for("/feed")
13
+ url = url_for("/api/v1/domains/")
14
14
  res = JSON.parse(get(url))
15
15
 
16
- domains = res.map { |item| item["domain"] }
16
+ domains = res.map { |item| item["fqdn"] }
17
17
  domains.map do |domain|
18
18
  [
19
19
  "https://#{domain}",
@@ -21,7 +21,7 @@ module Miteru
21
21
  ]
22
22
  end.flatten
23
23
  rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
24
- puts "Failed to load ayashige feed (#{e})"
24
+ Miteru.logger.error "Failed to load ayashige feed (#{e})"
25
25
  []
26
26
  end
27
27
 
@@ -3,10 +3,43 @@
3
3
  module Miteru
4
4
  class Feeds
5
5
  class Feed
6
+ include Mixins::URL
7
+
8
+ def source
9
+ @source ||= self.class.to_s.split("::").last
10
+ end
11
+
12
+ #
13
+ # Return URLs
14
+ #
15
+ # @return [Array<String>] URLs
16
+ #
6
17
  def urls
7
18
  raise NotImplementedError, "You must implement #{self.class}##{__method__}"
8
19
  end
9
20
 
21
+ #
22
+ # Return entries
23
+ #
24
+ # @return [Array<Miteru::Entry>]
25
+ #
26
+ def entries
27
+ breakdowend_urls.map do |url|
28
+ Entry.new(url, source)
29
+ end
30
+ end
31
+
32
+ #
33
+ # Return breakdowned URLs
34
+ #
35
+ # @return [Array<String>] Breakdowned URLs
36
+ #
37
+ def breakdowend_urls
38
+ urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
39
+ breakdown(url, Miteru.configuration.directory_traveling?)
40
+ end.flatten.uniq
41
+ end
42
+
10
43
  private
11
44
 
12
45
  def get(url)
@@ -12,7 +12,7 @@ module Miteru
12
12
  body = get(URL)
13
13
  body.to_s.lines.map(&:chomp)
14
14
  rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
15
- puts "Failed to load phishing database feed (#{e})"
15
+ info "Failed to load phishing database feed (#{e})"
16
16
  []
17
17
  end
18
18
  end
@@ -14,7 +14,7 @@ module Miteru
14
14
  entry["url"]
15
15
  end
16
16
  rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
17
- puts "Failed to load PhishStats feed (#{e})"
17
+ Miteru.logger.error "Failed to load PhishStats feed (#{e})"
18
18
  []
19
19
  end
20
20
 
@@ -13,13 +13,13 @@ module Miteru
13
13
  end
14
14
 
15
15
  def api
16
- @api ||= ::UrlScan::API.new
16
+ @api ||= ::UrlScan::API.new(Miteru.configuration.urlscan_api_key)
17
17
  end
18
18
 
19
19
  def urls
20
20
  urls_from_community_feed
21
21
  rescue ::UrlScan::ResponseError => e
22
- puts "Failed to load urlscan.io feed (#{e})"
22
+ Miteru.logger.error "Failed to load urlscan.io feed (#{e})"
23
23
  []
24
24
  end
25
25
 
@@ -6,20 +6,20 @@ module Miteru
6
6
  class Feeds
7
7
  class UrlScanPro < Feed
8
8
  def api
9
- @api ||= ::UrlScan::API.new
9
+ @api ||= ::UrlScan::API.new(Miteru.configuration.urlscan_api_key)
10
10
  end
11
11
 
12
12
  def urls
13
13
  urls_from_pro_feed
14
14
  rescue ::UrlScan::ResponseError => e
15
- puts "Failed to load urlscan.io pro feed (#{e})"
15
+ Miteru.logger.error "Failed to load urlscan.io pro feed (#{e})"
16
16
  []
17
17
  end
18
18
 
19
19
  private
20
20
 
21
21
  def api_key?
22
- ENV.key? "URLSCAN_API_KEY"
22
+ Miteru.configuration.urlscan_api_key?
23
23
  end
24
24
 
25
25
  def urls_from_pro_feed
data/lib/miteru/feeds.rb CHANGED
@@ -8,6 +8,18 @@ require_relative "./feeds/urlscan"
8
8
  require_relative "./feeds/urlscan_pro"
9
9
 
10
10
  module Miteru
11
+ class Entry
12
+ # @return [String]
13
+ attr_reader :url
14
+ # @return [String]
15
+ attr_reader :source
16
+
17
+ def initialize(url, source)
18
+ @url = url
19
+ @source = source
20
+ end
21
+ end
22
+
11
23
  class Feeds
12
24
  IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
13
25
 
@@ -21,43 +33,13 @@ module Miteru
21
33
  ].compact
22
34
  end
23
35
 
24
- def directory_traveling?
25
- Miteru.configuration.directory_traveling?
26
- end
27
-
28
- def suspicious_urls
29
- @suspicious_urls ||= [].tap do |arr|
30
- urls = @feeds.map do |feed|
31
- feed.urls.select { |url| url.start_with?("http://", "https://") }
32
- end.flatten.uniq
33
-
34
- urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
35
- end
36
- end
37
-
38
- def breakdown(url)
39
- begin
40
- uri = URI.parse(url)
41
- rescue URI::InvalidURIError => _e
42
- return []
43
- end
44
-
45
- base = "#{uri.scheme}://#{uri.hostname}"
46
- return [base] unless directory_traveling?
47
-
48
- segments = uri.path.split("/")
49
- return [base] if segments.length.zero?
50
-
51
- urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
52
-
53
- urls.reject do |breakdowned_url|
54
- # Reject a url which ends with specific extension names
55
- invalid_extension? breakdowned_url
56
- end
57
- end
58
-
59
- def invalid_extension?(url)
60
- IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
36
+ #
37
+ # Returns a list of suspicious entries
38
+ #
39
+ # @return [Array<Entry>]
40
+ #
41
+ def suspicious_entries
42
+ @suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
61
43
  end
62
44
  end
63
45
  end
@@ -7,7 +7,7 @@ require "uri"
7
7
  module Miteru
8
8
  class HTTPClient
9
9
  DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
10
- URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
10
+ URLSCAN_UA = "miteru/#{Miteru::VERSION}"
11
11
 
12
12
  attr_reader :ssl_context
13
13
 
data/lib/miteru/kit.rb CHANGED
@@ -9,10 +9,27 @@ module Miteru
9
9
  VALID_EXTENSIONS = Miteru.configuration.valid_extensions
10
10
  VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
11
11
 
12
- attr_reader :url, :status, :content_length, :mime_type, :headers
12
+ # @return [String]
13
+ attr_reader :url
13
14
 
14
- def initialize(url)
15
+ # @return [String]
16
+ attr_reader :source
17
+
18
+ # @return [Integer, nil]
19
+ attr_reader :status
20
+
21
+ # @return [Integer, nil]
22
+ attr_reader :content_length
23
+
24
+ # @return [String, nil]
25
+ attr_reader :mime_type
26
+
27
+ # @return [Hash, nil]
28
+ attr_reader :headers
29
+
30
+ def initialize(url, source)
15
31
  @url = url
32
+ @source = source
16
33
 
17
34
  @content_length = nil
18
35
  @mime_type = nil
@@ -45,8 +62,12 @@ module Miteru
45
62
  "#{base_dir}/#{filename_to_download}"
46
63
  end
47
64
 
65
+ def downloaded?
66
+ File.exist?(filepath_to_download)
67
+ end
68
+
48
69
  def filesize
49
- return nil unless File.exist?(filepath_to_download)
70
+ return nil unless downloaded?
50
71
 
51
72
  File.size filepath_to_download
52
73
  end
@@ -54,7 +75,8 @@ module Miteru
54
75
  def filename_with_size
55
76
  return filename unless filesize
56
77
 
57
- "#{filename}(#{filesize / 1024}KB)"
78
+ kb = (filesize.to_f / 1024.0).ceil
79
+ "#{filename}(#{kb}KB)"
58
80
  end
59
81
 
60
82
  def id
@@ -0,0 +1,47 @@
1
+ module Miteru
2
+ module Mixins
3
+ module URL
4
+ IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
5
+
6
+ #
7
+ # Validate extension of a URL
8
+ #
9
+ # @param [String] url
10
+ #
11
+ # @return [Boolean]
12
+ #
13
+ def invalid_extension?(url)
14
+ IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
15
+ end
16
+
17
+ #
18
+ # Breakdown a URL into URLs
19
+ #
20
+ # @param [String] url
21
+ # @param [Boolean] enable_directory_traveling
22
+ #
23
+ # @return [Array<String>]
24
+ #
25
+ def breakdown(url, enable_directory_traveling)
26
+ begin
27
+ uri = URI.parse(url)
28
+ rescue URI::InvalidURIError => _e
29
+ return []
30
+ end
31
+
32
+ base = "#{uri.scheme}://#{uri.hostname}"
33
+ return [base] unless enable_directory_traveling
34
+
35
+ segments = uri.path.split("/")
36
+ return [base] if segments.length.zero?
37
+
38
+ urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
39
+
40
+ urls.reject do |breakdowned_url|
41
+ # Reject a url which ends with specific extension names
42
+ invalid_extension? breakdowned_url
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Notifiers
5
+ class Base
6
+ def notify(website)
7
+ raise NotImplementedError
8
+ end
9
+
10
+ def notifiable?
11
+ raise NotImplementedError
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Notifiers
5
+ class Slack < Base
6
+ #
7
+ # Notifiy to Slack
8
+ #
9
+ # @param [Miteru::Website website
10
+ #
11
+ def notify(website)
12
+ attachement = Attachement.new(website.url)
13
+ kits = website.kits.select(&:downloaded?)
14
+
15
+ if notifiable? && kits.any?
16
+ notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
17
+ notifier.post(text: website.message.capitalize, attachments: attachement.to_a)
18
+ end
19
+
20
+ message = kits.any? ? website.message.colorize(:light_red) : website.message
21
+ Miteru.logger.info "#{website.url}: #{message}"
22
+ end
23
+
24
+ def notifiable?
25
+ Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "urlscan"
4
+
5
+ module Miteru
6
+ module Notifiers
7
+ class UrlScan < Base
8
+ #
9
+ # Notifiy to urlscan.io
10
+ #
11
+ # @param [Miteru::Website website
12
+ #
13
+ def notify(website)
14
+ kits = website.kits.select(&:downloaded?)
15
+ return unless notifiable? && kits.any?
16
+
17
+ kits.each { |kit| submit(kit.url) }
18
+ end
19
+
20
+ def notifiable?
21
+ Miteru.configuration.urlscan_api_key?
22
+ end
23
+
24
+ private
25
+
26
+ def api
27
+ @api ||= ::UrlScan::API.new(Miteru.configuration.urlscan_api_key)
28
+ end
29
+
30
+ def submit(url)
31
+ api.submit(url, tags: ["miteru", "phishkit"], visibility: Miteru.configuration.urlscan_submit_visibility)
32
+ rescue StandardError
33
+ # do nothing
34
+ end
35
+ end
36
+ end
37
+ end
data/lib/miteru/record.rb CHANGED
@@ -30,6 +30,7 @@ module Miteru
30
30
  def create_by_kit_and_hash(kit, hash)
31
31
  record = new(
32
32
  hash: hash,
33
+ source: kit.source,
33
34
  hostname: kit.hostname,
34
35
  url: kit.decoded_url,
35
36
  headers: kit.headers,
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "1.0.1"
4
+ VERSION = "1.2.0"
5
5
  end
@@ -6,10 +6,15 @@ module Miteru
6
6
  class Website
7
7
  VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
8
 
9
+ # @return [String]
9
10
  attr_reader :url
10
11
 
11
- def initialize(url)
12
+ # @return [String]
13
+ attr_reader :source
14
+
15
+ def initialize(url, source)
12
16
  @url = url
17
+ @source = source
13
18
  end
14
19
 
15
20
  def title
@@ -18,7 +23,7 @@ module Miteru
18
23
 
19
24
  def kits
20
25
  @kits ||= links.filter_map do |link|
21
- kit = Kit.new(link)
26
+ kit = Kit.new(link, source)
22
27
  kit.valid? ? kit : nil
23
28
  end
24
29
  end
data/lib/miteru.rb CHANGED
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "memist"
4
+ require "semantic_logger"
5
+
3
6
  require "miteru/version"
4
7
 
5
8
  require "miteru/configuration"
@@ -7,6 +10,12 @@ require "miteru/database"
7
10
 
8
11
  require "miteru/record"
9
12
 
13
+ require "miteru/mixin"
14
+
15
+ require "miteru/notifiers/base"
16
+ require "miteru/notifiers/slack"
17
+ require "miteru/notifiers/urlscan"
18
+
10
19
  require "miteru/error"
11
20
  require "miteru/http_client"
12
21
  require "miteru/kit"
@@ -14,8 +23,20 @@ require "miteru/website"
14
23
  require "miteru/downloader"
15
24
  require "miteru/feeds"
16
25
  require "miteru/attachement"
17
- require "miteru/notifier"
18
26
  require "miteru/crawler"
19
27
  require "miteru/cli"
20
28
 
21
- module Miteru; end
29
+ # Load .env
30
+ require "dotenv/load"
31
+
32
+ module Miteru
33
+ class << self
34
+ include Memist::Memoizable
35
+ def logger
36
+ SemanticLogger.default_level = :info
37
+ SemanticLogger.add_appender(io: $stderr, formatter: :color)
38
+ SemanticLogger["Miteru"]
39
+ end
40
+ memoize :logger
41
+ end
42
+ end
data/miteru.gemspec CHANGED
@@ -9,6 +9,7 @@ Gem::Specification.new do |spec|
9
9
  spec.version = Miteru::VERSION
10
10
  spec.authors = ["Manabu Niseki"]
11
11
  spec.email = ["manabu.niseki@gmail.com"]
12
+ spec.metadata["rubygems_mfa_required"] = "true"
12
13
 
13
14
  spec.summary = "An experimental phishing kit detector"
14
15
  spec.description = "An experimental phishing kit detector"
@@ -24,28 +25,31 @@ Gem::Specification.new do |spec|
24
25
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
26
  spec.require_paths = ["lib"]
26
27
 
27
- spec.add_development_dependency "bundler", "~> 2.2"
28
- spec.add_development_dependency "coveralls_reborn", "~> 0.22"
28
+ spec.add_development_dependency "bundler", "~> 2.3"
29
+ spec.add_development_dependency "coveralls_reborn", "~> 0.25"
29
30
  spec.add_development_dependency "glint", "~> 0.1"
30
31
  spec.add_development_dependency "mysql2", "~> 0.5"
31
- spec.add_development_dependency "overcommit", "~> 0.58"
32
- spec.add_development_dependency "pg", "~> 1.2"
32
+ spec.add_development_dependency "overcommit", "~> 0.59"
33
+ spec.add_development_dependency "pg", "~> 1.4"
33
34
  spec.add_development_dependency "rake", "~> 13.0"
34
- spec.add_development_dependency "rspec", "~> 3.10"
35
- spec.add_development_dependency "standard", "~> 1.3"
36
- spec.add_development_dependency "vcr", "~> 6.0"
37
- spec.add_development_dependency "webmock", "~> 3.14"
35
+ spec.add_development_dependency "rspec", "~> 3.11"
36
+ spec.add_development_dependency "standard", "~> 1.14"
37
+ spec.add_development_dependency "vcr", "~> 6.1"
38
+ spec.add_development_dependency "webmock", "~> 3.17"
38
39
  spec.add_development_dependency "webrick", "~> 1.7.0"
39
40
 
40
- spec.add_dependency "activerecord", "~> 6.1"
41
+ spec.add_dependency "activerecord", "~> 7.0"
41
42
  spec.add_dependency "colorize", "~> 0.8"
42
- spec.add_dependency "down", "~> 5.2"
43
- spec.add_dependency "http", "~> 5.0"
44
- spec.add_dependency "oga", "~> 3.3"
45
- spec.add_dependency "parallel", "~> 1.20"
43
+ spec.add_dependency "dotenv", "2.8.1"
44
+ spec.add_dependency "down", "~> 5.3"
45
+ spec.add_dependency "http", "~> 5.1"
46
+ spec.add_dependency "memist", "2.0.2"
47
+ spec.add_dependency "oga", "~> 3.4"
48
+ spec.add_dependency "parallel", "~> 1.22"
49
+ spec.add_dependency "semantic_logger", "4.11.0"
46
50
  spec.add_dependency "slack-notifier", "~> 2.4"
47
51
  spec.add_dependency "sqlite3", "~> 1.4"
48
- spec.add_dependency "thor", "~> 1.1"
49
- spec.add_dependency "urlscan", "~> 0.7"
52
+ spec.add_dependency "thor", "~> 1.2"
53
+ spec.add_dependency "urlscan", "~> 0.8"
50
54
  spec.add_dependency "uuidtools", "~> 2.2"
51
55
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.1
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-29 00:00:00.000000000 Z
11
+ date: 2022-08-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,28 +16,28 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '2.2'
19
+ version: '2.3'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '2.2'
26
+ version: '2.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: coveralls_reborn
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0.22'
33
+ version: '0.25'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0.22'
40
+ version: '0.25'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: glint
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -72,28 +72,28 @@ dependencies:
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '0.58'
75
+ version: '0.59'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '0.58'
82
+ version: '0.59'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: pg
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '1.2'
89
+ version: '1.4'
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '1.2'
96
+ version: '1.4'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: rake
99
99
  requirement: !ruby/object:Gem::Requirement
@@ -114,56 +114,56 @@ dependencies:
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: '3.10'
117
+ version: '3.11'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: '3.10'
124
+ version: '3.11'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: standard
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '1.3'
131
+ version: '1.14'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '1.3'
138
+ version: '1.14'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: vcr
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '6.0'
145
+ version: '6.1'
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '6.0'
152
+ version: '6.1'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: webmock
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
157
  - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '3.14'
159
+ version: '3.17'
160
160
  type: :development
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
164
  - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '3.14'
166
+ version: '3.17'
167
167
  - !ruby/object:Gem::Dependency
168
168
  name: webrick
169
169
  requirement: !ruby/object:Gem::Requirement
@@ -184,14 +184,14 @@ dependencies:
184
184
  requirements:
185
185
  - - "~>"
186
186
  - !ruby/object:Gem::Version
187
- version: '6.1'
187
+ version: '7.0'
188
188
  type: :runtime
189
189
  prerelease: false
190
190
  version_requirements: !ruby/object:Gem::Requirement
191
191
  requirements:
192
192
  - - "~>"
193
193
  - !ruby/object:Gem::Version
194
- version: '6.1'
194
+ version: '7.0'
195
195
  - !ruby/object:Gem::Dependency
196
196
  name: colorize
197
197
  requirement: !ruby/object:Gem::Requirement
@@ -206,62 +206,104 @@ dependencies:
206
206
  - - "~>"
207
207
  - !ruby/object:Gem::Version
208
208
  version: '0.8'
209
+ - !ruby/object:Gem::Dependency
210
+ name: dotenv
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - '='
214
+ - !ruby/object:Gem::Version
215
+ version: 2.8.1
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - '='
221
+ - !ruby/object:Gem::Version
222
+ version: 2.8.1
209
223
  - !ruby/object:Gem::Dependency
210
224
  name: down
211
225
  requirement: !ruby/object:Gem::Requirement
212
226
  requirements:
213
227
  - - "~>"
214
228
  - !ruby/object:Gem::Version
215
- version: '5.2'
229
+ version: '5.3'
216
230
  type: :runtime
217
231
  prerelease: false
218
232
  version_requirements: !ruby/object:Gem::Requirement
219
233
  requirements:
220
234
  - - "~>"
221
235
  - !ruby/object:Gem::Version
222
- version: '5.2'
236
+ version: '5.3'
223
237
  - !ruby/object:Gem::Dependency
224
238
  name: http
225
239
  requirement: !ruby/object:Gem::Requirement
226
240
  requirements:
227
241
  - - "~>"
228
242
  - !ruby/object:Gem::Version
229
- version: '5.0'
243
+ version: '5.1'
230
244
  type: :runtime
231
245
  prerelease: false
232
246
  version_requirements: !ruby/object:Gem::Requirement
233
247
  requirements:
234
248
  - - "~>"
235
249
  - !ruby/object:Gem::Version
236
- version: '5.0'
250
+ version: '5.1'
251
+ - !ruby/object:Gem::Dependency
252
+ name: memist
253
+ requirement: !ruby/object:Gem::Requirement
254
+ requirements:
255
+ - - '='
256
+ - !ruby/object:Gem::Version
257
+ version: 2.0.2
258
+ type: :runtime
259
+ prerelease: false
260
+ version_requirements: !ruby/object:Gem::Requirement
261
+ requirements:
262
+ - - '='
263
+ - !ruby/object:Gem::Version
264
+ version: 2.0.2
237
265
  - !ruby/object:Gem::Dependency
238
266
  name: oga
239
267
  requirement: !ruby/object:Gem::Requirement
240
268
  requirements:
241
269
  - - "~>"
242
270
  - !ruby/object:Gem::Version
243
- version: '3.3'
271
+ version: '3.4'
244
272
  type: :runtime
245
273
  prerelease: false
246
274
  version_requirements: !ruby/object:Gem::Requirement
247
275
  requirements:
248
276
  - - "~>"
249
277
  - !ruby/object:Gem::Version
250
- version: '3.3'
278
+ version: '3.4'
251
279
  - !ruby/object:Gem::Dependency
252
280
  name: parallel
253
281
  requirement: !ruby/object:Gem::Requirement
254
282
  requirements:
255
283
  - - "~>"
256
284
  - !ruby/object:Gem::Version
257
- version: '1.20'
285
+ version: '1.22'
258
286
  type: :runtime
259
287
  prerelease: false
260
288
  version_requirements: !ruby/object:Gem::Requirement
261
289
  requirements:
262
290
  - - "~>"
263
291
  - !ruby/object:Gem::Version
264
- version: '1.20'
292
+ version: '1.22'
293
+ - !ruby/object:Gem::Dependency
294
+ name: semantic_logger
295
+ requirement: !ruby/object:Gem::Requirement
296
+ requirements:
297
+ - - '='
298
+ - !ruby/object:Gem::Version
299
+ version: 4.11.0
300
+ type: :runtime
301
+ prerelease: false
302
+ version_requirements: !ruby/object:Gem::Requirement
303
+ requirements:
304
+ - - '='
305
+ - !ruby/object:Gem::Version
306
+ version: 4.11.0
265
307
  - !ruby/object:Gem::Dependency
266
308
  name: slack-notifier
267
309
  requirement: !ruby/object:Gem::Requirement
@@ -296,28 +338,28 @@ dependencies:
296
338
  requirements:
297
339
  - - "~>"
298
340
  - !ruby/object:Gem::Version
299
- version: '1.1'
341
+ version: '1.2'
300
342
  type: :runtime
301
343
  prerelease: false
302
344
  version_requirements: !ruby/object:Gem::Requirement
303
345
  requirements:
304
346
  - - "~>"
305
347
  - !ruby/object:Gem::Version
306
- version: '1.1'
348
+ version: '1.2'
307
349
  - !ruby/object:Gem::Dependency
308
350
  name: urlscan
309
351
  requirement: !ruby/object:Gem::Requirement
310
352
  requirements:
311
353
  - - "~>"
312
354
  - !ruby/object:Gem::Version
313
- version: '0.7'
355
+ version: '0.8'
314
356
  type: :runtime
315
357
  prerelease: false
316
358
  version_requirements: !ruby/object:Gem::Requirement
317
359
  requirements:
318
360
  - - "~>"
319
361
  - !ruby/object:Gem::Version
320
- version: '0.7'
362
+ version: '0.8'
321
363
  - !ruby/object:Gem::Dependency
322
364
  name: uuidtools
323
365
  requirement: !ruby/object:Gem::Requirement
@@ -370,7 +412,10 @@ files:
370
412
  - lib/miteru/feeds/urlscan_pro.rb
371
413
  - lib/miteru/http_client.rb
372
414
  - lib/miteru/kit.rb
373
- - lib/miteru/notifier.rb
415
+ - lib/miteru/mixin.rb
416
+ - lib/miteru/notifiers/base.rb
417
+ - lib/miteru/notifiers/slack.rb
418
+ - lib/miteru/notifiers/urlscan.rb
374
419
  - lib/miteru/record.rb
375
420
  - lib/miteru/version.rb
376
421
  - lib/miteru/website.rb
@@ -380,7 +425,8 @@ files:
380
425
  homepage: https://github.com/ninoseki/miteru
381
426
  licenses:
382
427
  - MIT
383
- metadata: {}
428
+ metadata:
429
+ rubygems_mfa_required: 'true'
384
430
  post_install_message:
385
431
  rdoc_options: []
386
432
  require_paths:
@@ -396,7 +442,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
396
442
  - !ruby/object:Gem::Version
397
443
  version: '0'
398
444
  requirements: []
399
- rubygems_version: 3.2.22
445
+ rubygems_version: 3.2.14
400
446
  signing_key:
401
447
  specification_version: 4
402
448
  summary: An experimental phishing kit detector
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "colorize"
4
- require "slack-notifier"
5
-
6
- module Miteru
7
- class Notifier
8
- def notify(url:, kits:, message:)
9
- attachement = Attachement.new(url)
10
- kits = kits.select(&:filesize)
11
-
12
- if notifiable? && kits.any?
13
- notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
14
- notifier.post(text: message.capitalize, attachments: attachement.to_a)
15
- end
16
-
17
- message = message.colorize(:light_red) if kits.any?
18
- puts "#{url}: #{message}"
19
- end
20
-
21
- def notifiable?
22
- Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
23
- end
24
- end
25
- end