miteru 1.0.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58464f5c1b0b17549201ea547edbcf22cd83bcbbd7ef1a1233ab2d04eeb4c13c
-  data.tar.gz: 6a8048507e00ded2fe11ef1a71b05536a267ee54b6e64b7662cf5f238ace84fd
+  metadata.gz: c858dd689d566a4a12270d7474b9174d60deeabb23f61e1a62440040558f0a3c
+  data.tar.gz: 0f53873e358ab18712c0e9d7c5fc6b6cd863464b467d78e82983ca2386834115
 SHA512:
-  metadata.gz: efb6fcb505fff1451ab5cbff5a0bbc1d3e480aa7cc4f51005b84065c5c174b4c12e9609f07217f7cbf0bdb844dd8efe7430e99e2f3bb7ea3fa7207d9ff531ae4
-  data.tar.gz: 5ab2d22a2c9c647665a3feefd9b579ac336e434a84320db6df4ba3b7e1ea4d44a14b1db7a91f8e63add7e7a0d544d0d791cd07f1d4258065642c2fae35c0002b
+  metadata.gz: 76359bf3d5883acdfdef815045d634a286be8985f21408f9f91349a7e048077309ab468d095d4b49e9e26b19018b4d280b2290beb8620b5d7729e1b9fc5d9586
+  data.tar.gz: 9bb8d92dbb325a74c38a8f05a4ed5967831a70601597d6d45235d9a1571228dcfcc543e255628b763b977bf2a649b3976779a2765c8ba19dd174dc988d275320

data/.github/workflows/test.yml

@@ -42,18 +42,17 @@ jobs:
         ruby: [2.7, "3.0"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
+          bundler: latest
           bundler-cache: true
 
       - name: Install dependencies
         run: |
           sudo apt-get -yqq install libpq-dev libmysqlclient-dev
-          gem install bundler
-          bundle install
 
       - name: Test with PostgreSQL
         env:

data/lib/miteru/attachement.rb

@@ -47,7 +47,7 @@ module Miteru
     end
 
     def defanged_url
-      @defanged_url ||= url.to_s.gsub /\./, "[.]"
+      @defanged_url ||= url.to_s.gsub(/\./, "[.]")
     end
 
     def domain

data/lib/miteru/configuration.rb

@@ -37,12 +37,21 @@ module Miteru
     # @return [String]
     attr_accessor :slack_channel
 
+    # @return [String]
+    attr_accessor :urlscan_api_key
+
+    # @return [String]
+    attr_accessor :urlscan_submit_visibility
+
     # @return [Array<String>]
     attr_reader :valid_extensions
 
     # @return [Array<String>]
     attr_reader :valid_mime_types
 
+    # @return [Integer]
+    attr_reader :file_maxsize
+
     def initialize
       @auto_download = false
       @ayashige = false
@@ -52,10 +61,14 @@ module Miteru
       @size = 100
       @threads = Parallel.processor_count
       @verbose = false
-      @database = ENV["MITERU_DATABASE"] || "miteru.db"
+      @database = ENV.fetch("MITERU_DATABASE", "miteru.db")
+      @file_maxsize = ENV.fetch("FILE_MAXSIZE", 1024 * 1024 * 100).to_i
+
+      @slack_webhook_url = ENV.fetch("SLACK_WEBHOOK_URL", nil)
+      @slack_channel = ENV.fetch("SLACK_CHANNEL", "#general")
 
-      @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
-      @slack_channel = ENV["SLACK_CHANNEL"] || "#general"
+      @urlscan_api_key = ENV.fetch("URLSCAN_API_KEY", nil)
+      @urlscan_submit_visibility = ENV.fetch("URLSCAN_SUBMIT_VISIBILITY", "public")
 
       @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
       @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
@@ -82,7 +95,11 @@ module Miteru
     end
 
     def slack_webhook_url?
-      @slack_webhook_url
+      !@slack_webhook_url.nil?
+    end
+
+    def urlscan_api_key?
+      !@urlscan_api_key.nil?
     end
   end
 

data/lib/miteru/crawler.rb

@@ -11,11 +11,10 @@ module Miteru
     def initialize
       @downloader = Downloader.new(Miteru.configuration.download_to)
       @feeds = Feeds.new
-      @notifier = Notifier.new
     end
 
-    def crawl(url)
-      website = Website.new(url)
+    def crawl(entry)
+      website = Website.new(entry.url, entry.source)
       downloader.download_kits(website.kits) if website.has_kits? && auto_download?
       notify(website) if website.has_kits? || verbose?
     rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
@@ -23,11 +22,11 @@ module Miteru
     end
 
     def execute
-      suspicious_urls = feeds.suspicious_urls
-      puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
+      suspicious_entries = feeds.suspicious_entries
+      Miteru.logger.info "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
 
-      Parallel.each(suspicious_urls, in_threads: threads) do |url|
-        crawl url
+      Parallel.each(suspicious_entries, in_threads: threads) do |entry|
+        crawl entry
       end
     end
 
@@ -36,7 +35,9 @@ module Miteru
     end
 
     def notify(website)
-      @notifier.notify(url: website.url, kits: website.kits, message: website.message)
+      Parallel.each(notifiers) do |notifier|
+        notifier.notify website
+      end
     end
 
     def auto_download?
@@ -47,6 +48,12 @@ module Miteru
       Miteru.configuration.verbose?
     end
 
+    private
+
+    def notifiers
+      @notifiers ||= [Notifiers::Slack.new, Notifiers::UrlScan.new].select(&:notifiable?)
+    end
+
     class << self
       def execute
         new.execute

data/lib/miteru/database.rb

@@ -2,7 +2,7 @@
 
 require "active_record"
 
-class InitialSchema < ActiveRecord::Migration[6.1]
+class InitialSchema < ActiveRecord::Migration[7.0]
   def change
     create_table :records, if_not_exists: true do |t|
       t.string :hash, null: false, index: { unique: true }
@@ -19,6 +19,12 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
+class V11Schema < ActiveRecord::Migration[7.0]
+  def change
+    add_column :records, :source, :string, if_not_exists: true
+  end
+end
+
 def adapter
   return "postgresql" if Miteru.configuration.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Miteru.configuration.database.start_with?("mysql2://")
@@ -44,6 +50,7 @@ module Miteru
         ActiveRecord::Migration.verbose = false
 
         InitialSchema.migrate(:up)
+        V11Schema.migrate(:up)
       rescue StandardError => _e
         # Do nothing
       end
@@ -57,6 +64,7 @@ module Miteru
         return unless ActiveRecord::Base.connected?
 
         InitialSchema.migrate(:down)
+        V11Schema.migrate(:down)
       end
     end
   end

data/lib/miteru/downloader.rb

@@ -26,7 +26,15 @@ module Miteru
       begin
         downloaded_as = HTTPClient.download(kit.url, destination)
       rescue Down::Error => e
-        puts "Failed to download: #{kit.url} (#{e})"
+        Miteru.logger.error "Failed to download: #{kit.url} (#{e})"
+        return
+      end
+
+      # check filesize
+      size = File.size downloaded_as
+      if size > Miteru.configuration.file_maxsize
+        Miteru.logger.info "#{kit.url}'s filesize exceeds the limit: #{size}"
+        FileUtils.rm downloaded_as
         return
       end
 
@@ -35,14 +43,14 @@ module Miteru
       ActiveRecord::Base.connection_pool.with_connection do
         # Remove a downloaded file if it is not unique
         unless Record.unique_hash?(hash)
-          puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
+          Miteru.logger.info "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
           FileUtils.rm downloaded_as
           return
         end
 
         # Record a kit in DB
         Record.create_by_kit_and_hash(kit, hash)
-        puts "Download #{kit.url} as #{downloaded_as}"
+        Miteru.logger.info "Download #{kit.url} as #{downloaded_as}"
       end
     end
 

data/lib/miteru/feeds/ayashige.rb

@@ -7,13 +7,13 @@ module Miteru
   class Feeds
     class Ayashige < Feed
       HOST = "ayashige.herokuapp.com"
-      URL = "https://#{HOST}".freeze
+      URL = "https://#{HOST}"
 
       def urls
-        url = url_for("/feed")
+        url = url_for("/api/v1/domains/")
         res = JSON.parse(get(url))
 
-        domains = res.map { |item| item["domain"] }
+        domains = res.map { |item| item["fqdn"] }
         domains.map do |domain|
           [
             "https://#{domain}",
@@ -21,7 +21,7 @@ module Miteru
           ]
         end.flatten
       rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
-        puts "Failed to load ayashige feed (#{e})"
+        Miteru.logger.error "Failed to load ayashige feed (#{e})"
         []
       end
 

data/lib/miteru/feeds/feed.rb

@@ -3,10 +3,43 @@
 module Miteru
   class Feeds
     class Feed
+      include Mixins::URL
+
+      def source
+        @source ||= self.class.to_s.split("::").last
+      end
+
+      #
+      # Return URLs
+      #
+      # @return [Array<String>] URLs
+      #
       def urls
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      #
+      # Return entries
+      #
+      # @return [Array<Miteru::Entry>]
+      #
+      def entries
+        breakdowend_urls.map do |url|
+          Entry.new(url, source)
+        end
+      end
+
+      #
+      # Return breakdowned URLs
+      #
+      # @return [Array<String>] Breakdowned URLs
+      #
+      def breakdowend_urls
+        urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
+          breakdown(url, Miteru.configuration.directory_traveling?)
+        end.flatten.uniq
+      end
+
       private
 
       def get(url)

data/lib/miteru/feeds/phishing_database.rb

@@ -12,7 +12,7 @@ module Miteru
         body = get(URL)
         body.to_s.lines.map(&:chomp)
       rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
-        puts "Failed to load phishing database feed (#{e})"
+        info "Failed to load phishing database feed (#{e})"
         []
       end
     end

data/lib/miteru/feeds/phishstats.rb

@@ -14,7 +14,7 @@ module Miteru
           entry["url"]
         end
       rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
-        puts "Failed to load PhishStats feed (#{e})"
+        Miteru.logger.error "Failed to load PhishStats feed (#{e})"
         []
       end
 

data/lib/miteru/feeds/urlscan.rb

@@ -13,13 +13,13 @@ module Miteru
       end
 
       def api
-        @api ||= ::UrlScan::API.new
+        @api ||= ::UrlScan::API.new(Miteru.configuration.urlscan_api_key)
       end
 
       def urls
         urls_from_community_feed
       rescue ::UrlScan::ResponseError => e
-        puts "Failed to load urlscan.io feed (#{e})"
+        Miteru.logger.error "Failed to load urlscan.io feed (#{e})"
         []
       end
 

data/lib/miteru/feeds/urlscan_pro.rb

@@ -6,20 +6,20 @@ module Miteru
   class Feeds
     class UrlScanPro < Feed
       def api
-        @api ||= ::UrlScan::API.new
+        @api ||= ::UrlScan::API.new(Miteru.configuration.urlscan_api_key)
       end
 
       def urls
         urls_from_pro_feed
       rescue ::UrlScan::ResponseError => e
-        puts "Failed to load urlscan.io pro feed (#{e})"
+        Miteru.logger.error "Failed to load urlscan.io pro feed (#{e})"
         []
       end
 
       private
 
       def api_key?
-        ENV.key? "URLSCAN_API_KEY"
+        Miteru.configuration.urlscan_api_key?
       end
 
       def urls_from_pro_feed

data/lib/miteru/feeds.rb

@@ -8,6 +8,18 @@ require_relative "./feeds/urlscan"
 require_relative "./feeds/urlscan_pro"
 
 module Miteru
+  class Entry
+    # @return [String]
+    attr_reader :url
+    # @return [String]
+    attr_reader :source
+
+    def initialize(url, source)
+      @url = url
+      @source = source
+    end
+  end
+
   class Feeds
     IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
 
@@ -21,43 +33,13 @@ module Miteru
       ].compact
     end
 
-    def directory_traveling?
-      Miteru.configuration.directory_traveling?
-    end
-
-    def suspicious_urls
-      @suspicious_urls ||= [].tap do |arr|
-        urls = @feeds.map do |feed|
-          feed.urls.select { |url| url.start_with?("http://", "https://") }
-        end.flatten.uniq
-
-        urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
-      end
-    end
-
-    def breakdown(url)
-      begin
-        uri = URI.parse(url)
-      rescue URI::InvalidURIError => _e
-        return []
-      end
-
-      base = "#{uri.scheme}://#{uri.hostname}"
-      return [base] unless directory_traveling?
-
-      segments = uri.path.split("/")
-      return [base] if segments.length.zero?
-
-      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
-
-      urls.reject do |breakdowned_url|
-        # Reject a url which ends with specific extension names
-        invalid_extension? breakdowned_url
-      end
-    end
-
-    def invalid_extension?(url)
-      IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+    #
+    # Returns a list of suspicious entries
+    #
+    # @return [Array<Entry>]
+    #
+    def suspicious_entries
+      @suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
     end
   end
 end

data/lib/miteru/http_client.rb

@@ -7,7 +7,7 @@ require "uri"
 module Miteru
   class HTTPClient
     DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
-    URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
+    URLSCAN_UA = "miteru/#{Miteru::VERSION}"
 
     attr_reader :ssl_context
 

data/lib/miteru/kit.rb

@@ -9,10 +9,27 @@ module Miteru
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
     VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
 
-    attr_reader :url, :status, :content_length, :mime_type, :headers
+    # @return [String]
+    attr_reader :url
 
-    def initialize(url)
+    # @return [String]
+    attr_reader :source
+
+    # @return [Integer, nil]
+    attr_reader :status
+
+    # @return [Integer, nil]
+    attr_reader :content_length
+
+    # @return [String, nil]
+    attr_reader :mime_type
+
+    # @return [Hash, nil]
+    attr_reader :headers
+
+    def initialize(url, source)
       @url = url
+      @source = source
 
       @content_length = nil
       @mime_type = nil
@@ -45,8 +62,12 @@ module Miteru
       "#{base_dir}/#{filename_to_download}"
     end
 
+    def downloaded?
+      File.exist?(filepath_to_download)
+    end
+
     def filesize
-      return nil unless File.exist?(filepath_to_download)
+      return nil unless downloaded?
 
       File.size filepath_to_download
     end
@@ -54,7 +75,8 @@ module Miteru
     def filename_with_size
       return filename unless filesize
 
-      "#{filename}(#{filesize / 1024}KB)"
+      kb = (filesize.to_f / 1024.0).ceil
+      "#{filename}(#{kb}KB)"
     end
 
     def id

data/lib/miteru/mixin.rb

@@ -0,0 +1,47 @@
+module Miteru
+  module Mixins
+    module URL
+      IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
+
+      #
+      # Validate extension of a URL
+      #
+      # @param [String] url
+      #
+      # @return [Boolean]
+      #
+      def invalid_extension?(url)
+        IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+      end
+
+      #
+      # Breakdown a URL into URLs
+      #
+      # @param [String] url
+      # @param [Boolean] enable_directory_traveling
+      #
+      # @return [Array<String>]
+      #
+      def breakdown(url, enable_directory_traveling)
+        begin
+          uri = URI.parse(url)
+        rescue URI::InvalidURIError => _e
+          return []
+        end
+
+        base = "#{uri.scheme}://#{uri.hostname}"
+        return [base] unless enable_directory_traveling
+
+        segments = uri.path.split("/")
+        return [base] if segments.length.zero?
+
+        urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
+
+        urls.reject do |breakdowned_url|
+          # Reject a url which ends with specific extension names
+          invalid_extension? breakdowned_url
+        end
+      end
+    end
+  end
+end

data/lib/miteru/notifiers/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Notifiers
+    class Base
+      def notify(website)
+        raise NotImplementedError
+      end
+
+      def notifiable?
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/miteru/notifiers/slack.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Notifiers
+    class Slack < Base
+      #
+      # Notifiy to Slack
+      #
+      # @param [Miteru::Website website
+      #
+      def notify(website)
+        attachement = Attachement.new(website.url)
+        kits = website.kits.select(&:downloaded?)
+
+        if notifiable? && kits.any?
+          notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
+          notifier.post(text: website.message.capitalize, attachments: attachement.to_a)
+        end
+
+        message = kits.any? ? website.message.colorize(:light_red) : website.message
+        Miteru.logger.info "#{website.url}: #{message}"
+      end
+
+      def notifiable?
+        Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
+      end
+    end
+  end
+end

data/lib/miteru/notifiers/urlscan.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "urlscan"
+
+module Miteru
+  module Notifiers
+    class UrlScan < Base
+      #
+      # Notifiy to urlscan.io
+      #
+      # @param [Miteru::Website website
+      #
+      def notify(website)
+        kits = website.kits.select(&:downloaded?)
+        return unless notifiable? && kits.any?
+
+        kits.each { |kit| submit(kit.url) }
+      end
+
+      def notifiable?
+        Miteru.configuration.urlscan_api_key?
+      end
+
+      private
+
+      def api
+        @api ||= ::UrlScan::API.new(Miteru.configuration.urlscan_api_key)
+      end
+
+      def submit(url)
+        api.submit(url, tags: ["miteru", "phishkit"], visibility: Miteru.configuration.urlscan_submit_visibility)
+      rescue StandardError
+        # do nothing
+      end
+    end
+  end
+end

data/lib/miteru/record.rb

@@ -30,6 +30,7 @@ module Miteru
       def create_by_kit_and_hash(kit, hash)
         record = new(
           hash: hash,
+          source: kit.source,
           hostname: kit.hostname,
           url: kit.decoded_url,
           headers: kit.headers,

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "1.0.1"
+  VERSION = "1.2.0"
 end

data/lib/miteru/website.rb

@@ -6,10 +6,15 @@ module Miteru
   class Website
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
 
+    # @return [String]
     attr_reader :url
 
-    def initialize(url)
+    # @return [String]
+    attr_reader :source
+
+    def initialize(url, source)
       @url = url
+      @source = source
     end
 
     def title
@@ -18,7 +23,7 @@ module Miteru
 
     def kits
       @kits ||= links.filter_map do |link|
-        kit = Kit.new(link)
+        kit = Kit.new(link, source)
         kit.valid? ? kit : nil
       end
     end

data/lib/miteru.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "memist"
+require "semantic_logger"
+
 require "miteru/version"
 
 require "miteru/configuration"
@@ -7,6 +10,12 @@ require "miteru/database"
 
 require "miteru/record"
 
+require "miteru/mixin"
+
+require "miteru/notifiers/base"
+require "miteru/notifiers/slack"
+require "miteru/notifiers/urlscan"
+
 require "miteru/error"
 require "miteru/http_client"
 require "miteru/kit"
@@ -14,8 +23,20 @@ require "miteru/website"
 require "miteru/downloader"
 require "miteru/feeds"
 require "miteru/attachement"
-require "miteru/notifier"
 require "miteru/crawler"
 require "miteru/cli"
 
-module Miteru; end
+# Load .env
+require "dotenv/load"
+
+module Miteru
+  class << self
+    include Memist::Memoizable
+    def logger
+      SemanticLogger.default_level = :info
+      SemanticLogger.add_appender(io: $stderr, formatter: :color)
+      SemanticLogger["Miteru"]
+    end
+    memoize :logger
+  end
+end

data/miteru.gemspec

@@ -9,6 +9,7 @@ Gem::Specification.new do |spec|
   spec.version = Miteru::VERSION
   spec.authors = ["Manabu Niseki"]
   spec.email = ["manabu.niseki@gmail.com"]
+  spec.metadata["rubygems_mfa_required"] = "true"
 
   spec.summary = "An experimental phishing kit detector"
   spec.description = "An experimental phishing kit detector"
@@ -24,28 +25,31 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.2"
-  spec.add_development_dependency "coveralls_reborn", "~> 0.22"
+  spec.add_development_dependency "bundler", "~> 2.3"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.25"
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "mysql2", "~> 0.5"
-  spec.add_development_dependency "overcommit", "~> 0.58"
-  spec.add_development_dependency "pg", "~> 1.2"
+  spec.add_development_dependency "overcommit", "~> 0.59"
+  spec.add_development_dependency "pg", "~> 1.4"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.10"
-  spec.add_development_dependency "standard", "~> 1.3"
-  spec.add_development_dependency "vcr", "~> 6.0"
-  spec.add_development_dependency "webmock", "~> 3.14"
+  spec.add_development_dependency "rspec", "~> 3.11"
+  spec.add_development_dependency "standard", "~> 1.14"
+  spec.add_development_dependency "vcr", "~> 6.1"
+  spec.add_development_dependency "webmock", "~> 3.17"
   spec.add_development_dependency "webrick", "~> 1.7.0"
 
-  spec.add_dependency "activerecord", "~> 6.1"
+  spec.add_dependency "activerecord", "~> 7.0"
   spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "down", "~> 5.2"
-  spec.add_dependency "http", "~> 5.0"
-  spec.add_dependency "oga", "~> 3.3"
-  spec.add_dependency "parallel", "~> 1.20"
+  spec.add_dependency "dotenv", "2.8.1"
+  spec.add_dependency "down", "~> 5.3"
+  spec.add_dependency "http", "~> 5.1"
+  spec.add_dependency "memist", "2.0.2"
+  spec.add_dependency "oga", "~> 3.4"
+  spec.add_dependency "parallel", "~> 1.22"
+  spec.add_dependency "semantic_logger", "4.11.0"
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "~> 1.1"
-  spec.add_dependency "urlscan", "~> 0.7"
+  spec.add_dependency "thor", "~> 1.2"
+  spec.add_dependency "urlscan", "~> 0.8"
   spec.add_dependency "uuidtools", "~> 2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-29 00:00:00.000000000 Z
+date: 2022-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '2.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.22'
+        version: '0.25'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.22'
+        version: '0.25'
 - !ruby/object:Gem::Dependency
   name: glint
   requirement: !ruby/object:Gem::Requirement
@@ -72,28 +72,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.58'
+        version: '0.59'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.58'
+        version: '0.59'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -114,56 +114,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.10'
+        version: '3.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.10'
+        version: '3.11'
 - !ruby/object:Gem::Dependency
   name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.14'
+        version: '3.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.14'
+        version: '3.17'
 - !ruby/object:Gem::Dependency
   name: webrick
   requirement: !ruby/object:Gem::Requirement
@@ -184,14 +184,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -206,62 +206,104 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
 - !ruby/object:Gem::Dependency
   name: down
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '5.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '5.3'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: memist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.2
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: '3.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: '3.4'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.20'
+        version: '1.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.20'
+        version: '1.22'
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.11.0
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -296,28 +338,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: urlscan
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement
@@ -370,7 +412,10 @@ files:
 - lib/miteru/feeds/urlscan_pro.rb
 - lib/miteru/http_client.rb
 - lib/miteru/kit.rb
-- lib/miteru/notifier.rb
+- lib/miteru/mixin.rb
+- lib/miteru/notifiers/base.rb
+- lib/miteru/notifiers/slack.rb
+- lib/miteru/notifiers/urlscan.rb
 - lib/miteru/record.rb
 - lib/miteru/version.rb
 - lib/miteru/website.rb
@@ -380,7 +425,8 @@ files:
 homepage: https://github.com/ninoseki/miteru
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -396,7 +442,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.2.14
 signing_key:
 specification_version: 4
 summary: An experimental phishing kit detector

data/lib/miteru/notifier.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require "colorize"
-require "slack-notifier"
-
-module Miteru
-  class Notifier
-    def notify(url:, kits:, message:)
-      attachement = Attachement.new(url)
-      kits = kits.select(&:filesize)
-
-      if notifiable? && kits.any?
-        notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
-        notifier.post(text: message.capitalize, attachments: attachement.to_a)
-      end
-
-      message = message.colorize(:light_red) if kits.any?
-      puts "#{url}: #{message}"
-    end
-
-    def notifiable?
-      Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
-    end
-  end
-end