miteru 0.14.7 → 1.1.0

data/lib/miteru/downloader.rb

@@ -6,13 +6,12 @@ require "uri"
 
 module Miteru
   class Downloader
-    attr_reader :base_dir
-    attr_reader :memo
+    attr_reader :base_dir, :memo
 
     def initialize(base_dir = "/tmp")
       @base_dir = base_dir
       @memo = {}
-      raise ArgumentError, "#{base_dir} is not exist." unless Dir.exist?(base_dir)
+      raise ArgumentError, "#{base_dir} doesn't exist." unless Dir.exist?(base_dir)
     end
 
     def download_kits(kits)
@@ -22,23 +21,29 @@ module Miteru
     private
 
     def download_kit(kit)
-      destination = kit.download_filepath
+      destination = kit.filepath_to_download
+
       begin
-        downloaded_filepath = HTTPClient.download(kit.url, destination)
-        hash = sha256(downloaded_filepath)
-        if duplicated?(hash)
-          puts "Do not download #{kit.url} because there is a duplicate file in the directory (SHA256: #{hash})."
-          FileUtils.rm downloaded_filepath
-        else
-          puts "Download #{kit.url} as #{downloaded_filepath}"
-        end
+        downloaded_as = HTTPClient.download(kit.url, destination)
       rescue Down::Error => e
         puts "Failed to download: #{kit.url} (#{e})"
+        return
       end
-    end
 
-    def filepath_to_download(filename)
-      "#{base_dir}/#{filename}"
+      hash = sha256(downloaded_as)
+
+      ActiveRecord::Base.connection_pool.with_connection do
+        # Remove a downloaded file if it is not unique
+        unless Record.unique_hash?(hash)
+          puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
+          FileUtils.rm downloaded_as
+          return
+        end
+
+        # Record a kit in DB
+        Record.create_by_kit_and_hash(kit, hash)
+        puts "Download #{kit.url} as #{downloaded_as}"
+      end
     end
 
     def sha256(path)
@@ -49,13 +54,5 @@ module Miteru
       memo[path] = hash
       hash
     end
-
-    def sha256s
-      Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
-    end
-
-    def duplicated?(hash)
-      sha256s.count(hash) > 1
-    end
   end
 end

data/lib/miteru/error.rb

@@ -2,5 +2,6 @@
 
 module Miteru
   class HTTPResponseError < StandardError; end
+
   class DownloadError < StandardError; end
 end

data/lib/miteru/feeds/ayashige.rb

@@ -10,10 +10,10 @@ module Miteru
       URL = "https://#{HOST}"
 
       def urls
-        url = url_for("/feed")
+        url = url_for("/api/v1/domains/")
         res = JSON.parse(get(url))
 
-        domains = res.map { |item| item["domain"] }
+        domains = res.map { |item| item["fqdn"] }
         domains.map do |domain|
           [
             "https://#{domain}",

data/lib/miteru/feeds/feed.rb

@@ -3,10 +3,43 @@
 module Miteru
   class Feeds
     class Feed
+      include Mixins::URL
+
+      def source
+        @source ||= self.class.to_s.split("::").last
+      end
+
+      #
+      # Return URLs
+      #
+      # @return [Array<String>] URLs
+      #
       def urls
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      #
+      # Return entries
+      #
+      # @return [Array<Miteru::Entry>]
+      #
+      def entries
+        breakdowend_urls.map do |url|
+          Entry.new(url, source)
+        end
+      end
+
+      #
+      # Return breakdowned URLs
+      #
+      # @return [Array<String>] Breakdowned URLs
+      #
+      def breakdowend_urls
+        urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
+          breakdown(url, Miteru.configuration.directory_traveling?)
+        end.flatten.uniq
+      end
+
       private
 
       def get(url)

data/lib/miteru/feeds/phishstats.rb

@@ -11,7 +11,7 @@ module Miteru
       def urls
         json = JSON.parse(get(URL))
         json.map do |entry|
-          entry.dig("url")
+          entry["url"]
         end
       rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
         puts "Failed to load PhishStats feed (#{e})"

data/lib/miteru/feeds/urlscan_pro.rb

@@ -27,7 +27,7 @@ module Miteru
 
         res = api.pro.phishfeed
         results = res["results"] || []
-        results.map { |result| result.dig("page_url") }
+        results.map { |result| result["page_url"] }
       rescue ArgumentError => _e
         []
       end

data/lib/miteru/feeds.rb

@@ -8,8 +8,20 @@ require_relative "./feeds/urlscan"
 require_relative "./feeds/urlscan_pro"
 
 module Miteru
+  class Entry
+    # @return [String]
+    attr_reader :url
+    # @return [String]
+    attr_reader :source
+
+    def initialize(url, source)
+      @url = url
+      @source = source
+    end
+  end
+
   class Feeds
-    IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
+    IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
 
     def initialize
       @feeds = [
@@ -21,43 +33,13 @@ module Miteru
       ].compact
     end
 
-    def directory_traveling?
-      Miteru.configuration.directory_traveling?
-    end
-
-    def suspicious_urls
-      @suspicious_urls ||= [].tap do |arr|
-        urls = @feeds.map do |feed|
-          feed.urls.select { |url| url.start_with?("http://", "https://") }
-        end.flatten.uniq
-
-        urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
-      end
-    end
-
-    def breakdown(url)
-      begin
-        uri = URI.parse(url)
-      rescue URI::InvalidURIError => _e
-        return []
-      end
-
-      base = "#{uri.scheme}://#{uri.hostname}"
-      return [base] unless directory_traveling?
-
-      segments = uri.path.split("/")
-      return [base] if segments.length.zero?
-
-      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
-
-      urls.reject do |breakdowned_url|
-        # Reject a url which ends with specific extension names
-        invalid_extension? breakdowned_url
-      end
-    end
-
-    def invalid_extension?(url)
-      IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+    #
+    # Returns a list of suspicious entries
+    #
+    # @return [Array<Entry>]
+    #
+    def suspicious_entries
+      @suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
     end
   end
 end

data/lib/miteru/http_client.rb

@@ -7,7 +7,7 @@ require "uri"
 module Miteru
   class HTTPClient
     DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
-    URLSCAN_UA = "miteru/#{Miteru::VERSION}"
+    URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
 
     attr_reader :ssl_context
 
@@ -27,18 +27,18 @@ module Miteru
       options = options.merge default_options
 
       HTTP.follow
-          .timeout(3)
-          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-          .head(url, options)
+        .timeout(3)
+        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+        .head(url, options)
     end
 
     def get(url, options = {})
       options = options.merge default_options
 
       HTTP.follow
-          .timeout(write: 2, connect: 5, read: 10)
-          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-          .get(url, options)
+        .timeout(write: 2, connect: 5, read: 10)
+        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+        .get(url, options)
     end
 
     def post(url, options = {})

data/lib/miteru/kit.rb

@@ -1,28 +1,43 @@
 # frozen_string_literal: true
 
 require "cgi"
-require "securerandom"
+require "uuidtools"
+require "uri"
 
 module Miteru
   class Kit
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
     VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
 
+    # @return [String]
     attr_reader :url
 
+    # @return [String]
+    attr_reader :source
+
+    # @return [Integer, nil]
     attr_reader :status
+
+    # @return [Integer, nil]
     attr_reader :content_length
+
+    # @return [String, nil]
     attr_reader :mime_type
 
-    def initialize(url)
+    # @return [Hash, nil]
+    attr_reader :headers
+
+    def initialize(url, source)
       @url = url
+      @source = source
 
       @content_length = nil
       @mime_type = nil
       @status = nil
+      @headers = nil
     end
 
-    def valid?;
+    def valid?
       # make a HEAD request for the validation
       before_validation
 
@@ -36,41 +51,50 @@ module Miteru
     end
 
     def basename
-      File.basename(url)
+      @basename ||= File.basename(url)
     end
 
     def filename
-      CGI.unescape basename
+      @filename ||= CGI.unescape(basename)
+    end
+
+    def filepath_to_download
+      "#{base_dir}/#{filename_to_download}"
     end
 
-    def download_filepath
-      "#{base_dir}/#{download_filename}"
+    def downloaded?
+      File.exist?(filepath_to_download)
     end
 
     def filesize
-      return nil unless File.exist?(download_filepath)
+      return nil unless downloaded?
 
-      File.size download_filepath
+      File.size filepath_to_download
     end
 
     def filename_with_size
       return filename unless filesize
 
-      "#{filename}(#{filesize / 1024}KB)"
+      kb = (filesize.to_f / 1024.0).ceil
+      "#{filename}(#{kb}KB)"
     end
 
-    private
-
     def id
-      @id ||= SecureRandom.hex(10)
+      @id ||= UUIDTools::UUID.random_create.to_s
     end
 
     def hostname
-      URI(url).hostname
+      @hostname ||= URI(url).hostname
+    end
+
+    def decoded_url
+      @decoded_url ||= URI.decode_www_form_component(url)
     end
 
-    def download_filename
-      "#{hostname}_#{filename}_#{id}#{extname}"
+    private
+
+    def filename_to_download
+      "#{id}#{extname}"
     end
 
     def base_dir
@@ -86,6 +110,7 @@ module Miteru
       @content_length = res.content_length
       @mime_type = res.content_type.mime_type.to_s
       @status = res.status
+      @headers = res.headers.to_h
     rescue StandardError
       # do nothing
     end

data/lib/miteru/mixin.rb

@@ -0,0 +1,47 @@
+module Miteru
+  module Mixins
+    module URL
+      IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
+
+      #
+      # Validate extension of a URL
+      #
+      # @param [String] url
+      #
+      # @return [Boolean]
+      #
+      def invalid_extension?(url)
+        IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+      end
+
+      #
+      # Breakdown a URL into URLs
+      #
+      # @param [String] url
+      # @param [Boolean] enable_directory_traveling
+      #
+      # @return [Array<String>]
+      #
+      def breakdown(url, enable_directory_traveling)
+        begin
+          uri = URI.parse(url)
+        rescue URI::InvalidURIError => _e
+          return []
+        end
+
+        base = "#{uri.scheme}://#{uri.hostname}"
+        return [base] unless enable_directory_traveling
+
+        segments = uri.path.split("/")
+        return [base] if segments.length.zero?
+
+        urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
+
+        urls.reject do |breakdowned_url|
+          # Reject a url which ends with specific extension names
+          invalid_extension? breakdowned_url
+        end
+      end
+    end
+  end
+end

data/lib/miteru/notifier.rb

@@ -7,31 +7,19 @@ module Miteru
   class Notifier
     def notify(url:, kits:, message:)
       attachement = Attachement.new(url)
-      kits = kits.select(&:filesize)
+      kits = kits.select(&:downloaded?)
 
-      if post_to_slack? && kits.any?
-        notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
-        notifier.post(text: message, attachments: attachement.to_a)
+      if notifiable? && kits.any?
+        notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
+        notifier.post(text: message.capitalize, attachments: attachement.to_a)
       end
 
       message = message.colorize(:light_red) if kits.any?
       puts "#{url}: #{message}"
     end
 
-    def post_to_slack?
-      slack_webhook_url? && Miteru.configuration.post_to_slack?
-    end
-
-    def slack_webhook_url
-      ENV.fetch "SLACK_WEBHOOK_URL"
-    end
-
-    def slack_channel
-      ENV.fetch "SLACK_CHANNEL", "#general"
-    end
-
-    def slack_webhook_url?
-      ENV.key? "SLACK_WEBHOOK_URL"
+    def notifiable?
+      Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
     end
   end
 end