miteru 0.14.5 → 1.0.1

data/lib/miteru/error.rb

@@ -2,5 +2,6 @@
 
 module Miteru
   class HTTPResponseError < StandardError; end
+
   class DownloadError < StandardError; end
 end

data/lib/miteru/feeds/ayashige.rb

@@ -7,7 +7,7 @@ module Miteru
   class Feeds
     class Ayashige < Feed
       HOST = "ayashige.herokuapp.com"
-      URL = "https://#{HOST}"
+      URL = "https://#{HOST}".freeze
 
       def urls
         url = url_for("/feed")

data/lib/miteru/feeds/phishing_database.rb

@@ -6,7 +6,7 @@ require "uri"
 module Miteru
   class Feeds
     class PhishingDatabase < Feed
-      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
+      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt"
 
       def urls
         body = get(URL)

data/lib/miteru/feeds/phishstats.rb

@@ -11,7 +11,7 @@ module Miteru
       def urls
         json = JSON.parse(get(URL))
         json.map do |entry|
-          entry.dig("url")
+          entry["url"]
         end
       rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
         puts "Failed to load PhishStats feed (#{e})"

data/lib/miteru/feeds/urlscan_pro.rb

@@ -27,7 +27,7 @@ module Miteru
 
         res = api.pro.phishfeed
         results = res["results"] || []
-        results.map { |result| result.dig("page_url") }
+        results.map { |result| result["page_url"] }
       rescue ArgumentError => _e
         []
       end

data/lib/miteru/feeds.rb

@@ -9,7 +9,7 @@ require_relative "./feeds/urlscan_pro"
 
 module Miteru
   class Feeds
-    IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
+    IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
 
     def initialize
       @feeds = [
@@ -48,7 +48,7 @@ module Miteru
       segments = uri.path.split("/")
       return [base] if segments.length.zero?
 
-      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
 
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names

data/lib/miteru/http_client.rb

@@ -7,7 +7,7 @@ require "uri"
 module Miteru
   class HTTPClient
     DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
-    URLSCAN_UA = "miteru/#{Miteru::VERSION}"
+    URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
 
     attr_reader :ssl_context
 
@@ -27,18 +27,18 @@ module Miteru
       options = options.merge default_options
 
       HTTP.follow
-          .timeout(3)
-          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-          .head(url, options)
+        .timeout(3)
+        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+        .head(url, options)
     end
 
     def get(url, options = {})
       options = options.merge default_options
 
       HTTP.follow
-          .timeout(write: 2, connect: 5, read: 10)
-          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-          .get(url, options)
+        .timeout(write: 2, connect: 5, read: 10)
+        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+        .get(url, options)
     end
 
     def post(url, options = {})

data/lib/miteru/kit.rb

@@ -1,21 +1,30 @@
 # frozen_string_literal: true
 
 require "cgi"
-require "securerandom"
+require "uuidtools"
+require "uri"
 
 module Miteru
   class Kit
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
     VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
 
-    attr_reader :url
+    attr_reader :url, :status, :content_length, :mime_type, :headers
 
     def initialize(url)
       @url = url
+
+      @content_length = nil
+      @mime_type = nil
+      @status = nil
+      @headers = nil
     end
 
-    def valid?;
-      valid_ext? && reachable_and_valid_mime_type?
+    def valid?
+      # make a HEAD request for the validation
+      before_validation
+
+      valid_ext? && reachable? && valid_mime_type? && valid_content_length?
     end
 
     def extname
@@ -25,21 +34,21 @@ module Miteru
     end
 
     def basename
-      File.basename(url)
+      @basename ||= File.basename(url)
     end
 
     def filename
-      CGI.unescape basename
+      @filename ||= CGI.unescape(basename)
     end
 
-    def download_filepath
-      "#{base_dir}/#{download_filename}"
+    def filepath_to_download
+      "#{base_dir}/#{filename_to_download}"
     end
 
     def filesize
-      return nil unless File.exist?(download_filepath)
+      return nil unless File.exist?(filepath_to_download)
 
-      File.size download_filepath
+      File.size filepath_to_download
     end
 
     def filename_with_size
@@ -48,18 +57,22 @@ module Miteru
       "#{filename}(#{filesize / 1024}KB)"
     end
 
-    private
-
     def id
-      @id ||= SecureRandom.hex(10)
+      @id ||= UUIDTools::UUID.random_create.to_s
     end
 
     def hostname
-      URI(url).hostname
+      @hostname ||= URI(url).hostname
     end
 
-    def download_filename
-      "#{hostname}_#{filename}_#{id}#{extname}"
+    def decoded_url
+      @decoded_url ||= URI.decode_www_form_component(url)
+    end
+
+    private
+
+    def filename_to_download
+      "#{id}#{extname}"
     end
 
     def base_dir
@@ -70,20 +83,26 @@ module Miteru
       VALID_EXTENSIONS.include? extname
     end
 
-    def reachable?(response)
-      response.status.success?
+    def before_validation
+      res = HTTPClient.head(url)
+      @content_length = res.content_length
+      @mime_type = res.content_type.mime_type.to_s
+      @status = res.status
+      @headers = res.headers.to_h
+    rescue StandardError
+      # do nothing
     end
 
-    def valid_mime_type?(response)
-      mime_type = response.content_type.mime_type.to_s
+    def reachable?
+      status&.success?
+    end
+
+    def valid_mime_type?
       VALID_MIME_TYPES.include? mime_type
     end
 
-    def reachable_and_valid_mime_type?
-      res = HTTPClient.head(url)
-      reachable?(res) && valid_mime_type?(res)
-    rescue StandardError
-      false
+    def valid_content_length?
+      content_length.to_i > 0
     end
   end
 end

data/lib/miteru/notifier.rb

@@ -9,29 +9,17 @@ module Miteru
       attachement = Attachement.new(url)
       kits = kits.select(&:filesize)
 
-      if post_to_slack? && kits.any?
-        notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
-        notifier.post(text: message, attachments: attachement.to_a)
+      if notifiable? && kits.any?
+        notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
+        notifier.post(text: message.capitalize, attachments: attachement.to_a)
       end
 
       message = message.colorize(:light_red) if kits.any?
       puts "#{url}: #{message}"
     end
 
-    def post_to_slack?
-      slack_webhook_url? && Miteru.configuration.post_to_slack?
-    end
-
-    def slack_webhook_url
-      ENV.fetch "SLACK_WEBHOOK_URL"
-    end
-
-    def slack_channel
-      ENV.fetch "SLACK_CHANNEL", "#general"
-    end
-
-    def slack_webhook_url?
-      ENV.key? "SLACK_WEBHOOK_URL"
+    def notifiable?
+      Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
     end
   end
 end

data/lib/miteru/record.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Miteru
+  class Record < ActiveRecord::Base
+    class << self
+      #
+      # Check uniqueness of a record by a hash
+      #
+      # @param [String] hash
+      #
+      # @return [Boolean] true if it is unique. Otherwise false.
+      #
+      def unique_hash?(hash)
+        record = find_by(hash: hash)
+        return true if record.nil?
+
+        false
+      end
+
+      #
+      # Create a new record based on a kit
+      #
+      # @param [Miteru::Kit] kit
+      # @param [String] hash
+      #
+      # @return [Miteru::Record]
+      #
+      def create_by_kit_and_hash(kit, hash)
+        record = new(
+          hash: hash,
+          hostname: kit.hostname,
+          url: kit.decoded_url,
+          headers: kit.headers,
+          filename: kit.filename,
+          filesize: kit.filesize,
+          mime_type: kit.mime_type,
+          downloaded_as: kit.filepath_to_download
+        )
+        record.save
+        record
+      rescue TypeError, ActiveRecord::RecordNotUnique => _e
+        nil
+      end
+    end
+  end
+end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.14.5"
+  VERSION = "1.0.1"
 end

data/lib/miteru/website.rb

@@ -17,10 +17,10 @@ module Miteru
     end
 
     def kits
-      @kits ||= links.map do |link|
+      @kits ||= links.filter_map do |link|
         kit = Kit.new(link)
         kit.valid? ? kit : nil
-      end.compact
+      end
     end
 
     def ok?
@@ -42,11 +42,11 @@ module Miteru
     end
 
     def message
-      return "It doesn't contain a phishing kit." unless kits?
+      return "it doesn't contain a phishing kit." unless kits?
 
       filename_with_sizes = kits.map(&:filename_with_size).join(", ")
       noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
-      "It might contain #{noun}: #{filename_with_sizes}."
+      "it might contain #{noun}: #{filename_with_sizes}."
     end
 
     def links
@@ -75,7 +75,7 @@ module Miteru
 
     def href_links
       if doc && ok? && index?
-        doc.css("a").map { |a| a.get("href") }.compact.map do |href|
+        doc.css("a").filter_map { |a| a.get("href") }.map do |href|
           href = href.start_with?("/") ? href : "/#{href}"
           url + href
         end

data/lib/miteru.rb

@@ -3,6 +3,10 @@
 require "miteru/version"
 
 require "miteru/configuration"
+require "miteru/database"
+
+require "miteru/record"
+
 require "miteru/error"
 require "miteru/http_client"
 require "miteru/kit"
@@ -14,6 +18,4 @@ require "miteru/notifier"
 require "miteru/crawler"
 require "miteru/cli"
 
-module Miteru
-  # Your code goes here...
-end
+module Miteru; end

data/miteru.gemspec

@@ -1,43 +1,51 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('lib', __dir__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "miteru/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "miteru"
-  spec.version       = Miteru::VERSION
-  spec.authors       = ["Manabu Niseki"]
-  spec.email         = ["manabu.niseki@gmail.com"]
+  spec.name = "miteru"
+  spec.version = Miteru::VERSION
+  spec.authors = ["Manabu Niseki"]
+  spec.email = ["manabu.niseki@gmail.com"]
 
-  spec.summary       = "An experimental phishing kit detector"
-  spec.description   = "An experimental phishing kit detector"
-  spec.homepage      = "https://github.com/ninoseki/miteru"
-  spec.license       = "MIT"
+  spec.summary = "An experimental phishing kit detector"
+  spec.description = "An experimental phishing kit detector"
+  spec.homepage = "https://github.com/ninoseki/miteru"
+  spec.license = "MIT"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "bundler", "~> 2.2"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.22"
   spec.add_development_dependency "glint", "~> 0.1"
+  spec.add_development_dependency "mysql2", "~> 0.5"
+  spec.add_development_dependency "overcommit", "~> 0.58"
+  spec.add_development_dependency "pg", "~> 1.2"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "standard", "~> 1.3"
   spec.add_development_dependency "vcr", "~> 6.0"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "webmock", "~> 3.14"
+  spec.add_development_dependency "webrick", "~> 1.7.0"
 
+  spec.add_dependency "activerecord", "~> 6.1"
   spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "down", "~> 5.1"
-  spec.add_dependency "http", "~> 4.4"
-  spec.add_dependency "oga", "~> 3.2"
-  spec.add_dependency "parallel", "~> 1.19"
-  spec.add_dependency "slack-notifier", "~> 2.3"
-  spec.add_dependency "thor", "~> 1.0"
-  spec.add_dependency "urlscan", "~> 0.5"
+  spec.add_dependency "down", "~> 5.2"
+  spec.add_dependency "http", "~> 5.0"
+  spec.add_dependency "oga", "~> 3.3"
+  spec.add_dependency "parallel", "~> 1.20"
+  spec.add_dependency "slack-notifier", "~> 2.4"
+  spec.add_dependency "sqlite3", "~> 1.4"
+  spec.add_dependency "thor", "~> 1.1"
+  spec.add_dependency "urlscan", "~> 0.7"
+  spec.add_dependency "uuidtools", "~> 2.2"
 end