miteru 1.2.1 → 2.0.0

data/.standard.yml

@@ -1,4 +0,0 @@
-ignore:
-  - "**/*":
-      - Layout/SpaceInsideHashLiteralBraces
-      - Style/RescueStandardError

data/lib/miteru/attachement.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-require "uri"
-
-module Miteru
-  class Attachement
-    attr_reader :url
-
-    def initialize(url)
-      @url = url
-    end
-
-    def to_a
-      [
-        {
-          text: defanged_url,
-          fallback: "VT & urlscan.io links",
-          actions: actions
-        }
-      ]
-    end
-
-    private
-
-    def actions
-      [vt_link, urlscan_link].compact
-    end
-
-    def vt_link
-      return nil unless _vt_link
-
-      {
-        type: "button",
-        text: "Lookup on VirusTotal",
-        url: _vt_link
-      }
-    end
-
-    def urlscan_link
-      return nil unless _urlscan_link
-
-      {
-        type: "button",
-        text: "Lookup on urlscan.io",
-        url: _urlscan_link
-      }
-    end
-
-    def defanged_url
-      @defanged_url ||= url.to_s.gsub(/\./, "[.]")
-    end
-
-    def domain
-      @domain ||=
-        [].tap do |out|
-          out << URI(url).hostname
-        rescue URI::Error => _e
-          out << nil
-        end.first
-    end
-
-    def _urlscan_link
-      return nil unless domain
-
-      "https://urlscan.io/domain/#{domain}"
-    end
-
-    def _vt_link
-      return nil unless domain
-
-      "https://www.virustotal.com/#/domain/#{domain}"
-    end
-  end
-end

data/lib/miteru/cli.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-require "thor"
-
-module Miteru
-  class CLI < Thor
-    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
-    method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable Ayashige(ninoseki/ayashige) feed"
-    method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
-    method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
-    method_option :post_to_slack, type: :boolean, default: false, desc: "Enable or disable Slack notification"
-    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results to fetch. (Max: 10,000)"
-    method_option :threads, type: :numeric, desc: "Number of threads to use"
-    method_option :verbose, type: :boolean, default: true
-    desc "execute", "Execute the crawler"
-    def execute
-      Miteru.configure do |config|
-        config.auto_download = options["auto_download"]
-        config.ayashige = options["ayashige"]
-        config.directory_traveling = options["directory_traveling"]
-        config.download_to = options["download_to"]
-        config.post_to_slack = options["post_to_slack"]
-        config.size = options["size"]
-        config.verbose = options["verbose"]
-
-        threads = options["threads"]
-        config.threads = threads if threads
-      end
-
-      Crawler.execute
-    end
-
-    default_command :execute
-
-    class << self
-      def exit_on_failure?
-        true
-      end
-    end
-  end
-end

data/lib/miteru/configuration.rb

@@ -1,122 +0,0 @@
-# frozen_string_literal: true
-
-require "parallel"
-
-module Miteru
-  class Configuration
-    # @return [Boolean]
-    attr_accessor :auto_download
-
-    # @return [Boolean]
-    attr_accessor :ayashige
-
-    # @return [Boolean]
-    attr_accessor :directory_traveling
-
-    # @return [String]
-    attr_accessor :download_to
-
-    # @return [Boolean]
-    attr_accessor :post_to_slack
-
-    # @return [Integer]
-    attr_accessor :size
-
-    # @return [Integer]
-    attr_accessor :threads
-
-    # @return [Boolean]
-    attr_accessor :verbose
-
-    # @return [String]
-    attr_accessor :database
-
-    # @return [String, nil]
-    attr_accessor :slack_webhook_url
-
-    # @return [String]
-    attr_accessor :slack_channel
-
-    # @return [String]
-    attr_accessor :urlscan_api_key
-
-    # @return [String]
-    attr_accessor :urlscan_submit_visibility
-
-    # @return [Array<String>]
-    attr_reader :valid_extensions
-
-    # @return [Array<String>]
-    attr_reader :valid_mime_types
-
-    # @return [Integer]
-    attr_reader :file_maxsize
-
-    def initialize
-      @auto_download = false
-      @ayashige = false
-      @directory_traveling = false
-      @download_to = "/tmp"
-      @post_to_slack = false
-      @size = 100
-      @threads = Parallel.processor_count
-      @verbose = false
-      @database = ENV.fetch("MITERU_DATABASE", "miteru.db")
-      @file_maxsize = ENV.fetch("FILE_MAXSIZE", 1024 * 1024 * 100).to_i
-
-      @slack_webhook_url = ENV.fetch("SLACK_WEBHOOK_URL", nil)
-      @slack_channel = ENV.fetch("SLACK_CHANNEL", "#general")
-
-      @urlscan_api_key = ENV.fetch("URLSCAN_API_KEY", nil)
-      @urlscan_submit_visibility = ENV.fetch("URLSCAN_SUBMIT_VISIBILITY", "public")
-
-      @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
-      @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
-    end
-
-    def auto_download?
-      @auto_download
-    end
-
-    def ayashige?
-      @ayashige
-    end
-
-    def directory_traveling?
-      @directory_traveling
-    end
-
-    def post_to_slack?
-      @post_to_slack
-    end
-
-    def verbose?
-      @verbose
-    end
-
-    def slack_webhook_url?
-      !@slack_webhook_url.nil?
-    end
-
-    def urlscan_api_key?
-      !@urlscan_api_key.nil?
-    end
-  end
-
-  class << self
-    # @return [Miteru::Configuration] Miteru's current configuration
-    def configuration
-      @configuration ||= Configuration.new
-    end
-
-    # Set Miteru's configuration
-    # @param config [Miteru::Configuration]
-    attr_writer :configuration
-
-    # Modify Miteru's current configuration
-    # @yieldparam [Miteru::Configuration] config current Miteru config
-    def configure
-      yield configuration
-    end
-  end
-end

data/lib/miteru/error.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Miteru
-  class HTTPResponseError < StandardError; end
-
-  class DownloadError < StandardError; end
-end

data/lib/miteru/feeds/feed.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Miteru
-  class Feeds
-    class Feed
-      include Mixins::URL
-
-      def source
-        @source ||= self.class.to_s.split("::").last
-      end
-
-      #
-      # Return URLs
-      #
-      # @return [Array<String>] URLs
-      #
-      def urls
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-      end
-
-      #
-      # Return entries
-      #
-      # @return [Array<Miteru::Entry>]
-      #
-      def entries
-        breakdowend_urls.map do |url|
-          Entry.new(url, source)
-        end
-      end
-
-      #
-      # Return breakdowned URLs
-      #
-      # @return [Array<String>] Breakdowned URLs
-      #
-      def breakdowend_urls
-        urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
-          breakdown(url, Miteru.configuration.directory_traveling?)
-        end.flatten.uniq
-      end
-
-      private
-
-      def get(url)
-        res = HTTPClient.get(url)
-        raise HTTPResponseError if res.code != 200
-
-        res.body.to_s
-      end
-    end
-  end
-end

data/lib/miteru/feeds/phishstats.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "uri"
-
-module Miteru
-  class Feeds
-    class PhishStats < Feed
-      URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
-
-      def urls
-        json = JSON.parse(get(URL))
-        json.map do |entry|
-          entry["url"]
-        end
-      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
-        Miteru.logger.error "Failed to load PhishStats feed (#{e})"
-        []
-      end
-
-      private
-
-      def url_for(path)
-        URI(URL + path)
-      end
-    end
-  end
-end

data/lib/miteru/feeds.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "./feeds/feed"
-require_relative "./feeds/phishing_database"
-require_relative "./feeds/phishstats"
-require_relative "./feeds/ayashige"
-require_relative "./feeds/urlscan"
-require_relative "./feeds/urlscan_pro"
-
-module Miteru
-  class Entry
-    # @return [String]
-    attr_reader :url
-    # @return [String]
-    attr_reader :source
-
-    def initialize(url, source)
-      @url = url
-      @source = source
-    end
-  end
-
-  class Feeds
-    IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
-
-    def initialize
-      @feeds = [
-        PhishingDatabase.new,
-        PhishStats.new,
-        UrlScan.new(Miteru.configuration.size),
-        UrlScanPro.new,
-        Miteru.configuration.ayashige? ? Ayashige.new : nil
-      ].compact
-    end
-
-    #
-    # Returns a list of suspicious entries
-    #
-    # @return [Array<Entry>]
-    #
-    def suspicious_entries
-      @suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
-    end
-  end
-end

data/lib/miteru/http_client.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-require "down/http"
-require "http"
-require "uri"
-
-module Miteru
-  class HTTPClient
-    DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
-    URLSCAN_UA = "miteru/#{Miteru::VERSION}"
-
-    attr_reader :ssl_context
-
-    def initialize
-      ctx = OpenSSL::SSL::SSLContext.new
-      ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      @ssl_context = ctx
-    end
-
-    def download(url, destination)
-      down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
-      down.download(url, destination: destination)
-      destination
-    end
-
-    def head(url, options = {})
-      options = options.merge default_options
-
-      HTTP.follow
-        .timeout(3)
-        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-        .head(url, options)
-    end
-
-    def get(url, options = {})
-      options = options.merge default_options
-
-      HTTP.follow
-        .timeout(write: 2, connect: 5, read: 10)
-        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-        .get(url, options)
-    end
-
-    def post(url, options = {})
-      HTTP.post url, options
-    end
-
-    class << self
-      def download(url, base_dir = "/tmp")
-        new.download(url, base_dir)
-      end
-
-      def get(url, options = {})
-        new.get url, options
-      end
-
-      def post(url, options = {})
-        new.post url, options
-      end
-
-      def head(url, options = {})
-        new.head url, options
-      end
-    end
-
-    private
-
-    def default_headers
-      { user_agent: DEFAULT_UA }
-    end
-
-    def default_options
-      { ssl_context: ssl_context }
-    end
-
-    def urlscan_headers
-      { user_agent: URLSCAN_UA }
-    end
-
-    def urlscan_url?(url)
-      uri = URI(url)
-      uri.hostname == "urlscan.io"
-    end
-  end
-end