miteru 1.2.2 → 2.0.1

data/lib/miteru/kit.rb

@@ -1,14 +1,7 @@
 # frozen_string_literal: true
 
-require "cgi"
-require "uuidtools"
-require "uri"
-
 module Miteru
-  class Kit
-    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
-    VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
-
+  class Kit < Service
     # @return [String]
     attr_reader :url
 
@@ -27,7 +20,11 @@ module Miteru
     # @return [Hash, nil]
     attr_reader :headers
 
-    def initialize(url, source)
+    #
+    # @param [String] url
+    # @param [String] source
+    #
+    def initialize(url, source:)
       @url = url
       @source = source
 
@@ -91,6 +88,13 @@ module Miteru
       @decoded_url ||= URI.decode_www_form_component(url)
     end
 
+    #
+    # @return [String]
+    #
+    def truncated_url
+      url.truncate(64)
+    end
+
     private
 
     def filename_to_download
@@ -98,21 +102,25 @@ module Miteru
     end
 
     def base_dir
-      @base_dir ||= Miteru.configuration.download_to
+      @base_dir ||= Miteru.config.download_to
     end
 
     def valid_ext?
-      VALID_EXTENSIONS.include? extname
+      Miteru.config.file_extensions.include? extname
+    end
+
+    def http
+      HTTP::Factory.build
     end
 
     def before_validation
-      res = HTTPClient.head(url)
-      @content_length = res.content_length
-      @mime_type = res.content_type.mime_type.to_s
-      @status = res.status
-      @headers = res.headers.to_h
-    rescue StandardError
-      # do nothing
+      Try[StandardError] do
+        res = http.head(url)
+        @content_length = res.content_length
+        @mime_type = res.content_type.mime_type.to_s
+        @status = res.status
+        @headers = res.headers.to_h
+      end.recover { nil }.value!
     end
 
     def reachable?
@@ -120,11 +128,11 @@ module Miteru
     end
 
     def valid_mime_type?
-      VALID_MIME_TYPES.include? mime_type
+      Miteru.config.file_mime_types.include? mime_type
     end
 
     def valid_content_length?
-      content_length.to_i > 0
+      content_length.to_i.positive?
     end
   end
 end

data/lib/miteru/mixin.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Miteru
   module Mixins
     module URL
@@ -13,35 +15,6 @@ module Miteru
       def invalid_extension?(url)
         IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
       end
-
-      #
-      # Breakdown a URL into URLs
-      #
-      # @param [String] url
-      # @param [Boolean] enable_directory_traveling
-      #
-      # @return [Array<String>]
-      #
-      def breakdown(url, enable_directory_traveling)
-        begin
-          uri = URI.parse(url)
-        rescue URI::InvalidURIError => _e
-          return []
-        end
-
-        base = "#{uri.scheme}://#{uri.hostname}"
-        return [base] unless enable_directory_traveling
-
-        segments = uri.path.split("/")
-        return [base] if segments.length.zero?
-
-        urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
-
-        urls.reject do |breakdowned_url|
-          # Reject a url which ends with specific extension names
-          invalid_extension? breakdowned_url
-        end
-      end
     end
   end
 end

data/lib/miteru/notifiers/base.rb

@@ -2,14 +2,21 @@
 
 module Miteru
   module Notifiers
-    class Base
-      def notify(website)
+    class Base < Service
+      def call(website)
         raise NotImplementedError
       end
 
-      def notifiable?
+      def callable?
         raise NotImplementedError
       end
+
+      class << self
+        def inherited(child)
+          super
+          Miteru.notifiers << child
+        end
+      end
     end
   end
 end

data/lib/miteru/notifiers/slack.rb

@@ -1,32 +1,107 @@
 # frozen_string_literal: true
 
-require "colorize"
 require "slack-notifier"
 
 module Miteru
   module Notifiers
+    class SlackAttachment
+      attr_reader :url
+
+      def initialize(url)
+        @url = url
+      end
+
+      def to_a
+        [
+          {
+            text: defanged_url,
+            fallback: "VT & urlscan.io links",
+            actions:
+          }
+        ]
+      end
+
+      private
+
+      def actions
+        [vt_link, urlscan_link].compact
+      end
+
+      def vt_link
+        return nil unless _vt_link
+
+        {
+          type: "button",
+          text: "Lookup on VirusTotal",
+          url: _vt_link
+        }
+      end
+
+      def urlscan_link
+        return nil unless _urlscan_link
+
+        {
+          type: "button",
+          text: "Lookup on urlscan.io",
+          url: _urlscan_link
+        }
+      end
+
+      def defanged_url
+        @defanged_url ||= url.to_s.gsub(".", "[.]")
+      end
+
+      def domain
+        @domain ||= [].tap do |out|
+          out << URI(url).hostname
+        rescue URI::Error => _e
+          out << nil
+        end.first
+      end
+
+      def _urlscan_link
+        return nil unless domain
+
+        "https://urlscan.io/domain/#{domain}"
+      end
+
+      def _vt_link
+        return nil unless domain
+
+        "https://www.virustotal.com/#/domain/#{domain}"
+      end
+    end
+
     class Slack < Base
       #
       # Notifiy to Slack
       #
-      # @param [Miteru::Website website
+      # @param [Miteru::Website] website
       #
-      def notify(website)
-        attachement = Attachement.new(website.url)
+      def call(website)
+        return unless callable?
+
+        attachment = SlackAttachment.new(website.url)
         kits = website.kits.select(&:downloaded?)
+        notifier.post(text: website.message.capitalize, attachments: attachment.to_a) if kits.any?
+      end
+
+      def callable?
+        !Miteru.config.slack_webhook_url.nil?
+      end
 
-        notifier.post(text: website.message.capitalize, attachments: attachement.to_a) if notifiable? && kits.any?
+      private
 
-        message = kits.any? ? website.message.colorize(:light_red) : website.message
-        Miteru.logger.info "#{website.url}: #{message}"
+      def webhook_url
+        Miteru.config.slack_webhook_url
       end
 
-      def notifiable?
-        Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
+      def channel
+        Miteru.config.slack_channel
       end
 
       def notifier
-        ::Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
+        ::Slack::Notifier.new(webhook_url, channel:)
       end
     end
   end

data/lib/miteru/notifiers/urlscan.rb

@@ -1,36 +1,51 @@
 # frozen_string_literal: true
 
-require "urlscan"
-
 module Miteru
   module Notifiers
     class UrlScan < Base
       #
-      # Notifiy to urlscan.io
-      #
-      # @param [Miteru::Website website
+      # @param [Miteru::Website] website
       #
-      def notify(website)
+      def call(website)
+        return unless callable?
+
         kits = website.kits.select(&:downloaded?)
-        return unless notifiable? && kits.any?
+        return unless kits.any?
 
         kits.each { |kit| submit(kit.url) }
       end
 
-      def notifiable?
-        Miteru.configuration.urlscan_api_key?
+      def callable?
+        !Miteru.config.urlscan_api_key.nil?
       end
 
       private
 
-      def api
-        @api ||= ::UrlScan::API.new(Miteru.configuration.urlscan_api_key)
+      #
+      # @return [::HTTP::Client]
+      #
+      def http
+        @http ||= HTTP::Factory.build(headers:, timeout:)
+      end
+
+      def headers
+        {"api-key": Miteru.config.urlscan_api_key}
+      end
+
+      def timeout
+        Miteru.config.timeout
+      end
+
+      def tags
+        %w[miteru phishkit]
+      end
+
+      def visibility
+        Miteru.config.urlscan_submit_visibility
       end
 
       def submit(url)
-        api.submit(url, tags: ["miteru", "phishkit"], visibility: Miteru.configuration.urlscan_submit_visibility)
-      rescue StandardError
-        # do nothing
+        http.post("/api/v1/scan/", json: {tags:, visibility:, url:})
       end
     end
   end

data/lib/miteru/orchestrator.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Miteru
+  class Orchestrator < Service
+    def call
+      Miteru.logger.info("#{websites.length} websites loaded in total.") if verbose?
+
+      if Miteru.sidekiq?
+
+        websites.each do |website|
+          Jobs::CrawleJob.perform_async(website.url, website.source)
+          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued") if verbose?
+        end
+      else
+        Miteru.logger.info("Use #{threads} thread(s).") if verbose?
+        Parallel.each(websites, in_threads: threads) do |website|
+          Miteru.logger.info("Website:#{website.truncated_url} crawling started") if verbose?
+          crawl(website)
+        end
+      end
+    end
+
+    #
+    # @return [Array<Miteru::Websites>]
+    #
+    def websites
+      @websites ||= [].tap do |out|
+        feeds.each do |feed|
+          result = feed.result
+          if result.success?
+            websites = result.value!
+            Miteru.logger.info("Feed:#{feed.source} has #{websites.length} websites.") if verbose?
+            out << websites
+          else
+            Miteru.logger.warn("Feed:#{feed.source} failed - #{result.failure}")
+          end
+        end
+      end.flatten
+    end
+
+    private
+
+    def threads
+      Miteru.config.threads
+    end
+
+    def verbose?
+      Miteru.config.verbose
+    end
+
+    #
+    # @return [Array<Miteru::Feeds::Base>]
+    #
+    def feeds
+      Miteru.feeds.map(&:new)
+    end
+  end
+end

data/lib/miteru/record.rb

@@ -1,35 +1,27 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Miteru
   class Record < ActiveRecord::Base
     class << self
       #
-      # Check uniqueness of a record by a hash
-      #
-      # @param [String] hash
+      # @param [String] sha256
       #
       # @return [Boolean] true if it is unique. Otherwise false.
       #
-      def unique_hash?(hash)
-        record = find_by(hash: hash)
-        return true if record.nil?
-
-        false
+      def unique_sha256?(sha256)
+        !where(sha256:).exists?
       end
 
       #
       # Create a new record based on a kit
       #
       # @param [Miteru::Kit] kit
-      # @param [String] hash
+      # @param [String] sha256
       #
       # @return [Miteru::Record]
       #
-      def create_by_kit_and_hash(kit, hash)
+      def create_by_kit_and_hash(kit, sha256:)
         record = new(
-          hash: hash,
           source: kit.source,
           hostname: kit.hostname,
           url: kit.decoded_url,
@@ -37,11 +29,12 @@ module Miteru
           filename: kit.filename,
           filesize: kit.filesize,
           mime_type: kit.mime_type,
-          downloaded_as: kit.filepath_to_download
+          downloaded_as: kit.filepath_to_download,
+          sha256:
         )
         record.save
         record
-      rescue TypeError, ActiveRecord::RecordNotUnique => _e
+      rescue TypeError, ActiveRecord::RecordNotUnique
         nil
       end
     end

data/lib/miteru/service.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Miteru
+  #
+  # Base class for services
+  #
+  class Service
+    include Dry::Monads[:result, :try]
+
+    def call(*args, **kwargs)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def result(...)
+      Try[StandardError] { call(...) }.to_result
+    end
+
+    class << self
+      def call(...)
+        new.call(...)
+      end
+
+      def result(...)
+        new.result(...)
+      end
+    end
+  end
+end

data/lib/miteru/sidekiq/application.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+require "miteru/sidekiq/jobs"
+
+Sidekiq.configure_server do |config|
+  config.redis = {url: Miteru.config.sidekiq_redis_url.to_s}
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = {url: Miteru.config.sidekiq_redis_url.to_s}
+end

data/lib/miteru/sidekiq/jobs.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+module Miteru
+  module Jobs
+    class CrawleJob
+      include Sidekiq::Job
+      include Concerns::DatabaseConnectable
+
+      #
+      # @param [String] url
+      # @param [String] source
+      #
+      def perform(url, source)
+        website = Miteru::Website.new(url, source:)
+        with_db_connection { Crawler.call(website) }
+      end
+    end
+  end
+end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "1.2.2"
+  VERSION = "2.0.1"
 end

data/lib/miteru/web/application.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Rack
+require "rack"
+require "rack/session"
+require "rackup"
+
+require "rack/handler/puma"
+
+# Sidekiq
+require "sidekiq/web"
+
+module Miteru
+  module Web
+    class App
+      class << self
+        def instance
+          Rack::Builder.new do
+            use Rack::Session::Cookie, secret: SecureRandom.hex(32), same_site: true, max_age: 86_400
+
+            map "/" do
+              run Sidekiq::Web
+            end
+
+            run App.new
+          end.to_app
+        end
+
+        def run!(port: 9292, host: "localhost", threads: "0:3", verbose: false, worker_timeout: 60, open: true)
+          Rackup::Handler::Puma.run(
+            instance,
+            Port: port,
+            Host: host,
+            Threads: threads,
+            Verbose: verbose,
+            worker_timeout:
+          )
+        end
+      end
+    end
+  end
+end