miteru 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7428f741cd791e37e5f201c978f4ee6b5f6fe239d350b69d3377a6af4604305d
-  data.tar.gz: 0057c9ee7ebe6dd5ee039939b30ff897d3298605c3843c980f2b30081ad8bc2d
+  metadata.gz: 62e5976eed670c3d2f2b0309e7723434c1e5c04ad337d4db519a58362618d566
+  data.tar.gz: fe54d9316505b380ac50ba9ab8cf520f2fd5b0e76cf88f0e520d7fb0c18a3b56
 SHA512:
-  metadata.gz: b4f69cf18c8db256f035466f669835810e53329f7af7df5fd3d58689002c2e7175abb9456b31e21c0b23369e31a1559040c82f92c662202661c50cc8cc702334
-  data.tar.gz: d7188fe6bc2372560031fb964254f5ea6df56672edd0f112a6ebe7b41d5ad7efa2a52e6c1deb666891573990f0cf75e8b4ea768f5240cd5d25127a544e06e26c
+  metadata.gz: 122f011baecb0a38639610ffaf68645c48f454ef7aa6ba79d5e812f71129f303a4386308cf639e233ff43473907f9567f90625b3651e027d18d6324e694ba00e
+  data.tar.gz: 0f21e5a5b6882fb9705d9988647de16aad44693ddfbbd9983a01f16c029923629322c40165a28dcb33e4a49482bcddbc1d272825beae7b0017400c560e16a57d

data/lib/miteru/cache.rb

@@ -0,0 +1,53 @@
+require "redis"
+
+module Miteru
+  class Cache < Service
+    # @return [String]
+    attr_reader :url
+
+    #
+    # @param [String] url
+    #
+    def initialize(url)
+      super()
+      @url = url
+    end
+
+    #
+    # @param [String] key
+    # @param [String] value
+    # @param [Integer. nil] ex
+    #
+    def set(key, value, ex:)
+      value = redis.set("#{prefix}:#{key}", value, ex:)
+      Miteru.logger.info("Cache:#{key} is set.") if verbose?
+      value
+    end
+
+    #
+    # @param [String] key
+    #
+    def cached?(key)
+      value = redis.exists?("#{prefix}:#{key}")
+      Miteru.logger.info("Cache:#{key} found.") if verbose?
+      value
+    end
+
+    private
+
+    def verbose?
+      Miteru.config.verbose
+    end
+
+    def prefix
+      Miteru.config.cache_prefix
+    end
+
+    #
+    # @return [Redis]
+    #
+    def redis
+      @redis ||= Redis.new(url:)
+    end
+  end
+end

data/lib/miteru/config.rb

@@ -22,6 +22,9 @@ module Miteru
       sentry_dsn: nil,
       sentry_trace_sample_rate: 0.25,
       sidekiq_redis_url: nil,
+      cache_redis_url: nil,
+      cache_ex: nil,
+      cache_prefix: "miteru:cache",
       slack_channel: "#general",
       slack_webhook_url: nil,
       threads: Parallel.processor_count,
@@ -39,6 +42,15 @@ module Miteru
     #   @return [Float]
 
     # @!attribute [r] sidekiq_redis_url
+    #   @return [String, nil]
+
+    # @!attribute [r] cache_redis_url
+    #   @return [String, nil]
+
+    # @!attribute [r] cache_ex
+    #   @return [Integer, nil]
+
+    # @!attribute [r] cache_prefix
     #   @return [String]
 
     # @!attribute [r] http_timeout

data/lib/miteru/crawler.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "colorize"
+
 module Miteru
   class Crawler < Service
     #
@@ -7,27 +9,46 @@ module Miteru
     #
     def call(website)
       Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
-        Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
-        return unless website.kits?
-
-        notify website
-
-        return unless auto_download?
+        info = "Website:#{website.info}."
+        info = info.colorize(:red) if website.kits?
+        Miteru.logger.info(info)
 
         website.kits.each do |kit|
           downloader = Downloader.new(kit)
           result = downloader.result
-          if result.success?
-            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}.")
-          else
+
+          unless result.success?
             Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}.")
+            next
           end
+
+          destination = result.value!
+          Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{destination}.")
+          # Remove downloaded file if auto_download is not allowed
+          FileUtils.rm(destination, force: true) unless auto_download?
+          # Notify the website
+          notify website
         end
+
+        # Cache the website
+        cache.set(website.url, website.source, ex: cache_ex) if cache?
       end.recover { nil }.value!
     end
 
     private
 
+    def cache?
+      Miteru.cache?
+    end
+
+    def cache
+      Miteru.cache
+    end
+
+    def cache_ex
+      Miteru.config.cache_ex
+    end
+
     def auto_download?
       Miteru.config.auto_download
     end

data/lib/miteru/notifiers/slack.rb

@@ -5,6 +5,7 @@ require "slack-notifier"
 module Miteru
   module Notifiers
     class SlackAttachment
+      # @return [String]
       attr_reader :url
 
       def initialize(url)
@@ -14,7 +15,7 @@ module Miteru
       def to_a
         [
           {
-            text: defanged_url,
+            text:,
             fallback: "VT & urlscan.io links",
             actions:
           }
@@ -47,10 +48,6 @@ module Miteru
         }
       end
 
-      def defanged_url
-        @defanged_url ||= url.to_s.gsub(".", "[.]")
-      end
-
       def domain
         @domain ||= [].tap do |out|
           out << URI(url).hostname
@@ -59,6 +56,10 @@ module Miteru
         end.first
       end
 
+      def text
+        domain.to_s.gsub(".", "[.]")
+      end
+
       def _urlscan_link
         return nil unless domain
 
@@ -82,12 +83,11 @@ module Miteru
         return unless callable?
 
         attachment = SlackAttachment.new(website.url)
-        kits = website.kits.select(&:downloaded?)
-        notifier.post(text: website.message.capitalize, attachments: attachment.to_a) if kits.any?
+        notifier.post(text: website.info, attachments: attachment.to_a) if website.kits?
       end
 
       def callable?
-        !slack_webhook_url.nil?
+        !webhook_url.nil?
       end
 
       private

data/lib/miteru/notifiers/urlscan.rb

@@ -9,7 +9,7 @@ module Miteru
       def call(website)
         return unless callable?
 
-        website.kits.each { |kit| submit(kit.url) }
+        website.kits.each { |kit| submit(kit.url, source: website.source) }
       end
 
       def callable?
@@ -41,8 +41,12 @@ module Miteru
         Miteru.config.urlscan_submit_visibility
       end
 
-      def submit(url)
-        http.post("https://urlscan.io/api/v1/scan/", json: {tags:, visibility:, url:})
+      #
+      # @param [String] url
+      # @param [String] source
+      #
+      def submit(url, source:)
+        http.post("https://urlscan.io/api/v1/scan/", json: {tags: tags + ["source:#{source}"], visibility:, url:})
       end
     end
   end

data/lib/miteru/orchestrator.rb

@@ -3,16 +3,16 @@
 module Miteru
   class Orchestrator < Service
     def call
-      Miteru.logger.info("#{websites.length} websites loaded in total.") if verbose?
+      Miteru.logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?
 
       if Miteru.sidekiq?
-        websites.each do |website|
+        non_cached_websites.each do |website|
           Jobs::CrawleJob.perform_async(website.url, website.source)
           Miteru.logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
         end
       else
         Miteru.logger.info("Use #{threads} thread(s).") if verbose?
-        Parallel.each(websites, in_threads: threads) do |website|
+        Parallel.each(non_cached_websites, in_threads: threads) do |website|
           Miteru.logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
 
           result = Crawler.result(website)
@@ -26,7 +26,7 @@ module Miteru
     end
 
     #
-    # @return [Array<Miteru::Websites>]
+    # @return [Array<Miteru::Website>]
     #
     def websites
       @websites ||= [].tap do |out|
@@ -43,6 +43,12 @@ module Miteru
       end.flatten.uniq(&:url)
     end
 
+    def non_cached_websites
+      return websites unless cache?
+
+      websites.reject { |website| cache.cached?(website.url) }
+    end
+
     #
     # @return [Array<Miteru::Feeds::Base>]
     #
@@ -52,6 +58,14 @@ module Miteru
 
     private
 
+    def cache?
+      Miteru.cache?
+    end
+
+    def cache
+      Miteru.cache
+    end
+
     def threads
       Miteru.config.threads
     end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/lib/miteru/website.rb

@@ -48,6 +48,14 @@ module Miteru
       url.truncate(64)
     end
 
+    def defanged_truncated_url
+      truncated_url.to_s.gsub(".", "[.]")
+    end
+
+    def info
+      "#{defanged_truncated_url} has #{kits.length} kit(s) (Source: #{source})"
+    end
+
     private
 
     def timeout

data/lib/miteru.rb

@@ -33,6 +33,9 @@ require "miteru/concerns/database_connectable"
 require "miteru/concerns/error_unwrappable"
 
 # Core classes
+require "miteru/service"
+
+require "miteru/cache"
 require "miteru/config"
 require "miteru/http"
 
@@ -93,6 +96,14 @@ module Miteru
       env == "development"
     end
 
+    def cache?
+      !Miteru.config.cache_redis_url.nil?
+    end
+
+    def cache
+      @cache ||= Cache.new(Miteru.config.cache_redis_url)
+    end
+
     #
     # @return [Boolean]
     #
@@ -117,8 +128,6 @@ module Miteru
 end
 
 # Services
-require "miteru/service"
-
 require "miteru/crawler"
 require "miteru/downloader"
 require "miteru/kit"

data/miteru.gemspec

@@ -1,67 +1,68 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('lib', __dir__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'miteru/version'
+require "miteru/version"
 
 Gem::Specification.new do |spec|
-  spec.name = 'miteru'
+  spec.name = "miteru"
   spec.version = Miteru::VERSION
-  spec.authors = ['Manabu Niseki']
-  spec.email = ['manabu.niseki@gmail.com']
-  spec.metadata['rubygems_mfa_required'] = 'true'
+  spec.authors = ["Manabu Niseki"]
+  spec.email = ["manabu.niseki@gmail.com"]
+  spec.metadata["rubygems_mfa_required"] = "true"
 
-  spec.summary = 'A phishing kit collector for scavengers'
-  spec.description = 'A phishing kit collector for scavengers'
-  spec.homepage = 'https://github.com/ninoseki/miteru'
-  spec.license = 'MIT'
+  spec.summary = "A phishing kit collector for scavengers"
+  spec.description = "A phishing kit collector for scavengers"
+  spec.homepage = "https://github.com/ninoseki/miteru"
+  spec.license = "MIT"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir = 'exe'
+  spec.bindir = "exe"
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ['lib']
+  spec.require_paths = ["lib"]
 
-  spec.add_development_dependency 'bundler', '~> 2.5'
-  spec.add_development_dependency 'capybara', '~> 3.40'
-  spec.add_development_dependency 'coveralls_reborn', '~> 0.28'
-  spec.add_development_dependency 'fuubar', '~> 2.5'
-  spec.add_development_dependency 'mysql2', '~> 0.5'
-  spec.add_development_dependency 'pg', '~> 1.5'
-  spec.add_development_dependency 'rake', '~> 13.1'
-  spec.add_development_dependency 'rspec', '~> 3.13'
-  spec.add_development_dependency 'simplecov-lcov', '~> 0.8'
-  spec.add_development_dependency 'standard', '~> 1.33'
-  spec.add_development_dependency 'test-prof', '~> 1.3'
-  spec.add_development_dependency 'vcr', '~> 6.2'
-  spec.add_development_dependency 'webmock', '~> 3.19'
+  spec.add_development_dependency "bundler", "~> 2.5"
+  spec.add_development_dependency "capybara", "~> 3.40"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.28"
+  spec.add_development_dependency "fuubar", "~> 2.5"
+  spec.add_development_dependency "mysql2", "~> 0.5"
+  spec.add_development_dependency "pg", "~> 1.5"
+  spec.add_development_dependency "rake", "~> 13.1"
+  spec.add_development_dependency "rspec", "~> 3.13"
+  spec.add_development_dependency "simplecov-lcov", "~> 0.8"
+  spec.add_development_dependency "standard", "~> 1.33"
+  spec.add_development_dependency "test-prof", "~> 1.3"
+  spec.add_development_dependency "vcr", "~> 6.2"
+  spec.add_development_dependency "webmock", "~> 3.19"
 
-  spec.add_dependency 'activerecord', '7.1.3'
-  spec.add_dependency 'addressable', '2.8.6'
-  spec.add_dependency 'anyway_config', '2.6.2'
-  spec.add_dependency 'colorize', '1.1.0'
-  spec.add_dependency 'dotenv', '2.8.1'
-  spec.add_dependency 'down', '5.4.1'
-  spec.add_dependency 'dry-files', '1.1.0'
-  spec.add_dependency 'dry-monads', '1.6.0'
-  spec.add_dependency 'http', '5.2.0'
-  spec.add_dependency 'memo_wise', '1.8.0'
-  spec.add_dependency 'oga', '3.4'
-  spec.add_dependency 'parallel', '1.24.0'
-  spec.add_dependency 'puma', '6.4.2'
-  spec.add_dependency 'rack', '3.0.9'
-  spec.add_dependency 'rack-session', '2.0.0'
-  spec.add_dependency 'rackup', '2.1.0'
-  spec.add_dependency 'semantic_logger', '4.15.0'
-  spec.add_dependency 'sentry-ruby', '5.16.1'
-  spec.add_dependency 'sentry-sidekiq', '5.16.1'
-  spec.add_dependency 'sidekiq', '7.2.1'
-  spec.add_dependency 'slack-notifier', '2.4.0'
-  spec.add_dependency 'sqlite3', '1.7.2'
-  spec.add_dependency 'thor', '1.3.0'
-  spec.add_dependency 'thor-hollaback', '0.2.1'
-  spec.add_dependency 'uuidtools', '2.2.0'
+  spec.add_dependency "activerecord", "7.1.3"
+  spec.add_dependency "addressable", "2.8.6"
+  spec.add_dependency "anyway_config", "2.6.3"
+  spec.add_dependency "colorize", "1.1.0"
+  spec.add_dependency "dotenv", "2.8.1"
+  spec.add_dependency "down", "5.4.1"
+  spec.add_dependency "dry-files", "1.1.0"
+  spec.add_dependency "dry-monads", "1.6.0"
+  spec.add_dependency "http", "5.2.0"
+  spec.add_dependency "memo_wise", "1.8.0"
+  spec.add_dependency "oga", "3.4"
+  spec.add_dependency "parallel", "1.24.0"
+  spec.add_dependency "puma", "6.4.2"
+  spec.add_dependency "rack", "3.0.9"
+  spec.add_dependency "rack-session", "2.0.0"
+  spec.add_dependency "rackup", "2.1.0"
+  spec.add_dependency "redis", "5.0.8"
+  spec.add_dependency "semantic_logger", "4.15.0"
+  spec.add_dependency "sentry-ruby", "5.16.1"
+  spec.add_dependency "sentry-sidekiq", "5.16.1"
+  spec.add_dependency "sidekiq", "7.2.1"
+  spec.add_dependency "slack-notifier", "2.4.0"
+  spec.add_dependency "sqlite3", "1.7.2"
+  spec.add_dependency "thor", "1.3.0"
+  spec.add_dependency "thor-hollaback", "0.2.1"
+  spec.add_dependency "uuidtools", "2.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-06 00:00:00.000000000 Z
+date: 2024-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -226,14 +226,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: 2.6.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: 2.6.3
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -416,6 +416,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.0.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.0.8
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -566,6 +580,7 @@ files:
 - exe/miteru
 - lefthook.yml
 - lib/miteru.rb
+- lib/miteru/cache.rb
 - lib/miteru/cli/application.rb
 - lib/miteru/cli/base.rb
 - lib/miteru/cli/database.rb