miteru 1.2.2 → 2.0.1

data/lib/miteru/website.rb

@@ -3,16 +3,20 @@
 require "oga"
 
 module Miteru
-  class Website
-    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
-
+  class Website < Service
     # @return [String]
     attr_reader :url
 
     # @return [String]
     attr_reader :source
 
-    def initialize(url, source)
+    #
+    # @param [String] url
+    # @param [String] source
+    #
+    def initialize(url, source:)
+      super()
+
       @url = url
       @source = source
     end
@@ -22,86 +26,82 @@ module Miteru
     end
 
     def kits
-      @kits ||= links.filter_map do |link|
-        kit = Kit.new(link, source)
-        kit.valid? ? kit : nil
-      end
-    end
-
-    def ok?
-      response.code == 200
+      @kits ||= links.map { |link| Kit.new(link, source:) }.select(&:valid?)
     end
 
     def index?
       title.to_s.start_with? "Index of"
     end
 
-    def kits?
-      !kits.empty?
-    end
-
     def has_kits?
-      kits?
-    rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
-      false
-    end
-
-    def message
-      return "it doesn't contain a phishing kit." unless kits?
-
-      filename_with_sizes = kits.map(&:filename_with_size).join(", ")
-      noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
-      "it might contain #{noun}: #{filename_with_sizes}."
+      @has_kits ||= lambda do
+        Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
+          OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
+          !kits.empty?
+        end.recover do
+          false
+        end.value!
+      end.call
     end
 
     def links
       (href_links + possible_file_links).compact.uniq
     end
 
+    #
+    # @return [String]
+    #
+    def truncated_url
+      url.truncate(64)
+    end
+
     private
 
-    def response
-      @response ||= get
+    def timeout
+      Miteru.config.http_timeout
+    end
+
+    def http
+      @http ||= HTTP::Factory.build(timeout:)
     end
 
     def get
-      HTTPClient.get url
+      http.get url
     end
 
-    def doc
-      @doc ||= parse_html(response.body.to_s)
+    def response
+      @response ||= get
     end
 
-    def parse_html(html)
-      Oga.parse_html(html)
-    rescue ArgumentError, Encoding::CompatibilityError, LL::ParserError => _e
-      nil
+    def doc
+      Oga.parse_html response.body.to_s
     end
 
     def href_links
-      if doc && ok? && index?
+      Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
+        OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
         doc.css("a").filter_map { |a| a.get("href") }.map do |href|
           href = href.start_with?("/") ? href : "/#{href}"
           url + href
         end
-      else
-        []
-      end
-    rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
-      []
+      end.recover { [] }.value!
+    end
+
+    def file_extensions
+      Miteru.config.file_extensions
     end
 
     def possible_file_links
-      uri = URI.parse(url)
+      parsed = URI.parse(url)
 
-      segments = uri.path.split("/")
-      return [] if segments.length.zero?
+      segments = parsed.path.split("/")
+      return [] if segments.empty?
 
       last = segments.last
-      VALID_EXTENSIONS.map do |ext|
+      file_extensions.map do |ext|
         new_segments = segments[0..-2] + ["#{last}#{ext}"]
-        uri.path = new_segments.join("/")
-        uri.to_s
+        parsed.path = new_segments.join("/")
+        parsed.to_s
       end
     end
   end

data/lib/miteru.rb

@@ -1,42 +1,150 @@
 # frozen_string_literal: true
 
-require "memist"
+# Core standard libraries
+require "cgi"
+require "json"
+require "uri"
+require "uuidtools"
+
+# Core 3rd party libraries
+require "colorize"
+require "memo_wise"
+require "parallel"
 require "semantic_logger"
+require "sentry-ruby"
 
-require "miteru/version"
+require "dry/files"
+require "dry/monads"
 
-require "miteru/configuration"
-require "miteru/database"
+# Load .env
+require "dotenv/load"
 
-require "miteru/record"
+# Active Support & Active Record
+require "active_support"
+require "active_record"
 
-require "miteru/mixin"
+# Version
+require "miteru/version"
+# Errors
+require "miteru/errors"
 
-require "miteru/notifiers/base"
-require "miteru/notifiers/slack"
-require "miteru/notifiers/urlscan"
+# Concerns
+require "miteru/concerns/database_connectable"
+require "miteru/concerns/error_unwrappable"
 
-require "miteru/error"
-require "miteru/http_client"
-require "miteru/kit"
-require "miteru/website"
-require "miteru/downloader"
-require "miteru/feeds"
-require "miteru/attachement"
-require "miteru/crawler"
-require "miteru/cli"
+# Core classes
+require "miteru/config"
+require "miteru/http"
 
-# Load .env
-require "dotenv/load"
+# Database + ActiveRecord
+require "miteru/database"
+require "miteru/record"
 
 module Miteru
   class << self
-    include Memist::Memoizable
+    prepend MemoWise
+
+    #
+    # @return [SematicLogger]
+    #
     def logger
+      SemanticLogger.sync! unless sidekiq?
+
       SemanticLogger.default_level = :info
       SemanticLogger.add_appender(io: $stderr, formatter: :color)
       SemanticLogger["Miteru"]
     end
-    memoize :logger
+    memo_wise :logger
+
+    #
+    # @return [Array<Miteru::Feeds::Base>]
+    #
+    def feeds
+      []
+    end
+    memo_wise :feeds
+
+    #
+    # @return [Array<Miteru::Notifiers::Base>]
+    #
+    def notifiers
+      []
+    end
+    memo_wise :notifiers
+
+    #
+    # @return [Miteru::Config]
+    #
+    def config
+      @config ||= Config.new
+    end
+
+    #
+    # @return [String]
+    #
+    def env
+      ENV["APP_ENV"] || ENV["RACK_ENV"]
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def development?
+      env == "development"
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def sidekiq?
+      !Miteru.config.sidekiq_redis_url.nil?
+    end
+
+    def sentry?
+      !Miteru.config.sentry_dsn.nil?
+    end
+
+    def initialize_sentry
+      return if Sentry.initialized?
+
+      Sentry.init do |config|
+        config.dsn = Miteru.config.sentry_dsn
+        config.traces_sample_rate = Miteru.config.sentry_trace_sample_rate
+        config.breadcrumbs_logger = %i[sentry_logger http_logger]
+      end
+    end
   end
 end
+
+# Services
+require "miteru/service"
+
+require "miteru/crawler"
+require "miteru/downloader"
+require "miteru/kit"
+require "miteru/orchestrator"
+require "miteru/website"
+
+# Notifiers
+require "miteru/notifiers/base"
+require "miteru/notifiers/slack"
+require "miteru/notifiers/urlscan"
+
+# Feeds
+require "miteru/feeds/base"
+
+require "miteru/feeds/ayashige"
+require "miteru/feeds/phishing_database"
+require "miteru/feeds/urlscan_pro"
+require "miteru/feeds/urlscan"
+
+# CLI
+require "miteru/cli/application"
+
+# Sidekiq
+require "sidekiq"
+
+require "miteru/sidekiq/application"
+require "miteru/sidekiq/jobs"
+
+Miteru.initialize_sentry if Miteru.sentry?

data/miteru.gemspec

@@ -1,55 +1,66 @@
 # frozen_string_literal: true
 
-lib = File.expand_path("lib", __dir__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "miteru/version"
+require 'miteru/version'
 
 Gem::Specification.new do |spec|
-  spec.name = "miteru"
+  spec.name = 'miteru'
   spec.version = Miteru::VERSION
-  spec.authors = ["Manabu Niseki"]
-  spec.email = ["manabu.niseki@gmail.com"]
-  spec.metadata["rubygems_mfa_required"] = "true"
+  spec.authors = ['Manabu Niseki']
+  spec.email = ['manabu.niseki@gmail.com']
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
-  spec.summary = "An experimental phishing kit detector"
-  spec.description = "An experimental phishing kit detector"
-  spec.homepage = "https://github.com/ninoseki/miteru"
-  spec.license = "MIT"
+  spec.summary = 'A phishing kit collector for scavengers'
+  spec.description = 'A phishing kit collector for scavengers'
+  spec.homepage = 'https://github.com/ninoseki/miteru'
+  spec.license = 'MIT'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir = "exe"
+  spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 2.3"
-  spec.add_development_dependency "coveralls_reborn", "~> 0.25"
-  spec.add_development_dependency "glint", "~> 0.1"
-  spec.add_development_dependency "mysql2", "~> 0.5"
-  spec.add_development_dependency "overcommit", "~> 0.59"
-  spec.add_development_dependency "pg", "~> 1.4"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.11"
-  spec.add_development_dependency "standard", "~> 1.14"
-  spec.add_development_dependency "vcr", "~> 6.1"
-  spec.add_development_dependency "webmock", "~> 3.17"
-  spec.add_development_dependency "webrick", "~> 1.7.0"
+  spec.add_development_dependency 'bundler', '~> 2.5'
+  spec.add_development_dependency 'capybara', '~> 3.40'
+  spec.add_development_dependency 'coveralls_reborn', '~> 0.28'
+  spec.add_development_dependency 'fuubar', '~> 2.5'
+  spec.add_development_dependency 'mysql2', '~> 0.5'
+  spec.add_development_dependency 'pg', '~> 1.5'
+  spec.add_development_dependency 'rake', '~> 13.1'
+  spec.add_development_dependency 'rspec', '~> 3.12'
+  spec.add_development_dependency 'simplecov-lcov', '~> 0.8'
+  spec.add_development_dependency 'standard', '~> 1.33'
+  spec.add_development_dependency 'test-prof', '~> 1.3'
+  spec.add_development_dependency 'vcr', '~> 6.2'
+  spec.add_development_dependency 'webmock', '~> 3.19'
 
-  spec.add_dependency "activerecord", "~> 7.0"
-  spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "dotenv", "2.8.1"
-  spec.add_dependency "down", "~> 5.3"
-  spec.add_dependency "http", "~> 5.1"
-  spec.add_dependency "memist", "2.0.2"
-  spec.add_dependency "oga", "~> 3.4"
-  spec.add_dependency "parallel", "~> 1.22"
-  spec.add_dependency "semantic_logger", "4.11.0"
-  spec.add_dependency "slack-notifier", "~> 2.4"
-  spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "~> 1.2"
-  spec.add_dependency "urlscan", "~> 0.8"
-  spec.add_dependency "uuidtools", "~> 2.2"
+  spec.add_dependency 'activerecord', '7.1.3'
+  spec.add_dependency 'anyway_config', '2.6.2'
+  spec.add_dependency 'colorize', '1.1.0'
+  spec.add_dependency 'dotenv', '2.8.1'
+  spec.add_dependency 'down', '5.4.1'
+  spec.add_dependency 'dry-files', '1.1.0'
+  spec.add_dependency 'dry-monads', '1.6.0'
+  spec.add_dependency 'http', '5.1.1'
+  spec.add_dependency 'memo_wise', '1.8.0'
+  spec.add_dependency 'oga', '3.4'
+  spec.add_dependency 'parallel', '1.24.0'
+  spec.add_dependency 'puma', '6.4.2'
+  spec.add_dependency 'rack', '3.0.8'
+  spec.add_dependency 'rack-session', '2.0.0'
+  spec.add_dependency 'rackup', '2.1.0'
+  spec.add_dependency 'semantic_logger', '4.15.0'
+  spec.add_dependency 'sentry-ruby', '5.16.1'
+  spec.add_dependency 'sentry-sidekiq', '5.16.1'
+  spec.add_dependency 'sidekiq', '7.2.1'
+  spec.add_dependency 'slack-notifier', '2.4.0'
+  spec.add_dependency 'sqlite3', '1.7.1'
+  spec.add_dependency 'thor', '1.3.0'
+  spec.add_dependency 'thor-hollaback', '0.2.1'
+  spec.add_dependency 'uuidtools', '2.2.0'
 end