miteru 1.2.2 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f422d77d43c2e555014fc16e28564b207d67d70bdd84e61d2ce2da99f368caf
-  data.tar.gz: 74d7a5405799534981d3b273b69537b3d18c171f42d9956684fd8e905361628d
+  metadata.gz: aa673c60148a21b0838fec44df350670fad6af4873cf54922ff14f8042f8a781
+  data.tar.gz: 3b09b49f88c1a56224e5f472647646619061fc57a258d54bb5a51ce4420ed22e
 SHA512:
-  metadata.gz: 02c22920de1f06080f78e421c0ee73ec51e2bf95cb50f3ab2333debec2281fa40311d7759a252dc2fa844b678e95f73625727457cbcc33dfd4c4416a31bc743c
-  data.tar.gz: fe3e85ccf788b4ee2d504be1ef2cb72d4f2b5f06680c77318b1d56b3363dd0608e05334b04f5b7b6b2927f1ea8848bb4163909d619a8224d87d181585fe8d29a
+  metadata.gz: 396239204cda13c34766428ac7b223828413b56179b72eba137be1ded7af40ff7753ded1a504cef615b3654c43f597fd223c39324b063a4a281ea729e27882fa
+  data.tar.gz: f22eb47d90dc5ca7cd77896de1fcb9420450f9055f49da4d316c8cdd9f255acb5282ed91529109122811c300a0ecfc92de71a4b26031854f9671c58e6671990f

data/.github/workflows/gem.yml

@@ -0,0 +1,36 @@
+name: Release gem
+
+on:
+  workflow_dispatch:
+    inputs:
+      rubygems-otp-code:
+        description: RubyGems OTP code
+        required: true
+        type: string
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+          - 3.3
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler: latest
+          bundler-cache: true
+      - name: Configure Git
+        # Configure Git to push a tag
+        run: |
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+      - name: Release gem
+        run: bundle exec rake release
+        env:
+          GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
+          GEM_HOST_OTP_CODE: ${{ inputs.rubygems-otp-code }}

data/.github/workflows/{test.yml → ruby.yml}

@@ -5,10 +5,9 @@ on: [push, pull_request]
 jobs:
   build:
     runs-on: ubuntu-latest
-
     services:
       postgres:
-        image: postgres:12
+        image: postgres:16
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -20,7 +19,6 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
-
       mysql:
         image: mysql:8.0
         env:
@@ -35,31 +33,24 @@ jobs:
           --health-interval=10s
           --health-timeout=5s
           --health-retries=3
-
     strategy:
-      fail-fast: false
       matrix:
-        ruby: [2.7, "3.0"]
-
+        ruby: [3.1, 3.2, 3.3]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler: latest
           bundler-cache: true
-
       - name: Install dependencies
-        run: |
-          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
-
+        run: sudo apt-get -yqq install libpq-dev libmysqlclient-dev
       - name: Test with PostgreSQL
         env:
           MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
         run: |
           bundle exec rake
-
       - name: Test with MySQL
         env:
           MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test

data/.gitignore

@@ -52,5 +52,10 @@ Gemfile.lock
 ## RSpec
 .rspec_status
 
-# SQLite database
-*.db
+# SQLite
+*.db
+*.db-shm
+*.db-wal
+
+# .env
+.env

data/.rspec

@@ -1,3 +1,3 @@
---format documentation
+--format Fuubar
 --color
 --require spec_helper

data/README.md

@@ -5,33 +5,23 @@
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
-Miteru is an experimental phishing kit detection tool.
+A phishing kit collector for scavengers.
 
 ## Disclaimer
 
 This tool is for research purposes only. The use of this tool is your responsibility.
 I take no responsibility and/or liability for how you choose to use this tool.
 
-## How it works
+## How It Works
 
 - It collects phishy URLs from the following feeds:
-  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#task.source%3Acertstream-suspicious)
-  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#task.source%3Aopenphish)
-  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#task.source%3Aphishtank)
-  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#task.source%3Aurlhaus)
-  - urlscan.io phish feed (available for Pro users)
-  - [Ayashige feed](https://github.com/ninoseki/ayashige)
-  - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
-  - [PhishStats feed](https://phishstats.info/)
-- It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
+  - urlscan.io's automatic submissions. (`task.method:automatic`)
+  - urlscan.io phish feed. (available for Pro users)
+  - [mitchellkrogza/Phishing.Database](https://github.com/mitchellkrogza/Phishing.Database)'s `phishing-links-ACTIVE-NOW.txt`.
+  - [ninoseki/ayashige](https://github.com/ninoseki/ayashige) feed.
+- It checks each phishy URL whether it enables directory listing and contains phishing kits (compressed files) or not.
   - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
-## Features
-
-- [x] Phishing kit detection & collection
-- [x] Slack notification
-- [x] Threading
-
 ## Docs
 
 - [Requirements & Installation](https://github.com/ninoseki/miteru/wiki/Requirements-&-Installation)

data/docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3"
+services:
+  redis:
+    image: "redis/redis-stack:6.2.6-v10"
+    restart: always
+    ports:
+      - ${REDIS_PORT:-6379}:6379
+      - ${REDIS_INSIGHT_PORT:-8001}:8001
+    volumes:
+      - redis:/data
+volumes:
+  redis:

data/exe/miteru

@@ -3,7 +3,7 @@
 
 $LOAD_PATH.unshift("#{__dir__}/../lib")
 
-require "miteru"
+require 'miteru'
 
-ARGV.unshift(Miteru::CLI.default_task) unless Miteru::CLI.all_tasks.key?(ARGV[0])
-Miteru::CLI.start(ARGV)
+ARGV.unshift(Miteru::CLI::App.default_task) unless Miteru::CLI::App.all_tasks.key?(ARGV[0])
+Miteru::CLI::App.start(ARGV)

data/lefthook.yml

@@ -0,0 +1,9 @@
+pre-commit:
+  commands:
+    standard:
+      glob: "*.rb"
+      run: bundle exec standardrb --fix {staged_files}
+      stage_fixed: true
+    actionlint:
+      glob: ".github/workflows/*.yaml"
+      run: actionlint

data/lib/miteru/cli/application.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "thor"
+require "thor/hollaback"
+
+require "miteru/cli/base"
+require "miteru/cli/database"
+
+require "miteru/commands/main"
+require "miteru/commands/sidekiq"
+require "miteru/commands/web"
+
+module Miteru
+  module CLI
+    #
+    # Main CLI class
+    #
+    class App < Base
+      include Commands::Main
+      include Commands::Sidekiq
+      include Commands::Web
+
+      desc "db", "Sub commands for DB"
+      subcommand "db", CLI::Database
+    end
+  end
+end

data/lib/miteru/cli/base.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Miteru
+  module CLI
+    #
+    # Base class for Thor classes
+    #
+    class Base < Thor
+      class << self
+        def exit_on_failure?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/miteru/cli/database.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "miteru/commands/database"
+
+module Miteru
+  module CLI
+    class Database < Base
+      include Commands::Database
+    end
+  end
+end

data/lib/miteru/commands/database.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Commands
+    module Database
+      class << self
+        def included(thor)
+          thor.class_eval do
+            include Concerns::DatabaseConnectable
+
+            desc "migrate", "Migrate DB schemas"
+            around :with_db_connection
+            method_option :verbose, type: :boolean, default: true
+            def migrate(direction = "up")
+              ActiveRecord::Migration.verbose = options["verbose"]
+              Miteru::Database.migrate direction.to_sym
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/miteru/commands/main.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Commands
+    module Main
+      class << self
+        def included(thor)
+          thor.class_eval do
+            include Concerns::DatabaseConnectable
+
+            method_option :auto_download, type: :boolean, default: false,
+              desc: "Enable or disable auto-downloading of phishing kits"
+            method_option :directory_traveling, type: :boolean, default: false,
+              desc: "Enable or disable directory traveling"
+            method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
+            method_option :threads, type: :numeric, desc: "Number of threads to use", default: Parallel.processor_count
+            method_option :verbose, type: :boolean, default: true
+            desc "execute", "Execute the crawler"
+            around :with_db_connection
+            def execute
+              Miteru.config.tap do |config|
+                config.auto_download = options["auto_download"]
+                config.directory_traveling = options["directory_traveling"]
+                config.download_to = options["download_to"]
+                config.threads = options["threads"]
+                config.verbose = options["verbose"]
+              end
+
+              Orchestrator.call
+            end
+            default_command :execute
+          end
+        end
+      end
+    end
+  end
+end

data/lib/miteru/commands/sidekiq.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Commands
+    #
+    # Sidekiq sub-commands
+    #
+    module Sidekiq
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "sidekiq", "Start Sidekiq"
+            method_option :env, type: :string, default: "production", desc: "Environment"
+            method_option :concurrency, type: :numeric, default: 5, desc: "Sidekiq concurrency", aliases: "-c"
+            def sidekiq
+              require "sidekiq/cli"
+
+              ENV["APP_ENV"] ||= options["env"]
+              concurrency = options["concurrency"].to_s
+
+              cli = ::Sidekiq::CLI.instance
+              cli.parse [
+                "-r",
+                File.expand_path(File.join(__dir__, "..", "sidekiq", "application.rb")),
+                "-c",
+                concurrency
+              ]
+              cli.run
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/miteru/commands/web.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Commands
+    #
+    # Web sub-commands
+    #
+    module Web
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "web", "Start the web app"
+            method_option :port, type: :numeric, default: 9292, desc: "Port to listen on"
+            method_option :host, type: :string, default: "localhost", desc: "Hostname to listen on"
+            method_option :threads, type: :string, default: "0:3", desc: "min:max threads to use"
+            method_option :verbose, type: :boolean, default: false, desc: "Don't report each request"
+            method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
+            method_option :env, type: :string, default: "production", desc: "Environment"
+            def web
+              require "miteru/web/application"
+
+              ENV["APP_ENV"] ||= options["env"]
+
+              Miteru::Web::App.run!(
+                port: options["port"],
+                host: options["host"],
+                threads: options["threads"],
+                verbose: options["verbose"],
+                worker_timeout: options["worker_timeout"]
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/miteru/concerns/database_connectable.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Concerns
+    #
+    # Database connectable concern
+    #
+    module DatabaseConnectable
+      extend ActiveSupport::Concern
+
+      def with_db_connection(&block)
+        Database.with_db_connection(&block)
+      end
+    end
+  end
+end

data/lib/miteru/concerns/error_unwrappable.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Concerns
+    #
+    # Error unwrappable concern
+    #
+    module ErrorUnwrappable
+      extend ActiveSupport::Concern
+
+      def unwrap_error(err)
+        return err unless err.is_a?(Dry::Monads::UnwrapError)
+
+        # NOTE: UnwrapError's receiver can be either of:
+        #       - Dry::Monads::Try::Error
+        #       - Dry::Monads::Result::Failure
+        receiver = err.receiver
+        case receiver
+        when Dry::Monads::Try::Error
+          # Error may be wrapped like Matryoshka
+          unwrap_error receiver.exception
+        when Dry::Monads::Failure
+          unwrap_error receiver.failure
+        else
+          err
+        end
+      end
+    end
+  end
+end

data/lib/miteru/config.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "anyway_config"
+
+module Miteru
+  class Config < Anyway::Config
+    config_name :miteru
+    env_prefix ""
+
+    attr_config(
+      auto_download: false,
+      database_url: URI("sqlite3:miteru.db"),
+      directory_traveling: false,
+      download_to: "/tmp",
+      file_max_size: 1024 * 1024 * 100,
+      file_extensions: [".zip", ".rar", ".7z", ".tar", ".gz"],
+      file_mime_types: ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar",
+        "application/gzip"],
+      api_timeout: 60,
+      http_timeout: 60,
+      download_timeout: 60,
+      sentry_dsn: nil,
+      sentry_trace_sample_rate: 0.25,
+      sidekiq_redis_url: nil,
+      slack_channel: "#general",
+      slack_webhook_url: nil,
+      threads: Parallel.processor_count,
+      urlscan_api_key: nil,
+      urlscan_submit_visibility: "public",
+      urlscan_date_condition: ">now-1h",
+      verbose: false
+    )
+
+    # @!attribute [r] sentry_dsn
+    #   @return [String, nil]
+
+    # @!attribute [r] sentry_trace_sample_rate
+    #   @return [Float]
+
+    # @!attribute [r] sidekiq_redis_url
+    #   @return [String]
+
+    # @!attribute [r] http_timeout
+    #   @return [Integer]
+
+    # @!attribute [r] api_timeout
+    #   @return [Integer]
+
+    # @!attribute [r] download_timeout
+    #   @return [Integer]
+
+    # @!attribute [rw] auto_download
+    #   @return [Boolean]
+
+    # @!attribute [rw] directory_traveling
+    #   @return [Boolean]
+
+    # @!attribute [rw] download_to
+    #   @return [String]
+
+    # @!attribute [rw] threads
+    #   @return [Integer]
+
+    # @!attribute [rw] verbose
+    #   @return [Boolean]
+
+    # @!attribute [r] database_url
+    #   @return [URI]
+
+    # @!attribute [r] file_max_size
+    #   @return [Integer]
+
+    # @!attribute [r] file_extensions
+    #   @return [Array<String>]
+
+    # @!attribute [r] file_mime_types
+    #   @return [Array<String>]
+
+    # @!attribute [r] slack_webhook_url
+    #   @return [String, nil]
+
+    # @!attribute [r] slack_channel
+    #   @return [String]
+
+    # @!attribute [r] urlscan_api_key
+    #   @return [String, nil]
+
+    # @!attribute [r] urlscan_submit_visibility
+    #   @return [String]
+
+    # @!attribute [r] urlscan_date_condition
+    #   @return [String]
+
+    def database_url=(val)
+      super(URI(val.to_s))
+    end
+  end
+end

data/lib/miteru/crawler.rb

@@ -1,63 +1,47 @@
 # frozen_string_literal: true
 
-require "colorize"
-require "parallel"
-require "uri"
-
 module Miteru
-  class Crawler
-    attr_reader :downloader, :feeds
+  class Crawler < Service
+    #
+    # @param [Miteru::Website] website
+    #
+    def call(website)
+      Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
+        Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
+        return unless website.has_kits?
 
-    def initialize
-      @downloader = Downloader.new(Miteru.configuration.download_to)
-      @feeds = Feeds.new
-    end
+        notify website
 
-    def crawl(entry)
-      website = Website.new(entry.url, entry.source)
-      downloader.download_kits(website.kits) if website.has_kits? && auto_download?
-      notify(website) if website.has_kits? || verbose?
-    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
-      nil
-    end
+        return unless auto_download?
 
-    def execute
-      suspicious_entries = feeds.suspicious_entries
-      Miteru.logger.info "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
+        website.kits.each do |kit|
+          downloader = Downloader.new(kit)
+          result = downloader.result
 
-      Parallel.each(suspicious_entries, in_threads: threads) do |entry|
-        crawl entry
-      end
+          if result.success?
+            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}")
+          else
+            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}")
+          end
+        end
+      end.recover { nil }.value!
     end
 
-    def threads
-      @threads ||= Miteru.configuration.threads
-    end
-
-    def notify(website)
-      Parallel.each(notifiers) do |notifier|
-        notifier.notify website
-      end
-    end
+    private
 
     def auto_download?
-      Miteru.configuration.auto_download?
+      Miteru.config.auto_download
     end
 
-    def verbose?
-      Miteru.configuration.verbose?
+    def notify(website)
+      Parallel.each(notifiers) { |notifier| notifier.call(website) }
     end
 
-    private
-
+    #
+    # @return [Array<Miteru::Notifiers::Base>]
+    #
     def notifiers
-      @notifiers ||= [Notifiers::Slack.new, Notifiers::UrlScan.new].select(&:notifiable?)
-    end
-
-    class << self
-      def execute
-        new.execute
-      end
+      @notifiers ||= Miteru.notifiers.map(&:new)
     end
   end
 end