miteru 2.0.0 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2f6d2dc9ff42073c1f461c43f93a1fe3eaf9d89076b602b3cf08eb0457cf657
-  data.tar.gz: bc962aaee877692d88bcf982c754569e11665cc005ae5fad807470a4dc423fc8
+  metadata.gz: c5c223747e06cb6d010f4a5e3157b1a4999662761ccaef108521b1627489b800
+  data.tar.gz: 403ed81289db89582add2420290b6f8c343da88babf2c548940871f0791655dd
 SHA512:
-  metadata.gz: '08d48cb38e4d4cc0c3a2539a003ecce6d62659c72789de4f92f4663bce21c88e4e28dd67be4daff3154a624e8652e560d683b3decc1a4a7f3a513114c8fddd72'
-  data.tar.gz: 260b0237b2652ddf596e7a40b2060a14fb77d18182d4fe44c5ef9b74c3b53204d5907f8d46df6c3471797bc7f20b3befd518e8e399f9d62f26b857649cfd7f23
+  metadata.gz: 255a86779cdb3473ba4b7683159b1290b0774eae7863db2cd9e770610b6a14f1397ff1a07823132009daafc5b0c75aea2489cd39c051067311242cf30535517d
+  data.tar.gz: 3662c8946aba747ad6dcda2fbedc1dc34b0cba8fd680e9d573b3a1c24114b5658f5029ecb3bf5e222c2d4cced01dd0f6f1542d4878566fab0880456bb500997e

data/.gitignore

@@ -52,8 +52,10 @@ Gemfile.lock
 ## RSpec
 .rspec_status
 
-# SQLite database
+# SQLite
 *.db
+*.db-shm
+*.db-wal
 
 # .env
 .env

data/lib/miteru/commands/main.rb

@@ -15,7 +15,7 @@ module Miteru
             method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
             method_option :threads, type: :numeric, desc: "Number of threads to use", default: Parallel.processor_count
             method_option :verbose, type: :boolean, default: true
-            desc "execute", "Excecute the crawler"
+            desc "execute", "Execute the crawler"
             around :with_db_connection
             def execute
               Miteru.config.tap do |config|

data/lib/miteru/crawler.rb

@@ -8,7 +8,7 @@ module Miteru
     def call(website)
       Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
         Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
-        return unless website.has_kits?
+        return unless website.kits?
 
         notify website
 
@@ -17,11 +17,10 @@ module Miteru
         website.kits.each do |kit|
           downloader = Downloader.new(kit)
           result = downloader.result
-
           if result.success?
-            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}")
+            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}.")
           else
-            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}")
+            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}.")
           end
         end
       end.recover { nil }.value!
@@ -33,8 +32,18 @@ module Miteru
       Miteru.config.auto_download
     end
 
+    #
+    # @param [Miteru::Website] website
+    #
     def notify(website)
-      Parallel.each(notifiers) { |notifier| notifier.call(website) }
+      notifiers.each do |notifier|
+        result = notifier.result(website)
+        if result.success?
+          Miteru.logger.info("Notifier:#{notifier.name} succeeded.")
+        else
+          Miteru.logger.warn("Notifier:#{notifier.name} failed - #{result.failure}.")
+        end
+      end
     end
 
     #

data/lib/miteru/kit.rb

@@ -25,6 +25,8 @@ module Miteru
     # @param [String] source
     #
     def initialize(url, source:)
+      super()
+
       @url = url
       @source = source
 
@@ -37,7 +39,6 @@ module Miteru
     def valid?
       # make a HEAD request for the validation
       before_validation
-
       valid_ext? && reachable? && valid_mime_type? && valid_content_length?
     end
 
@@ -110,7 +111,7 @@ module Miteru
     end
 
     def http
-      HTTP::Factory.build
+      HTTP::Factory.build(raise_exception: false)
     end
 
     def before_validation

data/lib/miteru/notifiers/base.rb

@@ -11,6 +11,10 @@ module Miteru
         raise NotImplementedError
       end
 
+      def name
+        @name ||= self.class.to_s.split("::").last
+      end
+
       class << self
         def inherited(child)
           super

data/lib/miteru/notifiers/urlscan.rb

@@ -9,10 +9,7 @@ module Miteru
       def call(website)
         return unless callable?
 
-        kits = website.kits.select(&:downloaded?)
-        return unless kits.any?
-
-        kits.each { |kit| submit(kit.url) }
+        website.kits.each { |kit| submit(kit.url) }
       end
 
       def callable?
@@ -33,7 +30,7 @@ module Miteru
       end
 
       def timeout
-        Miteru.config.timeout
+        Miteru.config.api_timeout
       end
 
       def tags
@@ -45,7 +42,7 @@ module Miteru
       end
 
       def submit(url)
-        http.post("/api/v1/scan/", json: {tags:, visibility:, url:})
+        http.post("https://urlscan.io/api/v1/scan/", json: {tags:, visibility:, url:})
       end
     end
   end

data/lib/miteru/orchestrator.rb

@@ -6,10 +6,22 @@ module Miteru
       Miteru.logger.info("#{websites.length} websites loaded in total.") if verbose?
 
       if Miteru.sidekiq?
-        websites.each { |website| Jobs::CrawleJob.perform_async(website.url, website.source) }
+        websites.each do |website|
+          Jobs::CrawleJob.perform_async(website.url, website.source)
+          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
+        end
       else
         Miteru.logger.info("Use #{threads} thread(s).") if verbose?
-        Parallel.each(websites, in_threads: threads) { |website| crawl(website) }
+        Parallel.each(websites, in_threads: threads) do |website|
+          Miteru.logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
+
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
+        end
       end
     end
 

data/lib/miteru/sidekiq/jobs.rb

@@ -14,7 +14,14 @@ module Miteru
       #
       def perform(url, source)
         website = Miteru::Website.new(url, source:)
-        with_db_connection { Crawler.call(website) }
+        with_db_connection do
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
+        end
       end
     end
   end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.0.0"
+  VERSION = "2.0.2"
 end

data/lib/miteru/website.rb

@@ -33,15 +33,8 @@ module Miteru
       title.to_s.start_with? "Index of"
     end
 
-    def has_kits?
-      @has_kits ||= lambda do
-        Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
-          OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
-          !kits.empty?
-        end.recover do
-          false
-        end.value!
-      end.call
+    def kits?
+      kits.any?
     end
 
     def links
@@ -81,8 +74,9 @@ module Miteru
       Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
         OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
         doc.css("a").filter_map { |a| a.get("href") }.map do |href|
-          href = href.start_with?("/") ? href : "/#{href}"
-          url + href
+          normalized_href = href.start_with?("/") ? href : "/#{href}"
+          normalized_url = url.end_with?("/") ? url.delete_suffix("/") : url
+          normalized_url + normalized_href
         end
       end.recover { [] }.value!
     end

data/miteru.gemspec

@@ -40,6 +40,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'webmock', '~> 3.19'
 
   spec.add_dependency 'activerecord', '7.1.3'
+  spec.add_dependency 'addressable', '2.8.6'
   spec.add_dependency 'anyway_config', '2.6.2'
   spec.add_dependency 'colorize', '1.1.0'
   spec.add_dependency 'dotenv', '2.8.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-28 00:00:00.000000000 Z
+date: 2024-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 7.1.3
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.6
 - !ruby/object:Gem::Dependency
   name: anyway_config
   requirement: !ruby/object:Gem::Requirement
@@ -584,9 +598,6 @@ files:
 - lib/miteru/version.rb
 - lib/miteru/web/application.rb
 - lib/miteru/website.rb
-- miteru-sidekiq.service
-- miteru.db-shm
-- miteru.db-wal
 - miteru.gemspec
 - renovate.json
 - screenshots/slack.png

data/miteru-sidekiq.service

@@ -1,13 +0,0 @@
-[Unit]
-Description=Miteru Sidekiq
-After=network.target
-
-[Service]
-Type=simple
-User=ninoseki
-WorkingDirectory=/opt/jsac_mihari_workshop
-ExecStart=/usr/bin/npx
-Restart=on-failure
-
-[Install]
-WantedBy = multi-user.target