miteru 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa673c60148a21b0838fec44df350670fad6af4873cf54922ff14f8042f8a781
-  data.tar.gz: 3b09b49f88c1a56224e5f472647646619061fc57a258d54bb5a51ce4420ed22e
+  metadata.gz: c5c223747e06cb6d010f4a5e3157b1a4999662761ccaef108521b1627489b800
+  data.tar.gz: 403ed81289db89582add2420290b6f8c343da88babf2c548940871f0791655dd
 SHA512:
-  metadata.gz: 396239204cda13c34766428ac7b223828413b56179b72eba137be1ded7af40ff7753ded1a504cef615b3654c43f597fd223c39324b063a4a281ea729e27882fa
-  data.tar.gz: f22eb47d90dc5ca7cd77896de1fcb9420450f9055f49da4d316c8cdd9f255acb5282ed91529109122811c300a0ecfc92de71a4b26031854f9671c58e6671990f
+  metadata.gz: 255a86779cdb3473ba4b7683159b1290b0774eae7863db2cd9e770610b6a14f1397ff1a07823132009daafc5b0c75aea2489cd39c051067311242cf30535517d
+  data.tar.gz: 3662c8946aba747ad6dcda2fbedc1dc34b0cba8fd680e9d573b3a1c24114b5658f5029ecb3bf5e222c2d4cced01dd0f6f1542d4878566fab0880456bb500997e

data/lib/miteru/crawler.rb

@@ -8,7 +8,7 @@ module Miteru
     def call(website)
       Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
         Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
-        return unless website.has_kits?
+        return unless website.kits?
 
         notify website
 
@@ -17,11 +17,10 @@ module Miteru
         website.kits.each do |kit|
           downloader = Downloader.new(kit)
           result = downloader.result
-
           if result.success?
-            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}")
+            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}.")
           else
-            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}")
+            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}.")
           end
         end
       end.recover { nil }.value!
@@ -33,8 +32,18 @@ module Miteru
       Miteru.config.auto_download
     end
 
+    #
+    # @param [Miteru::Website] website
+    #
     def notify(website)
-      Parallel.each(notifiers) { |notifier| notifier.call(website) }
+      notifiers.each do |notifier|
+        result = notifier.result(website)
+        if result.success?
+          Miteru.logger.info("Notifier:#{notifier.name} succeeded.")
+        else
+          Miteru.logger.warn("Notifier:#{notifier.name} failed - #{result.failure}.")
+        end
+      end
     end
 
     #

data/lib/miteru/kit.rb

@@ -25,6 +25,8 @@ module Miteru
     # @param [String] source
     #
     def initialize(url, source:)
+      super()
+
       @url = url
       @source = source
 
@@ -37,7 +39,6 @@ module Miteru
     def valid?
       # make a HEAD request for the validation
       before_validation
-
       valid_ext? && reachable? && valid_mime_type? && valid_content_length?
     end
 
@@ -110,7 +111,7 @@ module Miteru
     end
 
     def http
-      HTTP::Factory.build
+      HTTP::Factory.build(raise_exception: false)
     end
 
     def before_validation

data/lib/miteru/notifiers/base.rb

@@ -11,6 +11,10 @@ module Miteru
         raise NotImplementedError
       end
 
+      def name
+        @name ||= self.class.to_s.split("::").last
+      end
+
       class << self
         def inherited(child)
           super

data/lib/miteru/notifiers/urlscan.rb

@@ -9,10 +9,7 @@ module Miteru
       def call(website)
         return unless callable?
 
-        kits = website.kits.select(&:downloaded?)
-        return unless kits.any?
-
-        kits.each { |kit| submit(kit.url) }
+        website.kits.each { |kit| submit(kit.url) }
       end
 
       def callable?
@@ -33,7 +30,7 @@ module Miteru
       end
 
       def timeout
-        Miteru.config.timeout
+        Miteru.config.api_timeout
       end
 
       def tags
@@ -45,7 +42,7 @@ module Miteru
       end
 
       def submit(url)
-        http.post("/api/v1/scan/", json: {tags:, visibility:, url:})
+        http.post("https://urlscan.io/api/v1/scan/", json: {tags:, visibility:, url:})
       end
     end
   end

data/lib/miteru/orchestrator.rb

@@ -6,16 +6,21 @@ module Miteru
       Miteru.logger.info("#{websites.length} websites loaded in total.") if verbose?
 
       if Miteru.sidekiq?
-
         websites.each do |website|
           Jobs::CrawleJob.perform_async(website.url, website.source)
-          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued") if verbose?
+          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
         end
       else
         Miteru.logger.info("Use #{threads} thread(s).") if verbose?
         Parallel.each(websites, in_threads: threads) do |website|
-          Miteru.logger.info("Website:#{website.truncated_url} crawling started") if verbose?
-          crawl(website)
+          Miteru.logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
+
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
         end
       end
     end

data/lib/miteru/sidekiq/jobs.rb

@@ -14,7 +14,14 @@ module Miteru
       #
       def perform(url, source)
         website = Miteru::Website.new(url, source:)
-        with_db_connection { Crawler.call(website) }
+        with_db_connection do
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
+        end
       end
     end
   end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.0.1"
+  VERSION = "2.0.2"
 end

data/lib/miteru/website.rb

@@ -33,15 +33,8 @@ module Miteru
       title.to_s.start_with? "Index of"
     end
 
-    def has_kits?
-      @has_kits ||= lambda do
-        Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
-          OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
-          !kits.empty?
-        end.recover do
-          false
-        end.value!
-      end.call
+    def kits?
+      kits.any?
     end
 
     def links
@@ -81,8 +74,9 @@ module Miteru
       Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
         OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
         doc.css("a").filter_map { |a| a.get("href") }.map do |href|
-          href = href.start_with?("/") ? href : "/#{href}"
-          url + href
+          normalized_href = href.start_with?("/") ? href : "/#{href}"
+          normalized_url = url.end_with?("/") ? url.delete_suffix("/") : url
+          normalized_url + normalized_href
         end
       end.recover { [] }.value!
     end

data/miteru.gemspec

@@ -40,6 +40,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'webmock', '~> 3.19'
 
   spec.add_dependency 'activerecord', '7.1.3'
+  spec.add_dependency 'addressable', '2.8.6'
   spec.add_dependency 'anyway_config', '2.6.2'
   spec.add_dependency 'colorize', '1.1.0'
   spec.add_dependency 'dotenv', '2.8.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-28 00:00:00.000000000 Z
+date: 2024-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 7.1.3
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.6
 - !ruby/object:Gem::Dependency
   name: anyway_config
   requirement: !ruby/object:Gem::Requirement