miteru 2.0.1 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa673c60148a21b0838fec44df350670fad6af4873cf54922ff14f8042f8a781
-  data.tar.gz: 3b09b49f88c1a56224e5f472647646619061fc57a258d54bb5a51ce4420ed22e
+  metadata.gz: 4133dc72d0d07c5719999c526b8534a2dd7b5db5fe74172042b747c5cc557d4a
+  data.tar.gz: d2f59be0a18bdaee1b74633a58b8ed59a396862defd4b80dff4ae8ae9d8199f4
 SHA512:
-  metadata.gz: 396239204cda13c34766428ac7b223828413b56179b72eba137be1ded7af40ff7753ded1a504cef615b3654c43f597fd223c39324b063a4a281ea729e27882fa
-  data.tar.gz: f22eb47d90dc5ca7cd77896de1fcb9420450f9055f49da4d316c8cdd9f255acb5282ed91529109122811c300a0ecfc92de71a4b26031854f9671c58e6671990f
+  metadata.gz: 4a311f7ffa407ed11951f8f3b7ce696510ba31f8888dff459d3a04e4a19ec892f6f7e5651b7616e70a4482cf07d92f657dc4765ef17b6f42e66a3b526c7e1e26
+  data.tar.gz: 70b43504f29dc3646a343a87029e5c7a785d777099352717a4d2c2f26801879d2541d90ec5568f04d56c6635a35067397c4042e64a3177b6375cb74e51b5eae3

data/lib/miteru/crawler.rb

@@ -8,7 +8,7 @@ module Miteru
     def call(website)
       Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
         Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
-        return unless website.has_kits?
+        return unless website.kits?
 
         notify website
 
@@ -17,11 +17,10 @@ module Miteru
         website.kits.each do |kit|
           downloader = Downloader.new(kit)
           result = downloader.result
-
           if result.success?
-            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}")
+            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}.")
           else
-            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}")
+            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}.")
           end
         end
       end.recover { nil }.value!
@@ -33,8 +32,18 @@ module Miteru
       Miteru.config.auto_download
     end
 
+    #
+    # @param [Miteru::Website] website
+    #
     def notify(website)
-      Parallel.each(notifiers) { |notifier| notifier.call(website) }
+      notifiers.each do |notifier|
+        result = notifier.result(website)
+        if result.success?
+          Miteru.logger.info("Notifier:#{notifier.name} succeeded.")
+        else
+          Miteru.logger.warn("Notifier:#{notifier.name} failed - #{result.failure}.")
+        end
+      end
     end
 
     #

data/lib/miteru/feeds/ayashige.rb

@@ -3,9 +3,6 @@
 module Miteru
   class Feeds
     class Ayashige < Base
-      HOST = "ayashige.herokuapp.com"
-      URL = "https://#{HOST}".freeze
-
       def initialize(base_url = "https://ayashige.herokuapp.com")
         super(base_url)
       end

data/lib/miteru/kit.rb

@@ -25,6 +25,8 @@ module Miteru
     # @param [String] source
     #
     def initialize(url, source:)
+      super()
+
       @url = url
       @source = source
 
@@ -37,7 +39,6 @@ module Miteru
     def valid?
       # make a HEAD request for the validation
       before_validation
-
       valid_ext? && reachable? && valid_mime_type? && valid_content_length?
     end
 
@@ -110,7 +111,7 @@ module Miteru
     end
 
     def http
-      HTTP::Factory.build
+      HTTP::Factory.build(raise_exception: false)
     end
 
     def before_validation

data/lib/miteru/notifiers/base.rb

@@ -11,6 +11,10 @@ module Miteru
         raise NotImplementedError
       end
 
+      def name
+        @name ||= self.class.to_s.split("::").last
+      end
+
       class << self
         def inherited(child)
           super

data/lib/miteru/notifiers/slack.rb

@@ -87,7 +87,7 @@ module Miteru
       end
 
       def callable?
-        !Miteru.config.slack_webhook_url.nil?
+        !slack_webhook_url.nil?
       end
 
       private

data/lib/miteru/notifiers/urlscan.rb

@@ -9,10 +9,7 @@ module Miteru
       def call(website)
         return unless callable?
 
-        kits = website.kits.select(&:downloaded?)
-        return unless kits.any?
-
-        kits.each { |kit| submit(kit.url) }
+        website.kits.each { |kit| submit(kit.url) }
       end
 
       def callable?
@@ -33,7 +30,7 @@ module Miteru
       end
 
       def timeout
-        Miteru.config.timeout
+        Miteru.config.api_timeout
       end
 
       def tags
@@ -45,7 +42,7 @@ module Miteru
       end
 
       def submit(url)
-        http.post("/api/v1/scan/", json: {tags:, visibility:, url:})
+        http.post("https://urlscan.io/api/v1/scan/", json: {tags:, visibility:, url:})
       end
     end
   end

data/lib/miteru/orchestrator.rb

@@ -6,16 +6,21 @@ module Miteru
       Miteru.logger.info("#{websites.length} websites loaded in total.") if verbose?
 
       if Miteru.sidekiq?
-
         websites.each do |website|
           Jobs::CrawleJob.perform_async(website.url, website.source)
-          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued") if verbose?
+          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
         end
       else
         Miteru.logger.info("Use #{threads} thread(s).") if verbose?
         Parallel.each(websites, in_threads: threads) do |website|
-          Miteru.logger.info("Website:#{website.truncated_url} crawling started") if verbose?
-          crawl(website)
+          Miteru.logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
+
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
         end
       end
     end
@@ -35,7 +40,14 @@ module Miteru
             Miteru.logger.warn("Feed:#{feed.source} failed - #{result.failure}")
           end
         end
-      end.flatten
+      end.flatten.uniq(&:url)
+    end
+
+    #
+    # @return [Array<Miteru::Feeds::Base>]
+    #
+    def feeds
+      Miteru.feeds.map(&:new)
     end
 
     private
@@ -47,12 +59,5 @@ module Miteru
     def verbose?
       Miteru.config.verbose
     end
-
-    #
-    # @return [Array<Miteru::Feeds::Base>]
-    #
-    def feeds
-      Miteru.feeds.map(&:new)
-    end
   end
 end

data/lib/miteru/sidekiq/jobs.rb

@@ -14,7 +14,14 @@ module Miteru
       #
       def perform(url, source)
         website = Miteru::Website.new(url, source:)
-        with_db_connection { Crawler.call(website) }
+        with_db_connection do
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
+        end
       end
     end
   end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.0.1"
+  VERSION = "2.0.3"
 end

data/lib/miteru/website.rb

@@ -33,15 +33,8 @@ module Miteru
       title.to_s.start_with? "Index of"
     end
 
-    def has_kits?
-      @has_kits ||= lambda do
-        Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
-          OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
-          !kits.empty?
-        end.recover do
-          false
-        end.value!
-      end.call
+    def kits?
+      kits.any?
     end
 
     def links
@@ -81,8 +74,9 @@ module Miteru
       Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
         OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
         doc.css("a").filter_map { |a| a.get("href") }.map do |href|
-          href = href.start_with?("/") ? href : "/#{href}"
-          url + href
+          normalized_href = href.start_with?("/") ? href : "/#{href}"
+          normalized_url = url.end_with?("/") ? url.delete_suffix("/") : url
+          normalized_url + normalized_href
         end
       end.recover { [] }.value!
     end

data/miteru.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'mysql2', '~> 0.5'
   spec.add_development_dependency 'pg', '~> 1.5'
   spec.add_development_dependency 'rake', '~> 13.1'
-  spec.add_development_dependency 'rspec', '~> 3.12'
+  spec.add_development_dependency 'rspec', '~> 3.13'
   spec.add_development_dependency 'simplecov-lcov', '~> 0.8'
   spec.add_development_dependency 'standard', '~> 1.33'
   spec.add_development_dependency 'test-prof', '~> 1.3'
@@ -40,18 +40,19 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'webmock', '~> 3.19'
 
   spec.add_dependency 'activerecord', '7.1.3'
+  spec.add_dependency 'addressable', '2.8.6'
   spec.add_dependency 'anyway_config', '2.6.2'
   spec.add_dependency 'colorize', '1.1.0'
   spec.add_dependency 'dotenv', '2.8.1'
   spec.add_dependency 'down', '5.4.1'
   spec.add_dependency 'dry-files', '1.1.0'
   spec.add_dependency 'dry-monads', '1.6.0'
-  spec.add_dependency 'http', '5.1.1'
+  spec.add_dependency 'http', '5.2.0'
   spec.add_dependency 'memo_wise', '1.8.0'
   spec.add_dependency 'oga', '3.4'
   spec.add_dependency 'parallel', '1.24.0'
   spec.add_dependency 'puma', '6.4.2'
-  spec.add_dependency 'rack', '3.0.8'
+  spec.add_dependency 'rack', '3.0.9'
   spec.add_dependency 'rack-session', '2.0.0'
   spec.add_dependency 'rackup', '2.1.0'
   spec.add_dependency 'semantic_logger', '4.15.0'
@@ -59,7 +60,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sentry-sidekiq', '5.16.1'
   spec.add_dependency 'sidekiq', '7.2.1'
   spec.add_dependency 'slack-notifier', '2.4.0'
-  spec.add_dependency 'sqlite3', '1.7.1'
+  spec.add_dependency 'sqlite3', '1.7.2'
   spec.add_dependency 'thor', '1.3.0'
   spec.add_dependency 'thor-hollaback', '0.2.1'
   spec.add_dependency 'uuidtools', '2.2.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.3
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-28 00:00:00.000000000 Z
+date: 2024-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '3.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '3.13'
 - !ruby/object:Gem::Dependency
   name: simplecov-lcov
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 7.1.3
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.6
 - !ruby/object:Gem::Dependency
   name: anyway_config
   requirement: !ruby/object:Gem::Requirement
@@ -296,14 +310,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.1.1
+        version: 5.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.1.1
+        version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: memo_wise
   requirement: !ruby/object:Gem::Requirement
@@ -366,14 +380,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.8
+        version: 3.0.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.8
+        version: 3.0.9
 - !ruby/object:Gem::Dependency
   name: rack-session
   requirement: !ruby/object:Gem::Requirement
@@ -478,14 +492,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: 1.7.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: 1.7.2
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -586,7 +600,6 @@ files:
 - lib/miteru/website.rb
 - miteru.gemspec
 - renovate.json
-- screenshots/slack.png
 homepage: https://github.com/ninoseki/miteru
 licenses:
 - MIT