miteru 2.2.0 → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62e5976eed670c3d2f2b0309e7723434c1e5c04ad337d4db519a58362618d566
-  data.tar.gz: fe54d9316505b380ac50ba9ab8cf520f2fd5b0e76cf88f0e520d7fb0c18a3b56
+  metadata.gz: 62cde56f6eff0777959ebcadb72619505b63d7fc88cde3a362effd9a707f364d
+  data.tar.gz: e3e0011791ab52b700f2fa824ee37285d1306426ec0cc54f5ca48f0e5bde8489
 SHA512:
-  metadata.gz: 122f011baecb0a38639610ffaf68645c48f454ef7aa6ba79d5e812f71129f303a4386308cf639e233ff43473907f9567f90625b3651e027d18d6324e694ba00e
-  data.tar.gz: 0f21e5a5b6882fb9705d9988647de16aad44693ddfbbd9983a01f16c029923629322c40165a28dcb33e4a49482bcddbc1d272825beae7b0017400c560e16a57d
+  metadata.gz: ebbf6663d3b6b07486c19f5dd56b2119e33ccafebe6f57ea5cf4629fa74a7ece9309ff6e8e900881d4bfe30d9f12e3d9cd5315f79ddec6b2f8e221ad824222dd
+  data.tar.gz: 63abffb3e35ab8f631188d2c116ab43c21535fbab9fb4c24d7cf304317f383e0cbc48dca01c17c600f67eeafa1a23cb93915f520eed92ff1997141b16e492be9

data/lib/miteru/cache.rb

@@ -19,8 +19,8 @@ module Miteru
     # @param [Integer. nil] ex
     #
     def set(key, value, ex:)
-      value = redis.set("#{prefix}:#{key}", value, ex:)
-      Miteru.logger.info("Cache:#{key} is set.") if verbose?
+      value = redis.set("#{cache_prefix}:#{key}", value, ex:)
+      logger.info("Cache:#{key} is set.") if verbose?
       value
     end
 
@@ -28,21 +28,13 @@ module Miteru
     # @param [String] key
     #
     def cached?(key)
-      value = redis.exists?("#{prefix}:#{key}")
-      Miteru.logger.info("Cache:#{key} found.") if verbose?
+      value = redis.exists?("#{cache_prefix}:#{key}")
+      logger.info("Cache:#{key} found.") if verbose?
       value
     end
 
     private
 
-    def verbose?
-      Miteru.config.verbose
-    end
-
-    def prefix
-      Miteru.config.cache_prefix
-    end
-
     #
     # @return [Redis]
     #

data/lib/miteru/concerns/url_truncatable.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Miteru
+  module Concerns
+    module UrlTruncatable
+      extend ActiveSupport::Concern
+
+      def decoded_url
+        @decoded_url ||= URI.decode_www_form_component(url)
+      end
+
+      #
+      # @return [String]
+      #
+      def truncated_url
+        @truncated_url ||= decoded_url.truncate(64)
+      end
+
+      def defanged_truncated_url
+        @defanged_truncated_url ||= truncated_url.to_s.gsub(".", "[.]")
+      end
+    end
+  end
+end

data/lib/miteru/crawler.rb

@@ -11,23 +11,21 @@ module Miteru
       Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
         info = "Website:#{website.info}."
         info = info.colorize(:red) if website.kits?
-        Miteru.logger.info(info)
+        logger.info(info)
 
         website.kits.each do |kit|
           downloader = Downloader.new(kit)
           result = downloader.result
-
           unless result.success?
-            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}.")
+            logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}.")
             next
           end
-
           destination = result.value!
-          Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{destination}.")
+          logger.info("Kit:#{kit.truncated_url} downloaded as #{destination}.")
           # Remove downloaded file if auto_download is not allowed
           FileUtils.rm(destination, force: true) unless auto_download?
-          # Notify the website
-          notify website
+          # Notify the kit
+          notify(kit)
         end
 
         # Cache the website
@@ -37,39 +35,22 @@ module Miteru
 
     private
 
-    def cache?
-      Miteru.cache?
-    end
-
-    def cache
-      Miteru.cache
-    end
-
-    def cache_ex
-      Miteru.config.cache_ex
-    end
-
-    def auto_download?
-      Miteru.config.auto_download
-    end
-
     #
-    # @param [Miteru::Website] website
+    # @param [Miteru::Kit] kit
     #
-    def notify(website)
+    def notify(kit)
       notifiers.each do |notifier|
-        result = notifier.result(website)
+        result = notifier.result(kit)
         if result.success?
-          Miteru.logger.info("Notifier:#{notifier.name} succeeded.")
+          logger.info("Notifier:#{notifier.name} succeeded.")
         else
-          Miteru.logger.warn("Notifier:#{notifier.name} failed - #{result.failure}.")
+          logger.warn("Notifier:#{notifier.name} failed - #{result.failure}.")
         end
       end
     end
 
-    #
-    # @return [Array<Miteru::Notifiers::Base>]
-    #
+    private
+
     def notifiers
       @notifiers ||= Miteru.notifiers.map(&:new)
     end

data/lib/miteru/downloader.rb

@@ -43,7 +43,7 @@ module Miteru
 
       # Record a kit in DB
       Record.create_by_kit_and_hash(kit, sha256: sha256(destination))
-      Miteru.logger.info "Download #{kit.url} as #{destination}"
+      logger.info "Download #{kit.url} as #{destination}"
 
       destination
     end

data/lib/miteru/kit.rb

@@ -2,6 +2,8 @@
 
 module Miteru
   class Kit < Service
+    include Concerns::UrlTruncatable
+
     # @return [String]
     attr_reader :url
 
@@ -85,17 +87,6 @@ module Miteru
       @hostname ||= URI(url).hostname
     end
 
-    def decoded_url
-      @decoded_url ||= URI.decode_www_form_component(url)
-    end
-
-    #
-    # @return [String]
-    #
-    def truncated_url
-      url.truncate(64)
-    end
-
     private
 
     def filename_to_download

data/lib/miteru/notifiers/base.rb

@@ -3,7 +3,7 @@
 module Miteru
   module Notifiers
     class Base < Service
-      def call(website)
+      def call(kit)
         raise NotImplementedError
       end
 

data/lib/miteru/notifiers/slack.rb

@@ -75,15 +75,15 @@ module Miteru
 
     class Slack < Base
       #
-      # Notifiy to Slack
+      # Notify to Slack
       #
-      # @param [Miteru::Website] website
+      # @param [Miteru::Kit] kit
       #
-      def call(website)
+      def call(kit)
         return unless callable?
 
-        attachment = SlackAttachment.new(website.url)
-        notifier.post(text: website.info, attachments: attachment.to_a) if website.kits?
+        attachment = SlackAttachment.new(kit.url)
+        notifier.post(text: kit.basename, attachments: attachment.to_a)
       end
 
       def callable?

data/lib/miteru/notifiers/urlscan.rb

@@ -4,12 +4,12 @@ module Miteru
   module Notifiers
     class UrlScan < Base
       #
-      # @param [Miteru::Website] website
+      # @param [Miteru::Kit] kit
       #
-      def call(website)
+      def call(kit)
         return unless callable?
 
-        website.kits.each { |kit| submit(kit.url, source: website.source) }
+        submit(kit.decoded_url, source: kit.source)
       end
 
       def callable?

data/lib/miteru/orchestrator.rb

@@ -3,24 +3,31 @@
 module Miteru
   class Orchestrator < Service
     def call
-      Miteru.logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?
+      logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?
 
-      if Miteru.sidekiq?
-        non_cached_websites.each do |website|
-          Jobs::CrawleJob.perform_async(website.url, website.source)
-          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
-        end
+      if sidekiq?
+        sidekiq_call
       else
-        Miteru.logger.info("Use #{threads} thread(s).") if verbose?
-        Parallel.each(non_cached_websites, in_threads: threads) do |website|
-          Miteru.logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
+        parallel_call
+      end
+    end
 
-          result = Crawler.result(website)
-          if result.success?
-            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
-          else
-            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
-          end
+    def sidekiq_call
+      non_cached_websites.each do |website|
+        Jobs::CrawleJob.perform_async(website.url, website.source)
+        logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
+      end
+    end
+
+    def parallel_call
+      logger.info("Use #{threads} thread(s).") if verbose?
+      Parallel.each(non_cached_websites, in_threads: threads) do |website|
+        logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
+        result = Crawler.result(website)
+        if result.success?
+          logger.info("Crawler:#{website.truncated_url} succeeded.")
+        else
+          logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
         end
       end
     end
@@ -34,44 +41,33 @@ module Miteru
           result = feed.result
           if result.success?
             websites = result.value!
-            Miteru.logger.info("Feed:#{feed.source} has #{websites.length} websites.") if verbose?
+            logger.info("Feed:#{feed.source} has #{websites.length} websites.") if verbose?
             out << websites
           else
-            Miteru.logger.warn("Feed:#{feed.source} failed - #{result.failure}")
+            logger.warn("Feed:#{feed.source} failed - #{result.failure}")
           end
         end
       end.flatten.uniq(&:url)
     end
 
+    #
+    # @return [Array<Miteru::Website>]
+    #
     def non_cached_websites
-      return websites unless cache?
-
-      websites.reject { |website| cache.cached?(website.url) }
+      @non_cached_websites ||= [].tap do |out|
+        out << if cache?
+          websites.reject { |website| cache.cached?(website.url) }
+        else
+          websites
+        end
+      end.flatten.uniq(&:url)
     end
 
     #
     # @return [Array<Miteru::Feeds::Base>]
     #
     def feeds
-      Miteru.feeds.map(&:new)
-    end
-
-    private
-
-    def cache?
-      Miteru.cache?
-    end
-
-    def cache
-      Miteru.cache
-    end
-
-    def threads
-      Miteru.config.threads
-    end
-
-    def verbose?
-      Miteru.config.verbose
+      @feeds ||= Miteru.feeds.map(&:new)
     end
   end
 end

data/lib/miteru/service.rb

@@ -24,5 +24,49 @@ module Miteru
         new.result(...)
       end
     end
+
+    private
+
+    def auto_download?
+      Miteru.config.auto_download
+    end
+
+    #
+    # @return [SemanticLogger]
+    #
+    def logger
+      Miteru.logger
+    end
+
+    def cache?
+      Miteru.cache?
+    end
+
+    def sidekiq?
+      Miteru.sidekiq?
+    end
+
+    #
+    # @return [Miteru::Cache]
+    #
+    def cache
+      Miteru.cache
+    end
+
+    def threads
+      Miteru.config.threads
+    end
+
+    def verbose?
+      Miteru.config.verbose
+    end
+
+    def cache_prefix
+      Miteru.config.cache_prefix
+    end
+
+    def cache_ex
+      Miteru.config.cache_ex
+    end
   end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.2.0"
+  VERSION = "2.2.2"
 end

data/lib/miteru/website.rb

@@ -4,6 +4,8 @@ require "oga"
 
 module Miteru
   class Website < Service
+    include Concerns::UrlTruncatable
+
     # @return [String]
     attr_reader :url
 
@@ -41,17 +43,6 @@ module Miteru
       (href_links + possible_file_links).compact.uniq
     end
 
-    #
-    # @return [String]
-    #
-    def truncated_url
-      url.truncate(64)
-    end
-
-    def defanged_truncated_url
-      truncated_url.to_s.gsub(".", "[.]")
-    end
-
     def info
       "#{defanged_truncated_url} has #{kits.length} kit(s) (Source: #{source})"
     end

data/lib/miteru.rb

@@ -31,6 +31,7 @@ require "miteru/errors"
 # Concerns
 require "miteru/concerns/database_connectable"
 require "miteru/concerns/error_unwrappable"
+require "miteru/concerns/url_truncatable"
 
 # Core classes
 require "miteru/service"

data/miteru.gemspec

@@ -34,16 +34,16 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.1"
   spec.add_development_dependency "rspec", "~> 3.13"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8"
-  spec.add_development_dependency "standard", "~> 1.33"
+  spec.add_development_dependency "standard", "~> 1.34"
   spec.add_development_dependency "test-prof", "~> 1.3"
   spec.add_development_dependency "vcr", "~> 6.2"
-  spec.add_development_dependency "webmock", "~> 3.19"
+  spec.add_development_dependency "webmock", "~> 3.20"
 
   spec.add_dependency "activerecord", "7.1.3"
   spec.add_dependency "addressable", "2.8.6"
   spec.add_dependency "anyway_config", "2.6.3"
   spec.add_dependency "colorize", "1.1.0"
-  spec.add_dependency "dotenv", "2.8.1"
+  spec.add_dependency "dotenv", "3.0.0"
   spec.add_dependency "down", "5.4.1"
   spec.add_dependency "dry-files", "1.1.0"
   spec.add_dependency "dry-monads", "1.6.0"
@@ -55,7 +55,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack", "3.0.9"
   spec.add_dependency "rack-session", "2.0.0"
   spec.add_dependency "rackup", "2.1.0"
-  spec.add_dependency "redis", "5.0.8"
+  spec.add_dependency "redis", "5.1.0"
   spec.add_dependency "semantic_logger", "4.15.0"
   spec.add_dependency "sentry-ruby", "5.16.1"
   spec.add_dependency "sentry-sidekiq", "5.16.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.2.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-08 00:00:00.000000000 Z
+date: 2024-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.33'
+        version: '1.34'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.33'
+        version: '1.34'
 - !ruby/object:Gem::Dependency
   name: test-prof
   requirement: !ruby/object:Gem::Requirement
@@ -184,14 +184,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.19'
+        version: '3.20'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.19'
+        version: '3.20'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -254,14 +254,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.8.1
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.8.1
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: down
   requirement: !ruby/object:Gem::Requirement
@@ -422,14 +422,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.8
+        version: 5.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.8
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -590,6 +590,7 @@ files:
 - lib/miteru/commands/web.rb
 - lib/miteru/concerns/database_connectable.rb
 - lib/miteru/concerns/error_unwrappable.rb
+- lib/miteru/concerns/url_truncatable.rb
 - lib/miteru/config.rb
 - lib/miteru/crawler.rb
 - lib/miteru/database.rb