miteru 2.3.2 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd3ba5f3cd08b4af7eb269c458323f65214491a6111a4d5277b0102913629a77
-  data.tar.gz: 47542263b5b2e60e5dfadf149d15bac3446f00b5e8aec15e9c83f60629328546
+  metadata.gz: 5d1c74e7ec6c2b56b4fa872ab3d7acb5c69be947a343eed838382bfb1dbc9e16
+  data.tar.gz: e57575a1fd2d5a6a859e045209b30ed1f6b80aef9e930a1146bfb2784c16bcd7
 SHA512:
-  metadata.gz: '028f612c45b87b4b9ec30cffad2168ec6b3983756e9bdd865835a38ba3c9533ccbe6238240f263f9f1cbf460cad8347e3ed1ec00623ef6cdbb1cfa4ebcf5da47'
-  data.tar.gz: bc1a770f94eda268ce90735a80d6b45fd6e2f3a900300de52c1499debf15d5e6f7e705d8e7bee068a5234d0f7cf744f6167a50c61914438be78fb68bed38a8e9
+  metadata.gz: 4da382b9c07c6a9ad1961487335eeea1509b1d82586dddc718c753a94ca63523cb460f948db8fc26dff231fdf151c418d207bcbce4edb435d72e4bd996f7e655
+  data.tar.gz: 475e3848581d32070621018992343a160f1133002f0d240204730d1342be0daddd5048d511424f824b56953935ebb3b04efe37b2ead84d46c85bd57794ce9002

data/.github/workflows/ruby.yml

@@ -35,7 +35,7 @@ jobs:
           --health-retries=3
     strategy:
       matrix:
-        ruby: [3.1, 3.2, 3.3]
+        ruby: [3.2, 3.3]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Ruby
@@ -49,10 +49,8 @@ jobs:
       - name: Test with PostgreSQL
         env:
           MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
-        run: |
-          bundle exec rake
+        run: bundle exec rake
       - name: Test with MySQL
         env:
           MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
-        run: |
-          bundle exec rake
+        run: bundle exec rake

data/lib/miteru/commands/main.rb

@@ -13,7 +13,6 @@ module Miteru
             method_option :directory_traveling, type: :boolean, default: false,
               desc: "Enable or disable directory traveling"
             method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
-            method_option :threads, type: :numeric, desc: "Number of threads to use", default: Parallel.processor_count
             method_option :verbose, type: :boolean, default: true
             desc "execute", "Execute the crawler"
             around :with_db_connection
@@ -22,10 +21,8 @@ module Miteru
                 config.auto_download = options["auto_download"]
                 config.directory_traveling = options["directory_traveling"]
                 config.download_to = options["download_to"]
-                config.threads = options["threads"]
                 config.verbose = options["verbose"]
               end
-
               Orchestrator.call
             end
             default_command :execute

data/lib/miteru/config.rb

@@ -26,14 +26,15 @@ module Miteru
       download_timeout: 60,
       sentry_dsn: nil,
       sentry_trace_sample_rate: 0.25,
-      sidekiq_redis_url: nil,
+      sidekiq_redis_url: "redis://localhost:6379",
       sidekiq_job_retry: 0,
+      sidekiq_batch_size: 50,
+      sidekiq_job_timeout: 600,
       cache_redis_url: nil,
       cache_ex: nil,
       cache_prefix: "miteru:cache",
       slack_channel: "#general",
       slack_webhook_url: nil,
-      threads: Parallel.processor_count,
       urlscan_api_key: nil,
       urlscan_submit_visibility: "public",
       urlscan_date_condition: "date:>now-1h",
@@ -48,11 +49,17 @@ module Miteru
     #   @return [Float]
 
     # @!attribute [r] sidekiq_redis_url
-    #   @return [String, nil]
+    #   @return [String]
 
     # @!attribute [r] sidekiq_job_retry
     #   @return [Integer]
 
+    # @!attribute [r] sidekiq_batch_size
+    #   @return [Integer]
+
+    # @!attribute [r] sidekiq_job_timeout
+    #   @return [Integer]
+
     # @!attribute [r] cache_redis_url
     #   @return [String, nil]
 
@@ -80,9 +87,6 @@ module Miteru
     # @!attribute [rw] download_to
     #   @return [String]
 
-    # @!attribute [rw] threads
-    #   @return [Integer]
-
     # @!attribute [r] cache_redis_url
     #   @return [String, nil]
 

data/lib/miteru/database.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class V2Schema < ActiveRecord::Migration[7.2]
+class V2Schema < ActiveRecord::Migration[8.0]
   def change
     create_table :records, if_not_exists: true do |t|
       t.string :sha256, null: false, index: {unique: true}

data/lib/miteru/orchestrator.rb

@@ -4,32 +4,8 @@ module Miteru
   class Orchestrator < Service
     def call
       logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?
-
-      if sidekiq?
-        sidekiq_call
-      else
-        parallel_call
-      end
-    end
-
-    def sidekiq_call
-      non_cached_websites.each do |website|
-        Jobs::CrawleJob.perform_async(website.url, website.source)
-        logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
-      end
-    end
-
-    def parallel_call
-      logger.info("Use #{threads} thread(s).") if verbose?
-      Parallel.each(non_cached_websites, in_threads: threads) do |website|
-        logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
-        result = Crawler.result(website)
-        if result.success?
-          logger.info("Crawler:#{website.truncated_url} succeeded.")
-        else
-          logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
-        end
-      end
+      array_of_args = non_cached_websites.map { |website| [website.url, website.source] }
+      Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size)
     end
 
     #

data/lib/miteru/sidekiq/application.rb

@@ -6,7 +6,10 @@ require "miteru/sidekiq/jobs"
 
 Sidekiq.configure_server do |config|
   config.redis = {url: Miteru.config.sidekiq_redis_url.to_s}
-  config.default_job_options = {"retry" => Miteru.config.sidekiq_job_retry}
+  config.default_job_options = {
+    retry: Miteru.config.sidekiq_job_retry,
+    expires_in: 0.second
+  }
 end
 
 Sidekiq.configure_client do |config|

data/lib/miteru/sidekiq/jobs.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq"
+require "timeout"
 
 module Miteru
   module Jobs
@@ -14,12 +15,15 @@ module Miteru
       #
       def perform(url, source)
         website = Miteru::Website.new(url, source:)
+
         with_db_connection do
-          result = Crawler.result(website)
-          if result.success?
-            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
-          else
-            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          Timeout.timeout(Miteru.config.sidekiq_job_timeout) do
+            result = Crawler.result(website)
+            if result.success?
+              Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+            else
+              Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+            end
           end
         end
       end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.3.2"
+  VERSION = "3.0.0"
 end

data/lib/miteru.rb

@@ -9,7 +9,6 @@ require "uuidtools"
 # Core 3rd party libraries
 require "colorize"
 require "memo_wise"
-require "parallel"
 require "semantic_logger"
 require "sentry-ruby"
 
@@ -108,13 +107,6 @@ module Miteru
       @cache ||= Cache.new(Miteru.config.cache_redis_url)
     end
 
-    #
-    # @return [Boolean]
-    #
-    def sidekiq?
-      !Miteru.config.sidekiq_redis_url.nil?
-    end
-
     def sentry?
       !Miteru.config.sentry_dsn.nil?
     end

data/miteru.gemspec

@@ -16,6 +16,8 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://github.com/ninoseki/miteru"
   spec.license = "MIT"
 
+  spec.required_ruby_version = ">= 3.2"
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
@@ -30,38 +32,37 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "coveralls_reborn", "~> 0.28"
   spec.add_development_dependency "fuubar", "~> 2.5.1"
   spec.add_development_dependency "mysql2", "~> 0.5.6"
-  spec.add_development_dependency "pg", "~> 1.5.8"
+  spec.add_development_dependency "pg", "~> 1.5.9"
   spec.add_development_dependency "rake", "~> 13.2.1"
   spec.add_development_dependency "rspec", "~> 3.13"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8"
-  spec.add_development_dependency "standard", "~> 1.40.0"
+  spec.add_development_dependency "standard", "~> 1.42.0"
   spec.add_development_dependency "test-prof", "~> 1.4.2"
   spec.add_development_dependency "vcr", "~> 6.3.1"
-  spec.add_development_dependency "webmock", "~> 3.23.1"
+  spec.add_development_dependency "webmock", "~> 3.24.0"
 
-  spec.add_dependency "activerecord", "7.2.1"
+  spec.add_dependency "activerecord", "8.0.0"
   spec.add_dependency "addressable", "2.8.7"
   spec.add_dependency "anyway_config", "2.6.4"
   spec.add_dependency "colorize", "1.1.0"
-  spec.add_dependency "dotenv", "3.1.2"
+  spec.add_dependency "dotenv", "3.1.4"
   spec.add_dependency "down", "5.4.2"
   spec.add_dependency "dry-files", "1.1.0"
   spec.add_dependency "dry-monads", "1.6.0"
   spec.add_dependency "http", "5.2.0"
-  spec.add_dependency "memo_wise", "1.9.0"
+  spec.add_dependency "memo_wise", "1.10.0"
   spec.add_dependency "oga", "3.4"
-  spec.add_dependency "parallel", "1.26.3"
-  spec.add_dependency "puma", "6.4.2"
-  spec.add_dependency "rack", "3.1.7"
+  spec.add_dependency "puma", "6.4.3"
+  spec.add_dependency "rack", "3.1.8"
   spec.add_dependency "rack-session", "2.0.0"
-  spec.add_dependency "rackup", "2.1.0"
+  spec.add_dependency "rackup", "2.2.1"
   spec.add_dependency "redis", "5.3.0"
   spec.add_dependency "semantic_logger", "4.16.1"
-  spec.add_dependency "sentry-ruby", "5.19.0"
-  spec.add_dependency "sentry-sidekiq", "5.19.0"
-  spec.add_dependency "sidekiq", "7.3.2"
+  spec.add_dependency "sentry-ruby", "5.21.0"
+  spec.add_dependency "sentry-sidekiq", "5.21.0"
+  spec.add_dependency "sidekiq", "7.3.5"
   spec.add_dependency "slack-notifier", "2.4.0"
-  spec.add_dependency "sqlite3", "1.7.2"
+  spec.add_dependency "sqlite3", "2.2.0"
   spec.add_dependency "thor", "1.3.2"
   spec.add_dependency "thor-hollaback", "0.2.1"
   spec.add_dependency "uuidtools", "2.2.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 3.0.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-10 00:00:00.000000000 Z
+date: 2024-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.8
+        version: 1.5.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.8
+        version: 1.5.9
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.40.0
+        version: 1.42.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.40.0
+        version: 1.42.0
 - !ruby/object:Gem::Dependency
   name: test-prof
   requirement: !ruby/object:Gem::Requirement
@@ -184,28 +184,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.23.1
+        version: 3.24.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.23.1
+        version: 3.24.0
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.2.1
+        version: 8.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.2.1
+        version: 8.0.0
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -254,14 +254,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.2
+        version: 3.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.2
+        version: 3.1.4
 - !ruby/object:Gem::Dependency
   name: down
   requirement: !ruby/object:Gem::Requirement
@@ -324,14 +324,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.10.0
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
@@ -346,48 +346,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: '3.4'
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.26.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.26.3
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.4.2
+        version: 6.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.4.2
+        version: 6.4.3
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.7
+        version: 3.1.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.7
+        version: 3.1.8
 - !ruby/object:Gem::Dependency
   name: rack-session
   requirement: !ruby/object:Gem::Requirement
@@ -408,14 +394,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.1
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -450,42 +436,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
 - !ruby/object:Gem::Dependency
   name: sentry-sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.3.2
+        version: 7.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.3.2
+        version: 7.3.5
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -506,14 +492,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -632,14 +618,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
+rubygems_version: 3.5.22
 signing_key: 
 specification_version: 4
 summary: A phishing kit collector for scavengers