miteru 2.4.0 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1a624406c34f6f2c4fd913029a408145cd4d24b4151914e9b9e366a48882ade
-  data.tar.gz: 745cb186ac2e294acaefb886f3cfd1e8b2c28d8ec19f9021844c823c9d1114c7
+  metadata.gz: 3a030ed3385f8c667ad95514cd0ca672c2682935afda529aa3ba97bec34da2f4
+  data.tar.gz: d3a79e73d68cf139bba9e1b692e990b87e1cde695756177c1d4c12e52b2a76db
 SHA512:
-  metadata.gz: 952e2b6379e775cfe6c8a7d3ec9c837ed1fb96408a3156edac20cc24c308c96263bb2f0b8e7128b4ee1c8607a2f2b77b4989ee4c826c88a5ffc72acc40f40b67
-  data.tar.gz: efed521f1f6fac24b7966e5f5dfbf321f98acfae6e80afa94b7661bfe12be7f4d709e9ad4344301ebe184c8d7a17261e771490d1921972203b2c2a4b25bee306
+  metadata.gz: c364bb3db19ea3ef41f14ab18b71ec6821e5bfb22e02050a0c11aca8d15abd908f2aa69f41809e22cb46d7b858936d09aa3e202a83269ad8afb6abebbe558c91
+  data.tar.gz: 9862e99d8ed38664c6b1b1ecd0456f517e189b646821b7338e5d71aa69fdb748da5e3be891bd1d5e9f3fcd69f06a515fd93ec92598def899e25cde064cfbfa16

data/.github/workflows/ruby.yml

@@ -35,7 +35,7 @@ jobs:
           --health-retries=3
     strategy:
       matrix:
-        ruby: [3.1, 3.2, 3.3]
+        ruby: [3.2, 3.3]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Ruby
@@ -49,10 +49,8 @@ jobs:
       - name: Test with PostgreSQL
         env:
           MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
-        run: |
-          bundle exec rake
+        run: bundle exec rake
       - name: Test with MySQL
         env:
           MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
-        run: |
-          bundle exec rake
+        run: bundle exec rake

data/lib/miteru/commands/main.rb

@@ -13,7 +13,6 @@ module Miteru
             method_option :directory_traveling, type: :boolean, default: false,
               desc: "Enable or disable directory traveling"
             method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
-            method_option :threads, type: :numeric, desc: "Number of threads to use", default: Parallel.processor_count
             method_option :verbose, type: :boolean, default: true
             desc "execute", "Execute the crawler"
             around :with_db_connection
@@ -22,10 +21,8 @@ module Miteru
                 config.auto_download = options["auto_download"]
                 config.directory_traveling = options["directory_traveling"]
                 config.download_to = options["download_to"]
-                config.threads = options["threads"]
                 config.verbose = options["verbose"]
               end
-
               Orchestrator.call
             end
             default_command :execute

data/lib/miteru/config.rb

@@ -26,7 +26,7 @@ module Miteru
       download_timeout: 60,
       sentry_dsn: nil,
       sentry_trace_sample_rate: 0.25,
-      sidekiq_redis_url: nil,
+      sidekiq_redis_url: "redis://localhost:6379",
       sidekiq_job_retry: 0,
       sidekiq_batch_size: 50,
       sidekiq_job_timeout: 600,
@@ -35,7 +35,6 @@ module Miteru
       cache_prefix: "miteru:cache",
       slack_channel: "#general",
       slack_webhook_url: nil,
-      threads: Parallel.processor_count,
       urlscan_api_key: nil,
       urlscan_submit_visibility: "public",
       urlscan_date_condition: "date:>now-1h",
@@ -50,7 +49,7 @@ module Miteru
     #   @return [Float]
 
     # @!attribute [r] sidekiq_redis_url
-    #   @return [String, nil]
+    #   @return [String]
 
     # @!attribute [r] sidekiq_job_retry
     #   @return [Integer]
@@ -88,9 +87,6 @@ module Miteru
     # @!attribute [rw] download_to
     #   @return [String]
 
-    # @!attribute [rw] threads
-    #   @return [Integer]
-
     # @!attribute [r] cache_redis_url
     #   @return [String, nil]
 

data/lib/miteru/database.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class V2Schema < ActiveRecord::Migration[7.2]
+class V2Schema < ActiveRecord::Migration[8.0]
   def change
     create_table :records, if_not_exists: true do |t|
       t.string :sha256, null: false, index: {unique: true}

data/lib/miteru/orchestrator.rb

@@ -4,32 +4,10 @@ module Miteru
   class Orchestrator < Service
     def call
       logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?
-
-      if sidekiq?
-        sidekiq_call
-      else
-        parallel_call
-      end
-    end
-
-    def sidekiq_call
       array_of_args = non_cached_websites.map { |website| [website.url, website.source] }
       Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size)
     end
 
-    def parallel_call
-      logger.info("Use #{threads} thread(s).") if verbose?
-      Parallel.each(non_cached_websites, in_threads: threads) do |website|
-        logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
-        result = Crawler.result(website)
-        if result.success?
-          logger.info("Crawler:#{website.truncated_url} succeeded.")
-        else
-          logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
-        end
-      end
-    end
-
     #
     # @return [Array<Miteru::Website>]
     #

data/lib/miteru/service.rb

@@ -42,10 +42,6 @@ module Miteru
       Miteru.cache?
     end
 
-    def sidekiq?
-      Miteru.sidekiq?
-    end
-
     #
     # @return [Miteru::Cache]
     #

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.4.0"
+  VERSION = "3.0.1"
 end

data/lib/miteru.rb

@@ -9,7 +9,6 @@ require "uuidtools"
 # Core 3rd party libraries
 require "colorize"
 require "memo_wise"
-require "parallel"
 require "semantic_logger"
 require "sentry-ruby"
 
@@ -55,8 +54,6 @@ module Miteru
     # @return [SematicLogger]
     #
     def logger
-      SemanticLogger.sync! unless sidekiq?
-
       SemanticLogger.default_level = :info
       SemanticLogger.add_appender(io: $stderr, formatter: :color)
       SemanticLogger["Miteru"]
@@ -108,13 +105,6 @@ module Miteru
       @cache ||= Cache.new(Miteru.config.cache_redis_url)
     end
 
-    #
-    # @return [Boolean]
-    #
-    def sidekiq?
-      !Miteru.config.sidekiq_redis_url.nil?
-    end
-
     def sentry?
       !Miteru.config.sentry_dsn.nil?
     end

data/miteru.gemspec

@@ -16,6 +16,8 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://github.com/ninoseki/miteru"
   spec.license = "MIT"
 
+  spec.required_ruby_version = ">= 3.2"
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
@@ -34,12 +36,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.2.1"
   spec.add_development_dependency "rspec", "~> 3.13"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8"
-  spec.add_development_dependency "standard", "~> 1.41.1"
+  spec.add_development_dependency "standard", "~> 1.42.1"
   spec.add_development_dependency "test-prof", "~> 1.4.2"
   spec.add_development_dependency "vcr", "~> 6.3.1"
   spec.add_development_dependency "webmock", "~> 3.24.0"
 
-  spec.add_dependency "activerecord", "7.2.2"
+  spec.add_dependency "activerecord", "8.0.0"
   spec.add_dependency "addressable", "2.8.7"
   spec.add_dependency "anyway_config", "2.6.4"
   spec.add_dependency "colorize", "1.1.0"
@@ -48,20 +50,19 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-files", "1.1.0"
   spec.add_dependency "dry-monads", "1.6.0"
   spec.add_dependency "http", "5.2.0"
-  spec.add_dependency "memo_wise", "1.9.0"
+  spec.add_dependency "memo_wise", "1.10.0"
   spec.add_dependency "oga", "3.4"
-  spec.add_dependency "parallel", "1.26.3"
-  spec.add_dependency "puma", "6.4.3"
+  spec.add_dependency "puma", "6.5.0"
   spec.add_dependency "rack", "3.1.8"
   spec.add_dependency "rack-session", "2.0.0"
-  spec.add_dependency "rackup", "2.2.0"
+  spec.add_dependency "rackup", "2.2.1"
   spec.add_dependency "redis", "5.3.0"
   spec.add_dependency "semantic_logger", "4.16.1"
-  spec.add_dependency "sentry-ruby", "5.21.0"
-  spec.add_dependency "sentry-sidekiq", "5.21.0"
-  spec.add_dependency "sidekiq", "7.3.4"
+  spec.add_dependency "sentry-ruby", "5.22.0"
+  spec.add_dependency "sentry-sidekiq", "5.22.0"
+  spec.add_dependency "sidekiq", "7.3.6"
   spec.add_dependency "slack-notifier", "2.4.0"
-  spec.add_dependency "sqlite3", "1.7.2"
+  spec.add_dependency "sqlite3", "2.4.1"
   spec.add_dependency "thor", "1.3.2"
   spec.add_dependency "thor-hollaback", "0.2.1"
   spec.add_dependency "uuidtools", "2.2.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 3.0.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-11-04 00:00:00.000000000 Z
+date: 2024-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.41.1
+        version: 1.42.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.41.1
+        version: 1.42.1
 - !ruby/object:Gem::Dependency
   name: test-prof
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.2.2
+        version: 8.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.2.2
+        version: 8.0.0
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -324,14 +324,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.10.0
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
@@ -346,34 +346,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: '3.4'
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.26.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.26.3
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.4.3
+        version: 6.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.4.3
+        version: 6.5.0
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -408,14 +394,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 2.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 2.2.1
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -450,42 +436,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.21.0
+        version: 5.22.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.21.0
+        version: 5.22.0
 - !ruby/object:Gem::Dependency
   name: sentry-sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.21.0
+        version: 5.22.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.21.0
+        version: 5.22.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.3.4
+        version: 7.3.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.3.4
+        version: 7.3.6
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -506,14 +492,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: 2.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: 2.4.1
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -632,14 +618,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
+rubygems_version: 3.5.22
 signing_key: 
 specification_version: 4
 summary: A phishing kit collector for scavengers