miteru 2.3.2 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd3ba5f3cd08b4af7eb269c458323f65214491a6111a4d5277b0102913629a77
-  data.tar.gz: 47542263b5b2e60e5dfadf149d15bac3446f00b5e8aec15e9c83f60629328546
+  metadata.gz: a1a624406c34f6f2c4fd913029a408145cd4d24b4151914e9b9e366a48882ade
+  data.tar.gz: 745cb186ac2e294acaefb886f3cfd1e8b2c28d8ec19f9021844c823c9d1114c7
 SHA512:
-  metadata.gz: '028f612c45b87b4b9ec30cffad2168ec6b3983756e9bdd865835a38ba3c9533ccbe6238240f263f9f1cbf460cad8347e3ed1ec00623ef6cdbb1cfa4ebcf5da47'
-  data.tar.gz: bc1a770f94eda268ce90735a80d6b45fd6e2f3a900300de52c1499debf15d5e6f7e705d8e7bee068a5234d0f7cf744f6167a50c61914438be78fb68bed38a8e9
+  metadata.gz: 952e2b6379e775cfe6c8a7d3ec9c837ed1fb96408a3156edac20cc24c308c96263bb2f0b8e7128b4ee1c8607a2f2b77b4989ee4c826c88a5ffc72acc40f40b67
+  data.tar.gz: efed521f1f6fac24b7966e5f5dfbf321f98acfae6e80afa94b7661bfe12be7f4d709e9ad4344301ebe184c8d7a17261e771490d1921972203b2c2a4b25bee306

data/lib/miteru/config.rb

@@ -28,6 +28,8 @@ module Miteru
       sentry_trace_sample_rate: 0.25,
       sidekiq_redis_url: nil,
       sidekiq_job_retry: 0,
+      sidekiq_batch_size: 50,
+      sidekiq_job_timeout: 600,
       cache_redis_url: nil,
       cache_ex: nil,
       cache_prefix: "miteru:cache",
@@ -53,6 +55,12 @@ module Miteru
     # @!attribute [r] sidekiq_job_retry
     #   @return [Integer]
 
+    # @!attribute [r] sidekiq_batch_size
+    #   @return [Integer]
+
+    # @!attribute [r] sidekiq_job_timeout
+    #   @return [Integer]
+
     # @!attribute [r] cache_redis_url
     #   @return [String, nil]
 

data/lib/miteru/orchestrator.rb

@@ -13,10 +13,8 @@ module Miteru
     end
 
     def sidekiq_call
-      non_cached_websites.each do |website|
-        Jobs::CrawleJob.perform_async(website.url, website.source)
-        logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
-      end
+      array_of_args = non_cached_websites.map { |website| [website.url, website.source] }
+      Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size)
     end
 
     def parallel_call

data/lib/miteru/sidekiq/application.rb

@@ -6,7 +6,10 @@ require "miteru/sidekiq/jobs"
 
 Sidekiq.configure_server do |config|
   config.redis = {url: Miteru.config.sidekiq_redis_url.to_s}
-  config.default_job_options = {"retry" => Miteru.config.sidekiq_job_retry}
+  config.default_job_options = {
+    retry: Miteru.config.sidekiq_job_retry,
+    expires_in: 0.second
+  }
 end
 
 Sidekiq.configure_client do |config|

data/lib/miteru/sidekiq/jobs.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq"
+require "timeout"
 
 module Miteru
   module Jobs
@@ -14,12 +15,15 @@ module Miteru
       #
       def perform(url, source)
         website = Miteru::Website.new(url, source:)
+
         with_db_connection do
-          result = Crawler.result(website)
-          if result.success?
-            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
-          else
-            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          Timeout.timeout(Miteru.config.sidekiq_job_timeout) do
+            result = Crawler.result(website)
+            if result.success?
+              Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+            else
+              Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+            end
           end
         end
       end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.3.2"
+  VERSION = "2.4.0"
 end

data/miteru.gemspec

@@ -30,20 +30,20 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "coveralls_reborn", "~> 0.28"
   spec.add_development_dependency "fuubar", "~> 2.5.1"
   spec.add_development_dependency "mysql2", "~> 0.5.6"
-  spec.add_development_dependency "pg", "~> 1.5.8"
+  spec.add_development_dependency "pg", "~> 1.5.9"
   spec.add_development_dependency "rake", "~> 13.2.1"
   spec.add_development_dependency "rspec", "~> 3.13"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8"
-  spec.add_development_dependency "standard", "~> 1.40.0"
+  spec.add_development_dependency "standard", "~> 1.41.1"
   spec.add_development_dependency "test-prof", "~> 1.4.2"
   spec.add_development_dependency "vcr", "~> 6.3.1"
-  spec.add_development_dependency "webmock", "~> 3.23.1"
+  spec.add_development_dependency "webmock", "~> 3.24.0"
 
-  spec.add_dependency "activerecord", "7.2.1"
+  spec.add_dependency "activerecord", "7.2.2"
   spec.add_dependency "addressable", "2.8.7"
   spec.add_dependency "anyway_config", "2.6.4"
   spec.add_dependency "colorize", "1.1.0"
-  spec.add_dependency "dotenv", "3.1.2"
+  spec.add_dependency "dotenv", "3.1.4"
   spec.add_dependency "down", "5.4.2"
   spec.add_dependency "dry-files", "1.1.0"
   spec.add_dependency "dry-monads", "1.6.0"
@@ -51,15 +51,15 @@ Gem::Specification.new do |spec|
   spec.add_dependency "memo_wise", "1.9.0"
   spec.add_dependency "oga", "3.4"
   spec.add_dependency "parallel", "1.26.3"
-  spec.add_dependency "puma", "6.4.2"
-  spec.add_dependency "rack", "3.1.7"
+  spec.add_dependency "puma", "6.4.3"
+  spec.add_dependency "rack", "3.1.8"
   spec.add_dependency "rack-session", "2.0.0"
-  spec.add_dependency "rackup", "2.1.0"
+  spec.add_dependency "rackup", "2.2.0"
   spec.add_dependency "redis", "5.3.0"
   spec.add_dependency "semantic_logger", "4.16.1"
-  spec.add_dependency "sentry-ruby", "5.19.0"
-  spec.add_dependency "sentry-sidekiq", "5.19.0"
-  spec.add_dependency "sidekiq", "7.3.2"
+  spec.add_dependency "sentry-ruby", "5.21.0"
+  spec.add_dependency "sentry-sidekiq", "5.21.0"
+  spec.add_dependency "sidekiq", "7.3.4"
   spec.add_dependency "slack-notifier", "2.4.0"
   spec.add_dependency "sqlite3", "1.7.2"
   spec.add_dependency "thor", "1.3.2"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.4.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-10 00:00:00.000000000 Z
+date: 2024-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.8
+        version: 1.5.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.8
+        version: 1.5.9
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.40.0
+        version: 1.41.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.40.0
+        version: 1.41.1
 - !ruby/object:Gem::Dependency
   name: test-prof
   requirement: !ruby/object:Gem::Requirement
@@ -184,28 +184,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.23.1
+        version: 3.24.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.23.1
+        version: 3.24.0
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.2.1
+        version: 7.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.2.1
+        version: 7.2.2
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -254,14 +254,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.2
+        version: 3.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.2
+        version: 3.1.4
 - !ruby/object:Gem::Dependency
   name: down
   requirement: !ruby/object:Gem::Requirement
@@ -366,28 +366,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.4.2
+        version: 6.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.4.2
+        version: 6.4.3
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.7
+        version: 3.1.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.7
+        version: 3.1.8
 - !ruby/object:Gem::Dependency
   name: rack-session
   requirement: !ruby/object:Gem::Requirement
@@ -408,14 +408,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -450,42 +450,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
 - !ruby/object:Gem::Dependency
   name: sentry-sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.19.0
+        version: 5.21.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.3.2
+        version: 7.3.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.3.2
+        version: 7.3.4
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement