miteru 2.3.1 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. checksums.yaml +4 -4
  2. data/docker-compose.yml +0 -1
  3. data/exe/miteru +1 -1
  4. data/lefthook.yml +2 -1
  5. data/lib/miteru/commands/sidekiq.rb +1 -1
  6. data/lib/miteru/config.rb +49 -2
  7. data/lib/miteru/feeds/ayashige.rb +1 -1
  8. data/lib/miteru/feeds/phishing_database.rb +1 -1
  9. data/lib/miteru/feeds/tweetfeed.rb +1 -1
  10. data/lib/miteru/feeds/urlscan.rb +1 -1
  11. data/lib/miteru/feeds/urlscan_pro.rb +1 -1
  12. data/lib/miteru/orchestrator.rb +2 -4
  13. data/lib/miteru/sidekiq/application.rb +4 -0
  14. data/lib/miteru/sidekiq/jobs.rb +9 -5
  15. data/lib/miteru/version.rb +1 -1
  16. data/miteru.gemspec +12 -12
  17. metadata +30 -30
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 15c49f2c418826deed2d6862018c10582649a91a26ca41218fba15bd7f80c11a
4
- data.tar.gz: 120181d2d55f7ebb5d7aa500624ff4a40827c5f3604f391b391b3b7ff7298ab4
3
+ metadata.gz: a1a624406c34f6f2c4fd913029a408145cd4d24b4151914e9b9e366a48882ade
4
+ data.tar.gz: 745cb186ac2e294acaefb886f3cfd1e8b2c28d8ec19f9021844c823c9d1114c7
5
5
  SHA512:
6
- metadata.gz: 317e72123d90abcc99ab24e0b7ce36c3dd0fbb27310c0a1382614420a17bb70440e7a21f657d780508dfddd3727679eef2f566afde64fb4ecaff6c552f683c3d
7
- data.tar.gz: 2ec13073b8711e381e97ed35882eb8342149289f277f0a4769c8277eb2bac57db262a3868ae65b3d0dc667466d4d563ed109974f8d4effc32bbba792b08f915e
6
+ metadata.gz: 952e2b6379e775cfe6c8a7d3ec9c837ed1fb96408a3156edac20cc24c308c96263bb2f0b8e7128b4ee1c8607a2f2b77b4989ee4c826c88a5ffc72acc40f40b67
7
+ data.tar.gz: efed521f1f6fac24b7966e5f5dfbf321f98acfae6e80afa94b7661bfe12be7f4d709e9ad4344301ebe184c8d7a17261e771490d1921972203b2c2a4b25bee306
data/docker-compose.yml CHANGED
@@ -1,4 +1,3 @@
1
- version: "3"
2
1
  services:
3
2
  redis:
4
3
  image: "redis/redis-stack:6.2.6-v10"
data/exe/miteru CHANGED
@@ -3,7 +3,7 @@
3
3
 
4
4
  $LOAD_PATH.unshift("#{__dir__}/../lib")
5
5
 
6
- require 'miteru'
6
+ require "miteru"
7
7
 
8
8
  ARGV.unshift(Miteru::CLI::App.default_task) unless Miteru::CLI::App.all_tasks.key?(ARGV[0])
9
9
  Miteru::CLI::App.start(ARGV)
data/lefthook.yml CHANGED
@@ -5,5 +5,6 @@ pre-commit:
5
5
  run: bundle exec standardrb --fix {staged_files}
6
6
  stage_fixed: true
7
7
  actionlint:
8
- glob: ".github/workflows/*.yaml"
8
+ root: ".github/workflows"
9
+ glob: "*.{yaml,yml}"
9
10
  run: actionlint
data/lib/miteru/commands/sidekiq.rb CHANGED
@@ -10,7 +10,7 @@ module Miteru
10
10
  def included(thor)
11
11
  thor.class_eval do
12
12
  desc "sidekiq", "Start Sidekiq"
13
- method_option :env, type: :string, default: "production", desc: "Environment"
13
+ method_option :env, type: :string, default: "production", desc: "Environment", aliases: "-e"
14
14
  method_option :concurrency, type: :numeric, default: 5, desc: "Sidekiq concurrency", aliases: "-c"
15
15
  def sidekiq
16
16
  require "sidekiq/cli"
data/lib/miteru/config.rb CHANGED
@@ -14,14 +14,22 @@ module Miteru
14
14
  download_to: "/tmp",
15
15
  file_max_size: 1024 * 1024 * 100,
16
16
  file_extensions: [".zip", ".rar", ".7z", ".tar", ".gz"],
17
- file_mime_types: ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar",
18
- "application/gzip"],
17
+ file_mime_types: [
18
+ "application/zip",
19
+ "application/vnd.rar",
20
+ "application/x-7z-compressed",
21
+ "application/x-tar",
22
+ "application/gzip"
23
+ ],
19
24
  api_timeout: 60,
20
25
  http_timeout: 60,
21
26
  download_timeout: 60,
22
27
  sentry_dsn: nil,
23
28
  sentry_trace_sample_rate: 0.25,
24
29
  sidekiq_redis_url: nil,
30
+ sidekiq_job_retry: 0,
31
+ sidekiq_batch_size: 50,
32
+ sidekiq_job_timeout: 600,
25
33
  cache_redis_url: nil,
26
34
  cache_ex: nil,
27
35
  cache_prefix: "miteru:cache",
@@ -44,6 +52,45 @@ module Miteru
44
52
  # @!attribute [r] sidekiq_redis_url
45
53
  # @return [String, nil]
46
54
 
55
+ # @!attribute [r] sidekiq_job_retry
56
+ # @return [Integer]
57
+
58
+ # @!attribute [r] sidekiq_batch_size
59
+ # @return [Integer]
60
+
61
+ # @!attribute [r] sidekiq_job_timeout
62
+ # @return [Integer]
63
+
64
+ # @!attribute [r] cache_redis_url
65
+ # @return [String, nil]
66
+
67
+ # @!attribute [r] cache_ex
68
+ # @return [Integer, nil]
69
+
70
+ # @!attribute [r] cache_prefix
71
+ # @return [String]
72
+
73
+ # @!attribute [r] http_timeout
74
+ # @return [Integer]
75
+
76
+ # @!attribute [r] api_timeout
77
+ # @return [Integer]
78
+
79
+ # @!attribute [r] download_timeout
80
+ # @return [Integer]
81
+
82
+ # @!attribute [rw] auto_download
83
+ # @return [Boolean]
84
+
85
+ # @!attribute [rw] directory_traveling
86
+ # @return [Boolean]
87
+
88
+ # @!attribute [rw] download_to
89
+ # @return [String]
90
+
91
+ # @!attribute [rw] threads
92
+ # @return [Integer]
93
+
47
94
  # @!attribute [r] cache_redis_url
48
95
  # @return [String, nil]
49
96
 
data/lib/miteru/feeds/ayashige.rb CHANGED
@@ -4,7 +4,7 @@ module Miteru
4
4
  class Feeds
5
5
  class Ayashige < Base
6
6
  def initialize(base_url = "https://ayashige.herokuapp.com")
7
- super(base_url)
7
+ super
8
8
  end
9
9
 
10
10
  def urls
data/lib/miteru/feeds/phishing_database.rb CHANGED
@@ -4,7 +4,7 @@ module Miteru
4
4
  class Feeds
5
5
  class PhishingDatabase < Base
6
6
  def initialize(base_url = "https://raw.githubusercontent.com")
7
- super(base_url)
7
+ super
8
8
  end
9
9
 
10
10
  def urls
data/lib/miteru/feeds/tweetfeed.rb CHANGED
@@ -4,7 +4,7 @@ module Miteru
4
4
  class Feeds
5
5
  class TweetFeed < Base
6
6
  def initialize(base_url = "https://api.tweetfeed.live")
7
- super(base_url)
7
+ super
8
8
  end
9
9
 
10
10
  def urls
data/lib/miteru/feeds/urlscan.rb CHANGED
@@ -7,7 +7,7 @@ module Miteru
7
7
  # @param [String] base_url
8
8
  #
9
9
  def initialize(base_url = "https://urlscan.io")
10
- super(base_url)
10
+ super
11
11
 
12
12
  @headers = {"api-key": api_key}
13
13
  end
data/lib/miteru/feeds/urlscan_pro.rb CHANGED
@@ -7,7 +7,7 @@ module Miteru
7
7
  # @param [String] base_url
8
8
  #
9
9
  def initialize(base_url = "https://urlscan.io")
10
- super(base_url)
10
+ super
11
11
 
12
12
  @headers = {"api-key": api_key}
13
13
  end
data/lib/miteru/orchestrator.rb CHANGED
@@ -13,10 +13,8 @@ module Miteru
13
13
  end
14
14
 
15
15
  def sidekiq_call
16
- non_cached_websites.each do |website|
17
- Jobs::CrawleJob.perform_async(website.url, website.source)
18
- logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
19
- end
16
+ array_of_args = non_cached_websites.map { |website| [website.url, website.source] }
17
+ Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size)
20
18
  end
21
19
 
22
20
  def parallel_call
data/lib/miteru/sidekiq/application.rb CHANGED
@@ -6,6 +6,10 @@ require "miteru/sidekiq/jobs"
6
6
 
7
7
  Sidekiq.configure_server do |config|
8
8
  config.redis = {url: Miteru.config.sidekiq_redis_url.to_s}
9
+ config.default_job_options = {
10
+ retry: Miteru.config.sidekiq_job_retry,
11
+ expires_in: 0.second
12
+ }
9
13
  end
10
14
 
11
15
  Sidekiq.configure_client do |config|
data/lib/miteru/sidekiq/jobs.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq"
4
+ require "timeout"
4
5
 
5
6
  module Miteru
6
7
  module Jobs
@@ -14,12 +15,15 @@ module Miteru
14
15
  #
15
16
  def perform(url, source)
16
17
  website = Miteru::Website.new(url, source:)
18
+
17
19
  with_db_connection do
18
- result = Crawler.result(website)
19
- if result.success?
20
- Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
21
- else
22
- Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
20
+ Timeout.timeout(Miteru.config.sidekiq_job_timeout) do
21
+ result = Crawler.result(website)
22
+ if result.success?
23
+ Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
24
+ else
25
+ Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
26
+ end
23
27
  end
24
28
  end
25
29
  end
data/lib/miteru/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "2.3.1"
4
+ VERSION = "2.4.0"
5
5
  end
data/miteru.gemspec CHANGED
@@ -30,20 +30,20 @@ Gem::Specification.new do |spec|
30
30
  spec.add_development_dependency "coveralls_reborn", "~> 0.28"
31
31
  spec.add_development_dependency "fuubar", "~> 2.5.1"
32
32
  spec.add_development_dependency "mysql2", "~> 0.5.6"
33
- spec.add_development_dependency "pg", "~> 1.5.7"
33
+ spec.add_development_dependency "pg", "~> 1.5.9"
34
34
  spec.add_development_dependency "rake", "~> 13.2.1"
35
35
  spec.add_development_dependency "rspec", "~> 3.13"
36
36
  spec.add_development_dependency "simplecov-lcov", "~> 0.8"
37
- spec.add_development_dependency "standard", "~> 1.40.0"
37
+ spec.add_development_dependency "standard", "~> 1.41.1"
38
38
  spec.add_development_dependency "test-prof", "~> 1.4.2"
39
39
  spec.add_development_dependency "vcr", "~> 6.3.1"
40
- spec.add_development_dependency "webmock", "~> 3.23.1"
40
+ spec.add_development_dependency "webmock", "~> 3.24.0"
41
41
 
42
- spec.add_dependency "activerecord", "7.2.1"
42
+ spec.add_dependency "activerecord", "7.2.2"
43
43
  spec.add_dependency "addressable", "2.8.7"
44
44
  spec.add_dependency "anyway_config", "2.6.4"
45
45
  spec.add_dependency "colorize", "1.1.0"
46
- spec.add_dependency "dotenv", "3.1.2"
46
+ spec.add_dependency "dotenv", "3.1.4"
47
47
  spec.add_dependency "down", "5.4.2"
48
48
  spec.add_dependency "dry-files", "1.1.0"
49
49
  spec.add_dependency "dry-monads", "1.6.0"
@@ -51,15 +51,15 @@ Gem::Specification.new do |spec|
51
51
  spec.add_dependency "memo_wise", "1.9.0"
52
52
  spec.add_dependency "oga", "3.4"
53
53
  spec.add_dependency "parallel", "1.26.3"
54
- spec.add_dependency "puma", "6.4.2"
55
- spec.add_dependency "rack", "3.1.7"
54
+ spec.add_dependency "puma", "6.4.3"
55
+ spec.add_dependency "rack", "3.1.8"
56
56
  spec.add_dependency "rack-session", "2.0.0"
57
- spec.add_dependency "rackup", "2.1.0"
57
+ spec.add_dependency "rackup", "2.2.0"
58
58
  spec.add_dependency "redis", "5.3.0"
59
- spec.add_dependency "semantic_logger", "4.16.0"
60
- spec.add_dependency "sentry-ruby", "5.19.0"
61
- spec.add_dependency "sentry-sidekiq", "5.19.0"
62
- spec.add_dependency "sidekiq", "7.3.2"
59
+ spec.add_dependency "semantic_logger", "4.16.1"
60
+ spec.add_dependency "sentry-ruby", "5.21.0"
61
+ spec.add_dependency "sentry-sidekiq", "5.21.0"
62
+ spec.add_dependency "sidekiq", "7.3.4"
63
63
  spec.add_dependency "slack-notifier", "2.4.0"
64
64
  spec.add_dependency "sqlite3", "1.7.2"
65
65
  spec.add_dependency "thor", "1.3.2"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.1
4
+ version: 2.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-09-05 00:00:00.000000000 Z
11
+ date: 2024-11-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -86,14 +86,14 @@ dependencies:
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: 1.5.7
89
+ version: 1.5.9
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: 1.5.7
96
+ version: 1.5.9
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: rake
99
99
  requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: 1.40.0
145
+ version: 1.41.1
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: 1.40.0
152
+ version: 1.41.1
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: test-prof
155
155
  requirement: !ruby/object:Gem::Requirement
@@ -184,28 +184,28 @@ dependencies:
184
184
  requirements:
185
185
  - - "~>"
186
186
  - !ruby/object:Gem::Version
187
- version: 3.23.1
187
+ version: 3.24.0
188
188
  type: :development
189
189
  prerelease: false
190
190
  version_requirements: !ruby/object:Gem::Requirement
191
191
  requirements:
192
192
  - - "~>"
193
193
  - !ruby/object:Gem::Version
194
- version: 3.23.1
194
+ version: 3.24.0
195
195
  - !ruby/object:Gem::Dependency
196
196
  name: activerecord
197
197
  requirement: !ruby/object:Gem::Requirement
198
198
  requirements:
199
199
  - - '='
200
200
  - !ruby/object:Gem::Version
201
- version: 7.2.1
201
+ version: 7.2.2
202
202
  type: :runtime
203
203
  prerelease: false
204
204
  version_requirements: !ruby/object:Gem::Requirement
205
205
  requirements:
206
206
  - - '='
207
207
  - !ruby/object:Gem::Version
208
- version: 7.2.1
208
+ version: 7.2.2
209
209
  - !ruby/object:Gem::Dependency
210
210
  name: addressable
211
211
  requirement: !ruby/object:Gem::Requirement
@@ -254,14 +254,14 @@ dependencies:
254
254
  requirements:
255
255
  - - '='
256
256
  - !ruby/object:Gem::Version
257
- version: 3.1.2
257
+ version: 3.1.4
258
258
  type: :runtime
259
259
  prerelease: false
260
260
  version_requirements: !ruby/object:Gem::Requirement
261
261
  requirements:
262
262
  - - '='
263
263
  - !ruby/object:Gem::Version
264
- version: 3.1.2
264
+ version: 3.1.4
265
265
  - !ruby/object:Gem::Dependency
266
266
  name: down
267
267
  requirement: !ruby/object:Gem::Requirement
@@ -366,28 +366,28 @@ dependencies:
366
366
  requirements:
367
367
  - - '='
368
368
  - !ruby/object:Gem::Version
369
- version: 6.4.2
369
+ version: 6.4.3
370
370
  type: :runtime
371
371
  prerelease: false
372
372
  version_requirements: !ruby/object:Gem::Requirement
373
373
  requirements:
374
374
  - - '='
375
375
  - !ruby/object:Gem::Version
376
- version: 6.4.2
376
+ version: 6.4.3
377
377
  - !ruby/object:Gem::Dependency
378
378
  name: rack
379
379
  requirement: !ruby/object:Gem::Requirement
380
380
  requirements:
381
381
  - - '='
382
382
  - !ruby/object:Gem::Version
383
- version: 3.1.7
383
+ version: 3.1.8
384
384
  type: :runtime
385
385
  prerelease: false
386
386
  version_requirements: !ruby/object:Gem::Requirement
387
387
  requirements:
388
388
  - - '='
389
389
  - !ruby/object:Gem::Version
390
- version: 3.1.7
390
+ version: 3.1.8
391
391
  - !ruby/object:Gem::Dependency
392
392
  name: rack-session
393
393
  requirement: !ruby/object:Gem::Requirement
@@ -408,14 +408,14 @@ dependencies:
408
408
  requirements:
409
409
  - - '='
410
410
  - !ruby/object:Gem::Version
411
- version: 2.1.0
411
+ version: 2.2.0
412
412
  type: :runtime
413
413
  prerelease: false
414
414
  version_requirements: !ruby/object:Gem::Requirement
415
415
  requirements:
416
416
  - - '='
417
417
  - !ruby/object:Gem::Version
418
- version: 2.1.0
418
+ version: 2.2.0
419
419
  - !ruby/object:Gem::Dependency
420
420
  name: redis
421
421
  requirement: !ruby/object:Gem::Requirement
@@ -436,56 +436,56 @@ dependencies:
436
436
  requirements:
437
437
  - - '='
438
438
  - !ruby/object:Gem::Version
439
- version: 4.16.0
439
+ version: 4.16.1
440
440
  type: :runtime
441
441
  prerelease: false
442
442
  version_requirements: !ruby/object:Gem::Requirement
443
443
  requirements:
444
444
  - - '='
445
445
  - !ruby/object:Gem::Version
446
- version: 4.16.0
446
+ version: 4.16.1
447
447
  - !ruby/object:Gem::Dependency
448
448
  name: sentry-ruby
449
449
  requirement: !ruby/object:Gem::Requirement
450
450
  requirements:
451
451
  - - '='
452
452
  - !ruby/object:Gem::Version
453
- version: 5.19.0
453
+ version: 5.21.0
454
454
  type: :runtime
455
455
  prerelease: false
456
456
  version_requirements: !ruby/object:Gem::Requirement
457
457
  requirements:
458
458
  - - '='
459
459
  - !ruby/object:Gem::Version
460
- version: 5.19.0
460
+ version: 5.21.0
461
461
  - !ruby/object:Gem::Dependency
462
462
  name: sentry-sidekiq
463
463
  requirement: !ruby/object:Gem::Requirement
464
464
  requirements:
465
465
  - - '='
466
466
  - !ruby/object:Gem::Version
467
- version: 5.19.0
467
+ version: 5.21.0
468
468
  type: :runtime
469
469
  prerelease: false
470
470
  version_requirements: !ruby/object:Gem::Requirement
471
471
  requirements:
472
472
  - - '='
473
473
  - !ruby/object:Gem::Version
474
- version: 5.19.0
474
+ version: 5.21.0
475
475
  - !ruby/object:Gem::Dependency
476
476
  name: sidekiq
477
477
  requirement: !ruby/object:Gem::Requirement
478
478
  requirements:
479
479
  - - '='
480
480
  - !ruby/object:Gem::Version
481
- version: 7.3.2
481
+ version: 7.3.4
482
482
  type: :runtime
483
483
  prerelease: false
484
484
  version_requirements: !ruby/object:Gem::Requirement
485
485
  requirements:
486
486
  - - '='
487
487
  - !ruby/object:Gem::Version
488
- version: 7.3.2
488
+ version: 7.3.4
489
489
  - !ruby/object:Gem::Dependency
490
490
  name: slack-notifier
491
491
  requirement: !ruby/object:Gem::Requirement
@@ -624,7 +624,7 @@ licenses:
624
624
  - MIT
625
625
  metadata:
626
626
  rubygems_mfa_required: 'true'
627
- post_install_message:
627
+ post_install_message:
628
628
  rdoc_options: []
629
629
  require_paths:
630
630
  - lib
@@ -639,8 +639,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
639
639
  - !ruby/object:Gem::Version
640
640
  version: '0'
641
641
  requirements: []
642
- rubygems_version: 3.5.6
643
- signing_key:
642
+ rubygems_version: 3.5.16
643
+ signing_key:
644
644
  specification_version: 4
645
645
  summary: A phishing kit collector for scavengers
646
646
  test_files: []