miteru 2.3.1 → 2.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (17) hide show
  1. checksums.yaml +4 -4
  2. data/docker-compose.yml +0 -1
  3. data/exe/miteru +1 -1
  4. data/lefthook.yml +2 -1
  5. data/lib/miteru/commands/sidekiq.rb +1 -1
  6. data/lib/miteru/config.rb +49 -2
  7. data/lib/miteru/feeds/ayashige.rb +1 -1
  8. data/lib/miteru/feeds/phishing_database.rb +1 -1
  9. data/lib/miteru/feeds/tweetfeed.rb +1 -1
  10. data/lib/miteru/feeds/urlscan.rb +1 -1
  11. data/lib/miteru/feeds/urlscan_pro.rb +1 -1
  12. data/lib/miteru/orchestrator.rb +2 -4
  13. data/lib/miteru/sidekiq/application.rb +4 -0
  14. data/lib/miteru/sidekiq/jobs.rb +9 -5
  15. data/lib/miteru/version.rb +1 -1
  16. data/miteru.gemspec +12 -12
  17. metadata +30 -30
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 15c49f2c418826deed2d6862018c10582649a91a26ca41218fba15bd7f80c11a
4
- data.tar.gz: 120181d2d55f7ebb5d7aa500624ff4a40827c5f3604f391b391b3b7ff7298ab4
3
+ metadata.gz: a1a624406c34f6f2c4fd913029a408145cd4d24b4151914e9b9e366a48882ade
4
+ data.tar.gz: 745cb186ac2e294acaefb886f3cfd1e8b2c28d8ec19f9021844c823c9d1114c7
5
5
  SHA512:
6
- metadata.gz: 317e72123d90abcc99ab24e0b7ce36c3dd0fbb27310c0a1382614420a17bb70440e7a21f657d780508dfddd3727679eef2f566afde64fb4ecaff6c552f683c3d
7
- data.tar.gz: 2ec13073b8711e381e97ed35882eb8342149289f277f0a4769c8277eb2bac57db262a3868ae65b3d0dc667466d4d563ed109974f8d4effc32bbba792b08f915e
6
+ metadata.gz: 952e2b6379e775cfe6c8a7d3ec9c837ed1fb96408a3156edac20cc24c308c96263bb2f0b8e7128b4ee1c8607a2f2b77b4989ee4c826c88a5ffc72acc40f40b67
7
+ data.tar.gz: efed521f1f6fac24b7966e5f5dfbf321f98acfae6e80afa94b7661bfe12be7f4d709e9ad4344301ebe184c8d7a17261e771490d1921972203b2c2a4b25bee306
data/docker-compose.yml CHANGED
@@ -1,4 +1,3 @@
1
- version: "3"
2
1
  services:
3
2
  redis:
4
3
  image: "redis/redis-stack:6.2.6-v10"
data/exe/miteru CHANGED
@@ -3,7 +3,7 @@
3
3
 
4
4
  $LOAD_PATH.unshift("#{__dir__}/../lib")
5
5
 
6
- require 'miteru'
6
+ require "miteru"
7
7
 
8
8
  ARGV.unshift(Miteru::CLI::App.default_task) unless Miteru::CLI::App.all_tasks.key?(ARGV[0])
9
9
  Miteru::CLI::App.start(ARGV)
data/lefthook.yml CHANGED
@@ -5,5 +5,6 @@ pre-commit:
5
5
  run: bundle exec standardrb --fix {staged_files}
6
6
  stage_fixed: true
7
7
  actionlint:
8
- glob: ".github/workflows/*.yaml"
8
+ root: ".github/workflows"
9
+ glob: "*.{yaml,yml}"
9
10
  run: actionlint
data/lib/miteru/commands/sidekiq.rb CHANGED
@@ -10,7 +10,7 @@ module Miteru
10
10
  def included(thor)
11
11
  thor.class_eval do
12
12
  desc "sidekiq", "Start Sidekiq"
13
- method_option :env, type: :string, default: "production", desc: "Environment"
13
+ method_option :env, type: :string, default: "production", desc: "Environment", aliases: "-e"
14
14
  method_option :concurrency, type: :numeric, default: 5, desc: "Sidekiq concurrency", aliases: "-c"
15
15
  def sidekiq
16
16
  require "sidekiq/cli"
data/lib/miteru/config.rb CHANGED
@@ -14,14 +14,22 @@ module Miteru
14
14
  download_to: "/tmp",
15
15
  file_max_size: 1024 * 1024 * 100,
16
16
  file_extensions: [".zip", ".rar", ".7z", ".tar", ".gz"],
17
- file_mime_types: ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar",
18
- "application/gzip"],
17
+ file_mime_types: [
18
+ "application/zip",
19
+ "application/vnd.rar",
20
+ "application/x-7z-compressed",
21
+ "application/x-tar",
22
+ "application/gzip"
23
+ ],
19
24
  api_timeout: 60,
20
25
  http_timeout: 60,
21
26
  download_timeout: 60,
22
27
  sentry_dsn: nil,
23
28
  sentry_trace_sample_rate: 0.25,
24
29
  sidekiq_redis_url: nil,
30
+ sidekiq_job_retry: 0,
31
+ sidekiq_batch_size: 50,
32
+ sidekiq_job_timeout: 600,
25
33
  cache_redis_url: nil,
26
34
  cache_ex: nil,
27
35
  cache_prefix: "miteru:cache",
@@ -44,6 +52,45 @@ module Miteru
44
52
  # @!attribute [r] sidekiq_redis_url
45
53
  # @return [String, nil]
46
54
 
55
+ # @!attribute [r] sidekiq_job_retry
56
+ # @return [Integer]
57
+
58
+ # @!attribute [r] sidekiq_batch_size
59
+ # @return [Integer]
60
+
61
+ # @!attribute [r] sidekiq_job_timeout
62
+ # @return [Integer]
63
+
64
+ # @!attribute [r] cache_redis_url
65
+ # @return [String, nil]
66
+
67
+ # @!attribute [r] cache_ex
68
+ # @return [Integer, nil]
69
+
70
+ # @!attribute [r] cache_prefix
71
+ # @return [String]
72
+
73
+ # @!attribute [r] http_timeout
74
+ # @return [Integer]
75
+
76
+ # @!attribute [r] api_timeout
77
+ # @return [Integer]
78
+
79
+ # @!attribute [r] download_timeout
80
+ # @return [Integer]
81
+
82
+ # @!attribute [rw] auto_download
83
+ # @return [Boolean]
84
+
85
+ # @!attribute [rw] directory_traveling
86
+ # @return [Boolean]
87
+
88
+ # @!attribute [rw] download_to
89
+ # @return [String]
90
+
91
+ # @!attribute [rw] threads
92
+ # @return [Integer]
93
+
47
94
  # @!attribute [r] cache_redis_url
48
95
  # @return [String, nil]
49
96
 
data/lib/miteru/feeds/ayashige.rb CHANGED
@@ -4,7 +4,7 @@ module Miteru
4
4
  class Feeds
5
5
  class Ayashige < Base
6
6
  def initialize(base_url = "https://ayashige.herokuapp.com")
7
- super(base_url)
7
+ super
8
8
  end
9
9
 
10
10
  def urls
data/lib/miteru/feeds/phishing_database.rb CHANGED
@@ -4,7 +4,7 @@ module Miteru
4
4
  class Feeds
5
5
  class PhishingDatabase < Base
6
6
  def initialize(base_url = "https://raw.githubusercontent.com")
7
- super(base_url)
7
+ super
8
8
  end
9
9
 
10
10
  def urls
data/lib/miteru/feeds/tweetfeed.rb CHANGED
@@ -4,7 +4,7 @@ module Miteru
4
4
  class Feeds
5
5
  class TweetFeed < Base
6
6
  def initialize(base_url = "https://api.tweetfeed.live")
7
- super(base_url)
7
+ super
8
8
  end
9
9
 
10
10
  def urls
data/lib/miteru/feeds/urlscan.rb CHANGED
@@ -7,7 +7,7 @@ module Miteru
7
7
  # @param [String] base_url
8
8
  #
9
9
  def initialize(base_url = "https://urlscan.io")
10
- super(base_url)
10
+ super
11
11
 
12
12
  @headers = {"api-key": api_key}
13
13
  end
data/lib/miteru/feeds/urlscan_pro.rb CHANGED
@@ -7,7 +7,7 @@ module Miteru
7
7
  # @param [String] base_url
8
8
  #
9
9
  def initialize(base_url = "https://urlscan.io")
10
- super(base_url)
10
+ super
11
11
 
12
12
  @headers = {"api-key": api_key}
13
13
  end
data/lib/miteru/orchestrator.rb CHANGED
@@ -13,10 +13,8 @@ module Miteru
13
13
  end
14
14
 
15
15
  def sidekiq_call
16
- non_cached_websites.each do |website|
17
- Jobs::CrawleJob.perform_async(website.url, website.source)
18
- logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
19
- end
16
+ array_of_args = non_cached_websites.map { |website| [website.url, website.source] }
17
+ Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size)
20
18
  end
21
19
 
22
20
  def parallel_call
data/lib/miteru/sidekiq/application.rb CHANGED
@@ -6,6 +6,10 @@ require "miteru/sidekiq/jobs"
6
6
 
7
7
  Sidekiq.configure_server do |config|
8
8
  config.redis = {url: Miteru.config.sidekiq_redis_url.to_s}
9
+ config.default_job_options = {
10
+ retry: Miteru.config.sidekiq_job_retry,
11
+ expires_in: 0.second
12
+ }
9
13
  end
10
14
 
11
15
  Sidekiq.configure_client do |config|
data/lib/miteru/sidekiq/jobs.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq"
4
+ require "timeout"
4
5
 
5
6
  module Miteru
6
7
  module Jobs
@@ -14,12 +15,15 @@ module Miteru
14
15
  #
15
16
  def perform(url, source)
16
17
  website = Miteru::Website.new(url, source:)
18
+
17
19
  with_db_connection do
18
- result = Crawler.result(website)
19
- if result.success?
20
- Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
21
- else
22
- Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
20
+ Timeout.timeout(Miteru.config.sidekiq_job_timeout) do
21
+ result = Crawler.result(website)
22
+ if result.success?
23
+ Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
24
+ else
25
+ Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
26
+ end
23
27
  end
24
28
  end
25
29
  end
data/lib/miteru/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "2.3.1"
4
+ VERSION = "2.4.0"
5
5
  end
data/miteru.gemspec CHANGED
@@ -30,20 +30,20 @@ Gem::Specification.new do |spec|
30
30
  spec.add_development_dependency "coveralls_reborn", "~> 0.28"
31
31
  spec.add_development_dependency "fuubar", "~> 2.5.1"
32
32
  spec.add_development_dependency "mysql2", "~> 0.5.6"
33
- spec.add_development_dependency "pg", "~> 1.5.7"
33
+ spec.add_development_dependency "pg", "~> 1.5.9"
34
34
  spec.add_development_dependency "rake", "~> 13.2.1"
35
35
  spec.add_development_dependency "rspec", "~> 3.13"
36
36
  spec.add_development_dependency "simplecov-lcov", "~> 0.8"
37
- spec.add_development_dependency "standard", "~> 1.40.0"
37
+ spec.add_development_dependency "standard", "~> 1.41.1"
38
38
  spec.add_development_dependency "test-prof", "~> 1.4.2"
39
39
  spec.add_development_dependency "vcr", "~> 6.3.1"
40
- spec.add_development_dependency "webmock", "~> 3.23.1"
40
+ spec.add_development_dependency "webmock", "~> 3.24.0"
41
41
 
42
- spec.add_dependency "activerecord", "7.2.1"
42
+ spec.add_dependency "activerecord", "7.2.2"
43
43
  spec.add_dependency "addressable", "2.8.7"
44
44
  spec.add_dependency "anyway_config", "2.6.4"
45
45
  spec.add_dependency "colorize", "1.1.0"
46
- spec.add_dependency "dotenv", "3.1.2"
46
+ spec.add_dependency "dotenv", "3.1.4"
47
47
  spec.add_dependency "down", "5.4.2"
48
48
  spec.add_dependency "dry-files", "1.1.0"
49
49
  spec.add_dependency "dry-monads", "1.6.0"
@@ -51,15 +51,15 @@ Gem::Specification.new do |spec|
51
51
  spec.add_dependency "memo_wise", "1.9.0"
52
52
  spec.add_dependency "oga", "3.4"
53
53
  spec.add_dependency "parallel", "1.26.3"
54
- spec.add_dependency "puma", "6.4.2"
55
- spec.add_dependency "rack", "3.1.7"
54
+ spec.add_dependency "puma", "6.4.3"
55
+ spec.add_dependency "rack", "3.1.8"
56
56
  spec.add_dependency "rack-session", "2.0.0"
57
- spec.add_dependency "rackup", "2.1.0"
57
+ spec.add_dependency "rackup", "2.2.0"
58
58
  spec.add_dependency "redis", "5.3.0"
59
- spec.add_dependency "semantic_logger", "4.16.0"
60
- spec.add_dependency "sentry-ruby", "5.19.0"
61
- spec.add_dependency "sentry-sidekiq", "5.19.0"
62
- spec.add_dependency "sidekiq", "7.3.2"
59
+ spec.add_dependency "semantic_logger", "4.16.1"
60
+ spec.add_dependency "sentry-ruby", "5.21.0"
61
+ spec.add_dependency "sentry-sidekiq", "5.21.0"
62
+ spec.add_dependency "sidekiq", "7.3.4"
63
63
  spec.add_dependency "slack-notifier", "2.4.0"
64
64
  spec.add_dependency "sqlite3", "1.7.2"
65
65
  spec.add_dependency "thor", "1.3.2"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.1
4
+ version: 2.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-09-05 00:00:00.000000000 Z
11
+ date: 2024-11-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -86,14 +86,14 @@ dependencies:
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: 1.5.7
89
+ version: 1.5.9
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: 1.5.7
96
+ version: 1.5.9
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: rake
99
99
  requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: 1.40.0
145
+ version: 1.41.1
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: 1.40.0
152
+ version: 1.41.1
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: test-prof
155
155
  requirement: !ruby/object:Gem::Requirement
@@ -184,28 +184,28 @@ dependencies:
184
184
  requirements:
185
185
  - - "~>"
186
186
  - !ruby/object:Gem::Version
187
- version: 3.23.1
187
+ version: 3.24.0
188
188
  type: :development
189
189
  prerelease: false
190
190
  version_requirements: !ruby/object:Gem::Requirement
191
191
  requirements:
192
192
  - - "~>"
193
193
  - !ruby/object:Gem::Version
194
- version: 3.23.1
194
+ version: 3.24.0
195
195
  - !ruby/object:Gem::Dependency
196
196
  name: activerecord
197
197
  requirement: !ruby/object:Gem::Requirement
198
198
  requirements:
199
199
  - - '='
200
200
  - !ruby/object:Gem::Version
201
- version: 7.2.1
201
+ version: 7.2.2
202
202
  type: :runtime
203
203
  prerelease: false
204
204
  version_requirements: !ruby/object:Gem::Requirement
205
205
  requirements:
206
206
  - - '='
207
207
  - !ruby/object:Gem::Version
208
- version: 7.2.1
208
+ version: 7.2.2
209
209
  - !ruby/object:Gem::Dependency
210
210
  name: addressable
211
211
  requirement: !ruby/object:Gem::Requirement
@@ -254,14 +254,14 @@ dependencies:
254
254
  requirements:
255
255
  - - '='
256
256
  - !ruby/object:Gem::Version
257
- version: 3.1.2
257
+ version: 3.1.4
258
258
  type: :runtime
259
259
  prerelease: false
260
260
  version_requirements: !ruby/object:Gem::Requirement
261
261
  requirements:
262
262
  - - '='
263
263
  - !ruby/object:Gem::Version
264
- version: 3.1.2
264
+ version: 3.1.4
265
265
  - !ruby/object:Gem::Dependency
266
266
  name: down
267
267
  requirement: !ruby/object:Gem::Requirement
@@ -366,28 +366,28 @@ dependencies:
366
366
  requirements:
367
367
  - - '='
368
368
  - !ruby/object:Gem::Version
369
- version: 6.4.2
369
+ version: 6.4.3
370
370
  type: :runtime
371
371
  prerelease: false
372
372
  version_requirements: !ruby/object:Gem::Requirement
373
373
  requirements:
374
374
  - - '='
375
375
  - !ruby/object:Gem::Version
376
- version: 6.4.2
376
+ version: 6.4.3
377
377
  - !ruby/object:Gem::Dependency
378
378
  name: rack
379
379
  requirement: !ruby/object:Gem::Requirement
380
380
  requirements:
381
381
  - - '='
382
382
  - !ruby/object:Gem::Version
383
- version: 3.1.7
383
+ version: 3.1.8
384
384
  type: :runtime
385
385
  prerelease: false
386
386
  version_requirements: !ruby/object:Gem::Requirement
387
387
  requirements:
388
388
  - - '='
389
389
  - !ruby/object:Gem::Version
390
- version: 3.1.7
390
+ version: 3.1.8
391
391
  - !ruby/object:Gem::Dependency
392
392
  name: rack-session
393
393
  requirement: !ruby/object:Gem::Requirement
@@ -408,14 +408,14 @@ dependencies:
408
408
  requirements:
409
409
  - - '='
410
410
  - !ruby/object:Gem::Version
411
- version: 2.1.0
411
+ version: 2.2.0
412
412
  type: :runtime
413
413
  prerelease: false
414
414
  version_requirements: !ruby/object:Gem::Requirement
415
415
  requirements:
416
416
  - - '='
417
417
  - !ruby/object:Gem::Version
418
- version: 2.1.0
418
+ version: 2.2.0
419
419
  - !ruby/object:Gem::Dependency
420
420
  name: redis
421
421
  requirement: !ruby/object:Gem::Requirement
@@ -436,56 +436,56 @@ dependencies:
436
436
  requirements:
437
437
  - - '='
438
438
  - !ruby/object:Gem::Version
439
- version: 4.16.0
439
+ version: 4.16.1
440
440
  type: :runtime
441
441
  prerelease: false
442
442
  version_requirements: !ruby/object:Gem::Requirement
443
443
  requirements:
444
444
  - - '='
445
445
  - !ruby/object:Gem::Version
446
- version: 4.16.0
446
+ version: 4.16.1
447
447
  - !ruby/object:Gem::Dependency
448
448
  name: sentry-ruby
449
449
  requirement: !ruby/object:Gem::Requirement
450
450
  requirements:
451
451
  - - '='
452
452
  - !ruby/object:Gem::Version
453
- version: 5.19.0
453
+ version: 5.21.0
454
454
  type: :runtime
455
455
  prerelease: false
456
456
  version_requirements: !ruby/object:Gem::Requirement
457
457
  requirements:
458
458
  - - '='
459
459
  - !ruby/object:Gem::Version
460
- version: 5.19.0
460
+ version: 5.21.0
461
461
  - !ruby/object:Gem::Dependency
462
462
  name: sentry-sidekiq
463
463
  requirement: !ruby/object:Gem::Requirement
464
464
  requirements:
465
465
  - - '='
466
466
  - !ruby/object:Gem::Version
467
- version: 5.19.0
467
+ version: 5.21.0
468
468
  type: :runtime
469
469
  prerelease: false
470
470
  version_requirements: !ruby/object:Gem::Requirement
471
471
  requirements:
472
472
  - - '='
473
473
  - !ruby/object:Gem::Version
474
- version: 5.19.0
474
+ version: 5.21.0
475
475
  - !ruby/object:Gem::Dependency
476
476
  name: sidekiq
477
477
  requirement: !ruby/object:Gem::Requirement
478
478
  requirements:
479
479
  - - '='
480
480
  - !ruby/object:Gem::Version
481
- version: 7.3.2
481
+ version: 7.3.4
482
482
  type: :runtime
483
483
  prerelease: false
484
484
  version_requirements: !ruby/object:Gem::Requirement
485
485
  requirements:
486
486
  - - '='
487
487
  - !ruby/object:Gem::Version
488
- version: 7.3.2
488
+ version: 7.3.4
489
489
  - !ruby/object:Gem::Dependency
490
490
  name: slack-notifier
491
491
  requirement: !ruby/object:Gem::Requirement
@@ -624,7 +624,7 @@ licenses:
624
624
  - MIT
625
625
  metadata:
626
626
  rubygems_mfa_required: 'true'
627
- post_install_message:
627
+ post_install_message:
628
628
  rdoc_options: []
629
629
  require_paths:
630
630
  - lib
@@ -639,8 +639,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
639
639
  - !ruby/object:Gem::Version
640
640
  version: '0'
641
641
  requirements: []
642
- rubygems_version: 3.5.6
643
- signing_key:
642
+ rubygems_version: 3.5.16
643
+ signing_key:
644
644
  specification_version: 4
645
645
  summary: A phishing kit collector for scavengers
646
646
  test_files: []