miteru 1.2.2 → 2.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/gem.yml +36 -0
  3. data/.github/workflows/{test.yml → ruby.yml} +4 -13
  4. data/.gitignore +7 -2
  5. data/.rspec +1 -1
  6. data/README.md +7 -17
  7. data/docker-compose.yml +12 -0
  8. data/exe/miteru +3 -3
  9. data/lefthook.yml +9 -0
  10. data/lib/miteru/cli/application.rb +27 -0
  11. data/lib/miteru/cli/base.rb +16 -0
  12. data/lib/miteru/cli/database.rb +11 -0
  13. data/lib/miteru/commands/database.rb +23 -0
  14. data/lib/miteru/commands/main.rb +37 -0
  15. data/lib/miteru/commands/sidekiq.rb +35 -0
  16. data/lib/miteru/commands/web.rb +37 -0
  17. data/lib/miteru/concerns/database_connectable.rb +16 -0
  18. data/lib/miteru/concerns/error_unwrappable.rb +30 -0
  19. data/lib/miteru/config.rb +98 -0
  20. data/lib/miteru/crawler.rb +28 -44
  21. data/lib/miteru/database.rb +50 -38
  22. data/lib/miteru/downloader.rb +52 -41
  23. data/lib/miteru/errors.rb +37 -0
  24. data/lib/miteru/feeds/ayashige.rb +9 -20
  25. data/lib/miteru/feeds/base.rb +141 -0
  26. data/lib/miteru/feeds/phishing_database.rb +11 -10
  27. data/lib/miteru/feeds/urlscan.rb +47 -19
  28. data/lib/miteru/feeds/urlscan_pro.rb +20 -18
  29. data/lib/miteru/http.rb +51 -0
  30. data/lib/miteru/kit.rb +28 -20
  31. data/lib/miteru/mixin.rb +2 -29
  32. data/lib/miteru/notifiers/base.rb +10 -3
  33. data/lib/miteru/notifiers/slack.rb +85 -10
  34. data/lib/miteru/notifiers/urlscan.rb +29 -14
  35. data/lib/miteru/orchestrator.rb +58 -0
  36. data/lib/miteru/record.rb +8 -15
  37. data/lib/miteru/service.rb +28 -0
  38. data/lib/miteru/sidekiq/application.rb +13 -0
  39. data/lib/miteru/sidekiq/jobs.rb +21 -0
  40. data/lib/miteru/version.rb +1 -1
  41. data/lib/miteru/web/application.rb +42 -0
  42. data/lib/miteru/website.rb +48 -48
  43. data/lib/miteru.rb +130 -22
  44. data/miteru.gemspec +49 -38
  45. metadata +262 -97
  46. data/.overcommit.yml +0 -12
  47. data/.standard.yml +0 -4
  48. data/lib/miteru/attachement.rb +0 -74
  49. data/lib/miteru/cli.rb +0 -41
  50. data/lib/miteru/configuration.rb +0 -122
  51. data/lib/miteru/error.rb +0 -7
  52. data/lib/miteru/feeds/feed.rb +0 -53
  53. data/lib/miteru/feeds/phishstats.rb +0 -28
  54. data/lib/miteru/feeds.rb +0 -45
  55. data/lib/miteru/http_client.rb +0 -85
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7f422d77d43c2e555014fc16e28564b207d67d70bdd84e61d2ce2da99f368caf
4
- data.tar.gz: 74d7a5405799534981d3b273b69537b3d18c171f42d9956684fd8e905361628d
3
+ metadata.gz: aa673c60148a21b0838fec44df350670fad6af4873cf54922ff14f8042f8a781
4
+ data.tar.gz: 3b09b49f88c1a56224e5f472647646619061fc57a258d54bb5a51ce4420ed22e
5
5
  SHA512:
6
- metadata.gz: 02c22920de1f06080f78e421c0ee73ec51e2bf95cb50f3ab2333debec2281fa40311d7759a252dc2fa844b678e95f73625727457cbcc33dfd4c4416a31bc743c
7
- data.tar.gz: fe3e85ccf788b4ee2d504be1ef2cb72d4f2b5f06680c77318b1d56b3363dd0608e05334b04f5b7b6b2927f1ea8848bb4163909d619a8224d87d181585fe8d29a
6
+ metadata.gz: 396239204cda13c34766428ac7b223828413b56179b72eba137be1ded7af40ff7753ded1a504cef615b3654c43f597fd223c39324b063a4a281ea729e27882fa
7
+ data.tar.gz: f22eb47d90dc5ca7cd77896de1fcb9420450f9055f49da4d316c8cdd9f255acb5282ed91529109122811c300a0ecfc92de71a4b26031854f9671c58e6671990f
@@ -0,0 +1,36 @@
1
+ name: Release gem
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ rubygems-otp-code:
7
+ description: RubyGems OTP code
8
+ required: true
9
+ type: string
10
+
11
+ jobs:
12
+ release:
13
+ runs-on: ubuntu-latest
14
+ strategy:
15
+ matrix:
16
+ ruby-version:
17
+ - 3.3
18
+ steps:
19
+ - name: Checkout
20
+ uses: actions/checkout@v4
21
+ - name: Set up Ruby
22
+ uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby-version }}
25
+ bundler: latest
26
+ bundler-cache: true
27
+ - name: Configure Git
28
+ # Configure Git to push a tag
29
+ run: |
30
+ git config --global user.name "${GITHUB_ACTOR}"
31
+ git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
32
+ - name: Release gem
33
+ run: bundle exec rake release
34
+ env:
35
+ GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
36
+ GEM_HOST_OTP_CODE: ${{ inputs.rubygems-otp-code }}
@@ -5,10 +5,9 @@ on: [push, pull_request]
5
5
  jobs:
6
6
  build:
7
7
  runs-on: ubuntu-latest
8
-
9
8
  services:
10
9
  postgres:
11
- image: postgres:12
10
+ image: postgres:16
12
11
  env:
13
12
  POSTGRES_USER: postgres
14
13
  POSTGRES_PASSWORD: postgres
@@ -20,7 +19,6 @@ jobs:
20
19
  --health-retries 5
21
20
  ports:
22
21
  - 5432:5432
23
-
24
22
  mysql:
25
23
  image: mysql:8.0
26
24
  env:
@@ -35,31 +33,24 @@ jobs:
35
33
  --health-interval=10s
36
34
  --health-timeout=5s
37
35
  --health-retries=3
38
-
39
36
  strategy:
40
- fail-fast: false
41
37
  matrix:
42
- ruby: [2.7, "3.0"]
43
-
38
+ ruby: [3.1, 3.2, 3.3]
44
39
  steps:
45
- - uses: actions/checkout@v3
40
+ - uses: actions/checkout@v4
46
41
  - name: Set up Ruby
47
42
  uses: ruby/setup-ruby@v1
48
43
  with:
49
44
  ruby-version: ${{ matrix.ruby }}
50
45
  bundler: latest
51
46
  bundler-cache: true
52
-
53
47
  - name: Install dependencies
54
- run: |
55
- sudo apt-get -yqq install libpq-dev libmysqlclient-dev
56
-
48
+ run: sudo apt-get -yqq install libpq-dev libmysqlclient-dev
57
49
  - name: Test with PostgreSQL
58
50
  env:
59
51
  MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
60
52
  run: |
61
53
  bundle exec rake
62
-
63
54
  - name: Test with MySQL
64
55
  env:
65
56
  MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
data/.gitignore CHANGED
@@ -52,5 +52,10 @@ Gemfile.lock
52
52
  ## RSpec
53
53
  .rspec_status
54
54
 
55
- # SQLite database
56
- *.db
55
+ # SQLite
56
+ *.db
57
+ *.db-shm
58
+ *.db-wal
59
+
60
+ # .env
61
+ .env
data/.rspec CHANGED
@@ -1,3 +1,3 @@
1
- --format documentation
1
+ --format Fuubar
2
2
  --color
3
3
  --require spec_helper
data/README.md CHANGED
@@ -5,33 +5,23 @@
5
5
  [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
6
6
  [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
7
7
 
8
- Miteru is an experimental phishing kit detection tool.
8
+ A phishing kit collector for scavengers.
9
9
 
10
10
  ## Disclaimer
11
11
 
12
12
  This tool is for research purposes only. The use of this tool is your responsibility.
13
13
  I take no responsibility and/or liability for how you choose to use this tool.
14
14
 
15
- ## How it works
15
+ ## How It Works
16
16
 
17
17
  - It collects phishy URLs from the following feeds:
18
- - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#task.source%3Acertstream-suspicious)
19
- - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#task.source%3Aopenphish)
20
- - [PhishTank feed via urlscan.io](https://urlscan.io/search/#task.source%3Aphishtank)
21
- - [URLhaus feed via urlscan.io](https://urlscan.io/search/#task.source%3Aurlhaus)
22
- - urlscan.io phish feed (available for Pro users)
23
- - [Ayashige feed](https://github.com/ninoseki/ayashige)
24
- - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
25
- - [PhishStats feed](https://phishstats.info/)
26
- - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
18
+ - urlscan.io's automatic submissions. (`task.method:automatic`)
19
+ - urlscan.io phish feed. (available for Pro users)
20
+ - [mitchellkrogza/Phishing.Database](https://github.com/mitchellkrogza/Phishing.Database)'s `phishing-links-ACTIVE-NOW.txt`.
21
+ - [ninoseki/ayashige](https://github.com/ninoseki/ayashige) feed.
22
+ - It checks each phishy URL whether it enables directory listing and contains phishing kits (compressed files) or not.
27
23
  - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
28
24
 
29
- ## Features
30
-
31
- - [x] Phishing kit detection & collection
32
- - [x] Slack notification
33
- - [x] Threading
34
-
35
25
  ## Docs
36
26
 
37
27
  - [Requirements & Installation](https://github.com/ninoseki/miteru/wiki/Requirements-&-Installation)
@@ -0,0 +1,12 @@
1
+ version: "3"
2
+ services:
3
+ redis:
4
+ image: "redis/redis-stack:6.2.6-v10"
5
+ restart: always
6
+ ports:
7
+ - ${REDIS_PORT:-6379}:6379
8
+ - ${REDIS_INSIGHT_PORT:-8001}:8001
9
+ volumes:
10
+ - redis:/data
11
+ volumes:
12
+ redis:
data/exe/miteru CHANGED
@@ -3,7 +3,7 @@
3
3
 
4
4
  $LOAD_PATH.unshift("#{__dir__}/../lib")
5
5
 
6
- require "miteru"
6
+ require 'miteru'
7
7
 
8
- ARGV.unshift(Miteru::CLI.default_task) unless Miteru::CLI.all_tasks.key?(ARGV[0])
9
- Miteru::CLI.start(ARGV)
8
+ ARGV.unshift(Miteru::CLI::App.default_task) unless Miteru::CLI::App.all_tasks.key?(ARGV[0])
9
+ Miteru::CLI::App.start(ARGV)
data/lefthook.yml ADDED
@@ -0,0 +1,9 @@
1
+ pre-commit:
2
+ commands:
3
+ standard:
4
+ glob: "*.rb"
5
+ run: bundle exec standardrb --fix {staged_files}
6
+ stage_fixed: true
7
+ actionlint:
8
+ glob: ".github/workflows/*.yaml"
9
+ run: actionlint
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+ require "thor/hollaback"
5
+
6
+ require "miteru/cli/base"
7
+ require "miteru/cli/database"
8
+
9
+ require "miteru/commands/main"
10
+ require "miteru/commands/sidekiq"
11
+ require "miteru/commands/web"
12
+
13
+ module Miteru
14
+ module CLI
15
+ #
16
+ # Main CLI class
17
+ #
18
+ class App < Base
19
+ include Commands::Main
20
+ include Commands::Sidekiq
21
+ include Commands::Web
22
+
23
+ desc "db", "Sub commands for DB"
24
+ subcommand "db", CLI::Database
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module CLI
5
+ #
6
+ # Base class for Thor classes
7
+ #
8
+ class Base < Thor
9
+ class << self
10
+ def exit_on_failure?
11
+ true
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "miteru/commands/database"
4
+
5
+ module Miteru
6
+ module CLI
7
+ class Database < Base
8
+ include Commands::Database
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Commands
5
+ module Database
6
+ class << self
7
+ def included(thor)
8
+ thor.class_eval do
9
+ include Concerns::DatabaseConnectable
10
+
11
+ desc "migrate", "Migrate DB schemas"
12
+ around :with_db_connection
13
+ method_option :verbose, type: :boolean, default: true
14
+ def migrate(direction = "up")
15
+ ActiveRecord::Migration.verbose = options["verbose"]
16
+ Miteru::Database.migrate direction.to_sym
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Commands
5
+ module Main
6
+ class << self
7
+ def included(thor)
8
+ thor.class_eval do
9
+ include Concerns::DatabaseConnectable
10
+
11
+ method_option :auto_download, type: :boolean, default: false,
12
+ desc: "Enable or disable auto-downloading of phishing kits"
13
+ method_option :directory_traveling, type: :boolean, default: false,
14
+ desc: "Enable or disable directory traveling"
15
+ method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
16
+ method_option :threads, type: :numeric, desc: "Number of threads to use", default: Parallel.processor_count
17
+ method_option :verbose, type: :boolean, default: true
18
+ desc "execute", "Execute the crawler"
19
+ around :with_db_connection
20
+ def execute
21
+ Miteru.config.tap do |config|
22
+ config.auto_download = options["auto_download"]
23
+ config.directory_traveling = options["directory_traveling"]
24
+ config.download_to = options["download_to"]
25
+ config.threads = options["threads"]
26
+ config.verbose = options["verbose"]
27
+ end
28
+
29
+ Orchestrator.call
30
+ end
31
+ default_command :execute
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Commands
5
+ #
6
+ # Sidekiq sub-commands
7
+ #
8
+ module Sidekiq
9
+ class << self
10
+ def included(thor)
11
+ thor.class_eval do
12
+ desc "sidekiq", "Start Sidekiq"
13
+ method_option :env, type: :string, default: "production", desc: "Environment"
14
+ method_option :concurrency, type: :numeric, default: 5, desc: "Sidekiq concurrency", aliases: "-c"
15
+ def sidekiq
16
+ require "sidekiq/cli"
17
+
18
+ ENV["APP_ENV"] ||= options["env"]
19
+ concurrency = options["concurrency"].to_s
20
+
21
+ cli = ::Sidekiq::CLI.instance
22
+ cli.parse [
23
+ "-r",
24
+ File.expand_path(File.join(__dir__, "..", "sidekiq", "application.rb")),
25
+ "-c",
26
+ concurrency
27
+ ]
28
+ cli.run
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Commands
5
+ #
6
+ # Web sub-commands
7
+ #
8
+ module Web
9
+ class << self
10
+ def included(thor)
11
+ thor.class_eval do
12
+ desc "web", "Start the web app"
13
+ method_option :port, type: :numeric, default: 9292, desc: "Port to listen on"
14
+ method_option :host, type: :string, default: "localhost", desc: "Hostname to listen on"
15
+ method_option :threads, type: :string, default: "0:3", desc: "min:max threads to use"
16
+ method_option :verbose, type: :boolean, default: false, desc: "Don't report each request"
17
+ method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
18
+ method_option :env, type: :string, default: "production", desc: "Environment"
19
+ def web
20
+ require "miteru/web/application"
21
+
22
+ ENV["APP_ENV"] ||= options["env"]
23
+
24
+ Miteru::Web::App.run!(
25
+ port: options["port"],
26
+ host: options["host"],
27
+ threads: options["threads"],
28
+ verbose: options["verbose"],
29
+ worker_timeout: options["worker_timeout"]
30
+ )
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Concerns
5
+ #
6
+ # Database connectable concern
7
+ #
8
+ module DatabaseConnectable
9
+ extend ActiveSupport::Concern
10
+
11
+ def with_db_connection(&block)
12
+ Database.with_db_connection(&block)
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Miteru
4
+ module Concerns
5
+ #
6
+ # Error unwrappable concern
7
+ #
8
+ module ErrorUnwrappable
9
+ extend ActiveSupport::Concern
10
+
11
+ def unwrap_error(err)
12
+ return err unless err.is_a?(Dry::Monads::UnwrapError)
13
+
14
+ # NOTE: UnwrapError's receiver can be either of:
15
+ # - Dry::Monads::Try::Error
16
+ # - Dry::Monads::Result::Failure
17
+ receiver = err.receiver
18
+ case receiver
19
+ when Dry::Monads::Try::Error
20
+ # Error may be wrapped like Matryoshka
21
+ unwrap_error receiver.exception
22
+ when Dry::Monads::Failure
23
+ unwrap_error receiver.failure
24
+ else
25
+ err
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "anyway_config"
4
+
5
+ module Miteru
6
+ class Config < Anyway::Config
7
+ config_name :miteru
8
+ env_prefix ""
9
+
10
+ attr_config(
11
+ auto_download: false,
12
+ database_url: URI("sqlite3:miteru.db"),
13
+ directory_traveling: false,
14
+ download_to: "/tmp",
15
+ file_max_size: 1024 * 1024 * 100,
16
+ file_extensions: [".zip", ".rar", ".7z", ".tar", ".gz"],
17
+ file_mime_types: ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar",
18
+ "application/gzip"],
19
+ api_timeout: 60,
20
+ http_timeout: 60,
21
+ download_timeout: 60,
22
+ sentry_dsn: nil,
23
+ sentry_trace_sample_rate: 0.25,
24
+ sidekiq_redis_url: nil,
25
+ slack_channel: "#general",
26
+ slack_webhook_url: nil,
27
+ threads: Parallel.processor_count,
28
+ urlscan_api_key: nil,
29
+ urlscan_submit_visibility: "public",
30
+ urlscan_date_condition: ">now-1h",
31
+ verbose: false
32
+ )
33
+
34
+ # @!attribute [r] sentry_dsn
35
+ # @return [String, nil]
36
+
37
+ # @!attribute [r] sentry_trace_sample_rate
38
+ # @return [Float]
39
+
40
+ # @!attribute [r] sidekiq_redis_url
41
+ # @return [String]
42
+
43
+ # @!attribute [r] http_timeout
44
+ # @return [Integer]
45
+
46
+ # @!attribute [r] api_timeout
47
+ # @return [Integer]
48
+
49
+ # @!attribute [r] download_timeout
50
+ # @return [Integer]
51
+
52
+ # @!attribute [rw] auto_download
53
+ # @return [Boolean]
54
+
55
+ # @!attribute [rw] directory_traveling
56
+ # @return [Boolean]
57
+
58
+ # @!attribute [rw] download_to
59
+ # @return [String]
60
+
61
+ # @!attribute [rw] threads
62
+ # @return [Integer]
63
+
64
+ # @!attribute [rw] verbose
65
+ # @return [Boolean]
66
+
67
+ # @!attribute [r] database_url
68
+ # @return [URI]
69
+
70
+ # @!attribute [r] file_max_size
71
+ # @return [Integer]
72
+
73
+ # @!attribute [r] file_extensions
74
+ # @return [Array<String>]
75
+
76
+ # @!attribute [r] file_mime_types
77
+ # @return [Array<String>]
78
+
79
+ # @!attribute [r] slack_webhook_url
80
+ # @return [String, nil]
81
+
82
+ # @!attribute [r] slack_channel
83
+ # @return [String]
84
+
85
+ # @!attribute [r] urlscan_api_key
86
+ # @return [String, nil]
87
+
88
+ # @!attribute [r] urlscan_submit_visibility
89
+ # @return [String]
90
+
91
+ # @!attribute [r] urlscan_date_condition
92
+ # @return [String]
93
+
94
+ def database_url=(val)
95
+ super(URI(val.to_s))
96
+ end
97
+ end
98
+ end
@@ -1,63 +1,47 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "colorize"
4
- require "parallel"
5
- require "uri"
6
-
7
3
  module Miteru
8
- class Crawler
9
- attr_reader :downloader, :feeds
4
+ class Crawler < Service
5
+ #
6
+ # @param [Miteru::Website] website
7
+ #
8
+ def call(website)
9
+ Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
10
+ Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
11
+ return unless website.has_kits?
10
12
 
11
- def initialize
12
- @downloader = Downloader.new(Miteru.configuration.download_to)
13
- @feeds = Feeds.new
14
- end
13
+ notify website
15
14
 
16
- def crawl(entry)
17
- website = Website.new(entry.url, entry.source)
18
- downloader.download_kits(website.kits) if website.has_kits? && auto_download?
19
- notify(website) if website.has_kits? || verbose?
20
- rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
21
- nil
22
- end
15
+ return unless auto_download?
23
16
 
24
- def execute
25
- suspicious_entries = feeds.suspicious_entries
26
- Miteru.logger.info "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
17
+ website.kits.each do |kit|
18
+ downloader = Downloader.new(kit)
19
+ result = downloader.result
27
20
 
28
- Parallel.each(suspicious_entries, in_threads: threads) do |entry|
29
- crawl entry
30
- end
21
+ if result.success?
22
+ Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}")
23
+ else
24
+ Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}")
25
+ end
26
+ end
27
+ end.recover { nil }.value!
31
28
  end
32
29
 
33
- def threads
34
- @threads ||= Miteru.configuration.threads
35
- end
36
-
37
- def notify(website)
38
- Parallel.each(notifiers) do |notifier|
39
- notifier.notify website
40
- end
41
- end
30
+ private
42
31
 
43
32
  def auto_download?
44
- Miteru.configuration.auto_download?
33
+ Miteru.config.auto_download
45
34
  end
46
35
 
47
- def verbose?
48
- Miteru.configuration.verbose?
36
+ def notify(website)
37
+ Parallel.each(notifiers) { |notifier| notifier.call(website) }
49
38
  end
50
39
 
51
- private
52
-
40
+ #
41
+ # @return [Array<Miteru::Notifiers::Base>]
42
+ #
53
43
  def notifiers
54
- @notifiers ||= [Notifiers::Slack.new, Notifiers::UrlScan.new].select(&:notifiable?)
55
- end
56
-
57
- class << self
58
- def execute
59
- new.execute
60
- end
44
+ @notifiers ||= Miteru.notifiers.map(&:new)
61
45
  end
62
46
  end
63
47
  end