miteru 0.12.8 → 0.12.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45b69c99d0a7475dbc37340003f846679b20fbf401726c76f30bc1ac9f456393
-  data.tar.gz: 33d7dfc24d05922f586f870928e0a5a0876d2f25217136e7918daf96ad2dda3c
+  metadata.gz: 82dd344be4204fb882bc86551cdc78d98f2e3822867bae69b06b34458de795d6
+  data.tar.gz: 8fe4f2cd077c697e52fca5016138ebb6420e4e335abafeda42a98a29624b6f79
 SHA512:
-  metadata.gz: 2b5956777da3f37214b7bd9e715e3aefdd6d417aba579e90a1011df322c26ed033bb34d1891203849198a20958c3e6cf7a884c362f2026c19719a8f5252d23d2
-  data.tar.gz: 6cb041ca54de93a22ed52453403e23206dbc0c26070fe00df65aed2122df7502cf9568a914cf6421de97dfdca73f6cf75b46f33d9ac0f6428b42357bd8ccf172
+  metadata.gz: 37a1635d4e782e52aef19858e6da6a1130db3bbc36e93e428da1f3e7a169bc52a74b1fb3f3d98e8b778c63dddc85e9e43672903dd68048cec83d824faed42dce
+  data.tar.gz: 706bad5277ca67650ff671b6ef761cd06429a915ef83b0934e29138fa1f61c6b3fb647ccf10f7675070e5fcade56914658edf5ec3b0daa76777996b9e11fe768

data/lib/miteru.rb

@@ -2,6 +2,7 @@
 
 require "miteru/version"
 
+require "miteru/configuration"
 require "miteru/error"
 require "miteru/http_client"
 require "miteru/kit"

data/lib/miteru/cli.rb

@@ -14,7 +14,20 @@ module Miteru
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
     def execute
-      Crawler.execute options.map { |k, v| [k.to_sym, v] }.to_h
+      Miteru.configure do |config|
+        config.auto_download = options["auto_download"]
+        config.ayashige = options["ayashige"]
+        config.directory_traveling = options["directory_traveling"]
+        config.download_to = options["download_to"]
+        config.post_to_slack = options["post_to_slack"]
+        config.size = options["size"]
+        config.verbose = options["verbose"]
+
+        threads = options["threads"]
+        config.threads = threads if threads
+      end
+
+      Crawler.execute
     end
   end
 end

data/lib/miteru/configuration.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "parallel"
+
+module Miteru
+  class Configuration
+    # @return [Boolean]
+    attr_accessor :auto_download
+
+    # @return [Boolean]
+    attr_accessor :ayashige
+
+    # @return [Boolean]
+    attr_accessor :directory_traveling
+
+    # @return [String]
+    attr_accessor :download_to
+
+    # @return [Boolean]
+    attr_accessor :post_to_slack
+
+    # @return [Integer]
+    attr_accessor :size
+
+    # @return [Integer]
+    attr_accessor :threads
+
+    # @return [Boolean]
+    attr_accessor :verbose
+
+    def initialize
+      @auto_download = false
+      @ayashige = false
+      @directory_traveling = false
+      @download_to = "/tmp"
+      @post_to_slack = false
+      @size = 100
+      @threads = Parallel.processor_count
+      @verbaose = false
+    end
+
+    def auto_download?
+      @auto_download
+    end
+
+    def ayashige?
+      @ayashige
+    end
+
+    def directory_traveling?
+      @directory_traveling
+    end
+
+    def post_to_slack?
+      @post_to_slack
+    end
+
+    def verbose?
+      @verbaose
+    end
+  end
+
+  # @return [Miteru::Configuration] Miteru's current configuration
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  # Set Miteru's configuration
+  # @param config [Miteru::Configuration]
+  def self.configuration=(config)
+    @configuration = config
+  end
+
+  # Modify Miteru's current configuration
+  # @yieldparam [Miteru::Configuration] config current Miteru config
+  def self.configure
+    yield configuration
+  end
+end

data/lib/miteru/crawler.rb

@@ -6,50 +6,36 @@ require "uri"
 
 module Miteru
   class Crawler
-    attr_reader :ayashige
-    attr_reader :directory_traveling
     attr_reader :downloader
     attr_reader :feeds
-    attr_reader :size
-    attr_reader :threads
-    attr_reader :verbose
-
-    def initialize(auto_download: false, ayashige: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: Parallel.processor_count, verbose: false)
-      @auto_download = auto_download
-      @ayashige = ayashige
-      @directory_traveling = directory_traveling
-      @downloader = Downloader.new(download_to)
-      @size = size
-      @threads = threads
-      @verbose = verbose
-
-      @feeds = Feeds.new(size: size, ayashige: ayashige, directory_traveling: directory_traveling)
-      @notifier = Notifier.new(post_to_slack)
+
+    def initialize
+      @downloader = Downloader.new(Miteru.configuration.download_to)
+
+      @feeds = Feeds.new
+      @notifier = Notifier.new
+    end
+
+    def crawl(url)
+      website = Website.new(url)
+      downloader.download_kits(website.kits) if website.has_kits? && auto_download?
+      notify(website) if website.has_kits? || verbose?
+    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
+      nil
     end
 
     def execute
-      puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose
-
-      Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
-        website = Website.new(url)
-        downloader.download_kits(website.kits) if website.has_kits? && auto_download?
-        notify(website) if verbose || website.has_kits?
-      rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
-        next
+      threads = Miteru.configuration.threads
+      suspicious_urls = feeds.suspicious_urls
+      puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
+
+      Parallel.each(suspicious_urls, in_threads: threads) do |url|
+        crawl url
       end
     end
 
-    def self.execute(auto_download: false, ayashige: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: Parallel.processor_count, verbose: false)
-      new(
-        auto_download: auto_download,
-        ayashige: ayashige,
-        directory_traveling: directory_traveling,
-        download_to: download_to,
-        post_to_slack: post_to_slack,
-        size: size,
-        threads: threads,
-        verbose: verbose
-      ).execute
+    def self.execute
+      new.execute
     end
 
     def notify(website)
@@ -57,7 +43,11 @@ module Miteru
     end
 
     def auto_download?
-      @auto_download
+      Miteru.configuration.auto_download?
+    end
+
+    def verbose?
+      Miteru.configuration.verbose?
     end
   end
 end

data/lib/miteru/downloader.rb

@@ -38,7 +38,7 @@ module Miteru
     def download_filename(kit)
       domain = URI(kit.base_url).hostname
 
-      "#{domain}_#{kit.basename}_#{SecureRandom.alphanumeric(10)}#{kit.extname}"
+      "#{domain}_#{kit.filename}_#{SecureRandom.alphanumeric(10)}#{kit.extname}"
     end
 
     def filepath_to_download(filename)

data/lib/miteru/feeds.rb

@@ -6,14 +6,13 @@ require_relative "./feeds/urlscan"
 
 module Miteru
   class Feeds
-    def initialize(size: 100, ayashige: false, directory_traveling: false)
-      @feeds = [UrlScan.new(size)]
-      @feeds << Ayashige.new if ayashige
-      @directory_traveling = directory_traveling
+    def initialize
+      @feeds = [UrlScan.new(Miteru.configuration.size)]
+      @feeds << Ayashige.new if Miteru.configuration.ayashige?
     end
 
     def directory_traveling?
-      @directory_traveling
+      Miteru.configuration.directory_traveling?
     end
 
     def suspicious_urls

data/lib/miteru/feeds/ayashige.rb

@@ -20,7 +20,7 @@ module Miteru
             "http://#{domain}"
           ]
         end.flatten
-      rescue HTTPResponseError => e
+      rescue HTTPResponseError, JSON::ParserError => e
         puts "Failed to load ayashige feed (#{e})"
         []
       end

data/lib/miteru/feeds/urlscan.rb

@@ -25,7 +25,7 @@ module Miteru
 
         res = JSON.parse(get(url))
         res["results"].map { |result| result.dig("task", "url") }
-      rescue HTTPResponseError => e
+      rescue HTTPResponseError, JSON::ParserError => e
         puts "Failed to load urlscan.io feed (#{e})"
         []
       end

data/lib/miteru/kit.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "cgi"
+
 module Miteru
   class Kit
     VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
@@ -25,6 +27,10 @@ module Miteru
       File.basename(link)
     end
 
+    def filename
+      CGI.unescape basename
+    end
+
     def url
       "#{base_url}/#{basename}"
     end

data/lib/miteru/notifier.rb

@@ -1,23 +1,16 @@
 # frozen_string_literal: true
 
 require "colorize"
-require "slack/incoming/webhooks"
+require "slack-notifier"
 
 module Miteru
   class Notifier
-    def initialize(post_to_slack = false)
-      @post_to_slack = post_to_slack
-    end
-
     def notify(url:, kits:, message:)
       attachement = Attachement.new(url)
 
       if post_to_slack? && !kits.empty?
-        slack = Slack::Incoming::Webhooks.new(slack_webhook_url, channel: slack_channel)
-        slack.post(
-          message,
-          attachments: attachement.to_a
-        )
+        notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
+        notifier.post(text: message, attachments: attachement.to_a)
       end
 
       message = message.colorize(:light_red) unless kits.empty?
@@ -25,7 +18,7 @@ module Miteru
     end
 
     def post_to_slack?
-      @post_to_slack && slack_webhook_url?
+      slack_webhook_url? && Miteru.configuration.post_to_slack?
     end
 
     def slack_webhook_url

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.8"
+  VERSION = "0.12.9"
 end

data/lib/miteru/website.rb

@@ -41,7 +41,7 @@ module Miteru
     def message
       return "It doesn't contain a phishing kit." unless kits?
 
-      kit_names = kits.map(&:basename).join(", ")
+      kit_names = kits.map(&:filename).join(", ")
       noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
       "It might contain #{noun}: #{kit_names}."
     end

data/miteru.gemspec

@@ -37,6 +37,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "http", "~> 4.1"
   spec.add_dependency "oga", "~> 2.15"
   spec.add_dependency "parallel", "~> 1.17"
-  spec.add_dependency "slack-incoming-webhooks", "~> 0.2"
+  spec.add_dependency "slack-notifier", "~> 2.3"
   spec.add_dependency "thor", "~> 0.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.8
+  version: 0.12.9
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-24 00:00:00.000000000 Z
+date: 2019-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -179,19 +179,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.17'
 - !ruby/object:Gem::Dependency
-  name: slack-incoming-webhooks
+  name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -228,6 +228,7 @@ files:
 - lib/miteru.rb
 - lib/miteru/attachement.rb
 - lib/miteru/cli.rb
+- lib/miteru/configuration.rb
 - lib/miteru/crawler.rb
 - lib/miteru/downloader.rb
 - lib/miteru/error.rb