miteru 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8370829b3ad8da53b7c73070fd972269b09ea27c6c7dce1a3da5d855f81da27
-  data.tar.gz: c017feecfeaa27912de5dc898f23b430652270a16852510785109d60ed04891a
+  metadata.gz: 89ed6dea77f4809ef7cacfd7543a35d96f8032807d6cc93fe5e6aa52e752d5d1
+  data.tar.gz: 812d51bdbc1c245c87f7a3fcc7e70577536cc8eacd0cfc9d10b6be3b146b96d9
 SHA512:
-  metadata.gz: '0796be92ea1586ec57f12a4157097d65ea0d49e151eb8876eda3b406ca05908b0bc01ff021f270ab5a1467bda415e281909ef47502f1fb490d5285c69dd1bd9b'
-  data.tar.gz: edbc8c6cce3d85cbfb8bdac217d7ca0ee9d3dfc353a3b7a3c1c7f7ad132b7d9ee80960bda1bba638b859f53d2805a7634901d47fd9a913630b16b0552d463078
+  metadata.gz: cb94419c132e650a1e57f3bccd134b91be5847ab9197ceb69c71a08b8bfec3667a5e63f504892ef4d08a74f049f4bc979de4ac7c145a2a30502ed92b91f73c78
+  data.tar.gz: 60b012761f7eced3acf58a7acf4487b2d0466a9ebc2fa1b9fe9b23e3b315b3ed4d5bb798e1786788c904691656c989f7ddda962a52ca307abe4e762d7281ad65

data/.travis.yml

@@ -1,7 +1,6 @@
----
 sudo: false
 language: ruby
 cache: bundler
 rvm:
   - 2.5.1
-before_install: gem install bundler -v 1.16.4
+before_install: gem install bundler -v 1.17.1

data/README.md

@@ -42,7 +42,7 @@ Options:
   [--download-to=DOWNLOAD_TO]                          # Directory to download file(s)
                                                        # Default: /tmp
   [--post-to-slack], [--no-post-to-slack]              # Post a message to Slack if it detects a phishing kit
-  [--size=N]                                           # Number of urlscan.io's results. (Max: 100,000)
+  [--size=N]                                           # Number of urlscan.io's results. (Max: 10,000)
                                                        # Default: 100
   [--threads=N]                                        # Number of threads to use
                                                        # Default: 10

data/lib/miteru.rb

@@ -4,6 +4,7 @@ require "miteru/error"
 require "miteru/http_client"
 require "miteru/website"
 require "miteru/downloader"
+require "miteru/feeds"
 require "miteru/crawler"
 require "miteru/cli"
 require "miteru/version"

data/lib/miteru/cli.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "colorize"
-require "digest"
-require "fileutils"
-require "http"
 require "thor"
 
 module Miteru
@@ -12,7 +8,7 @@ module Miteru
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
-    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 100,000)"
+    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
     method_option :threads, type: :numeric, default: 10, desc: "Number of threads to use"
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"

data/lib/miteru/crawler.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-require "csv"
-require "http"
-require "json"
+require "colorize"
 require "parallel"
 require "uri"
 
@@ -11,14 +9,11 @@ module Miteru
     attr_reader :auto_download
     attr_reader :directory_traveling
     attr_reader :downloader
+    attr_reader :feeds
     attr_reader :size
     attr_reader :threads
     attr_reader :verbose
 
-    URLSCAN_ENDPOINT = "https://urlscan.io/api/v1"
-    OPENPHISH_ENDPOINT = "https://openphish.com"
-    PHISHTANK_ENDPOINT = "http://data.phishtank.com"
-
     def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
       @auto_download = auto_download
       @directory_traveling = directory_traveling
@@ -27,63 +22,14 @@ module Miteru
       @size = size
       @threads = threads
       @verbose = verbose
-      raise ArgumentError, "size must be less than 100,000" if size > 100_000
-    end
-
-    def urlscan_feed
-      url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
-      res = JSON.parse(get(url))
-      res["results"].map { |result| result.dig("task", "url") }
-    rescue HTTPResponseError => _
-      []
-    end
-
-    def openphish_feed
-      res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
-      res.lines.map(&:chomp)
-    rescue HTTPResponseError => _
-      []
-    end
-
-    def phishtank_feed
-      res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
-      table = CSV.parse(res, headers: true)
-      table.map { |row| row["url"] }
-    rescue HTTPResponseError => _
-      []
-    end
-
-    def breakdown(url)
-      begin
-        uri = URI.parse(url)
-      rescue URI::InvalidURIError => _
-        return []
-      end
-
-      base = "#{uri.scheme}://#{uri.hostname}"
-      return [base] unless directory_traveling
-
-      segments = uri.path.split("/")
-      return [base] if segments.length.zero?
 
-      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
-      urls.reject do |breakdowned_url|
-        # Reject a url which ends with specific extension names
-        %w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
-      end
-    end
-
-    def suspicious_urls
-      @suspicious_urls ||= [].tap do |arr|
-        urls = (urlscan_feed + openphish_feed + phishtank_feed).select { |url| url.start_with?("http://", "https://") }
-        urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
-      end
+      @feeds = Feeds.new(size, directory_traveling: directory_traveling)
     end
 
     def execute
-      puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
+      puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl." if verbose
 
-      Parallel.each(suspicious_urls, in_threads: threads) do |url|
+      Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
         website = Website.new(url)
         if website.has_kit?
           message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
@@ -115,7 +61,7 @@ module Miteru
       channel = ENV["SLACK_CHANNEL"] || "#general"
 
       payload = { text: message, channel: channel }
-      HTTP.post(webhook_url, json: payload)
+      HTTPClient.post(webhook_url, json: payload)
     end
 
     def post_to_slack?
@@ -130,13 +76,6 @@ module Miteru
       ENV["SLACK_WEBHOOK_URL"] != nil
     end
 
-    private
 
-    def get(url)
-      res = HTTP.follow(max_hops: 3).get(url)
-      raise HTTPResponseError if res.code != 200
-
-      res.body.to_s
-    end
   end
 end

data/lib/miteru/downloader.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "digest"
+require "fileutils"
+
 module Miteru
   class Downloader
     attr_reader :base_dir

data/lib/miteru/feeds.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "./feeds/feed"
+require_relative "./feeds/openphish"
+require_relative "./feeds/phishtank"
+require_relative "./feeds/urlscan"
+
+module Miteru
+  class Feeds
+    attr_reader :openphish, :phishtank, :urlscan
+    attr_reader :directory_traveling
+
+    def initialize(urlscan_size = 100, directory_traveling: false)
+      @openphish = OpenPhish.new
+      @phishtank = PhishTank.new
+      @urlscan = UrlScan.new(urlscan_size)
+      @directory_traveling = directory_traveling
+    end
+
+    def suspicious_urls
+      @suspicious_urls ||= [].tap do |arr|
+        urls = (openphish.urls + phishtank.urls + urlscan.urls).select { |url| url.start_with?("http://", "https://") }
+        urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
+      end
+    end
+
+    def breakdown(url)
+      begin
+        uri = URI.parse(url)
+      rescue URI::InvalidURIError => _
+        return []
+      end
+
+      base = "#{uri.scheme}://#{uri.hostname}"
+      return [base] unless directory_traveling
+
+      segments = uri.path.split("/")
+      return [base] if segments.length.zero?
+
+      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+      urls.reject do |breakdowned_url|
+        # Reject a url which ends with specific extension names
+        %w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
+      end
+    end
+  end
+end

data/lib/miteru/feeds/feed.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Miteru
+  class Feeds
+    class Feed
+      def urls
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      private
+
+      def get(url)
+        res = HTTPClient.get(url)
+        raise HTTPResponseError if res.code != 200
+
+        res.body.to_s
+      end
+    end
+  end
+end

data/lib/miteru/feeds/openphish.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Miteru
+  class Feeds
+    class OpenPhish < Feed
+      ENDPOINT = "https://openphish.com"
+
+      def urls
+        res = get("#{ENDPOINT}/feed.txt")
+        res.lines.map(&:chomp)
+      rescue HTTPResponseError => e
+        puts "Failed to load OpenPhish feed (#{e})"
+        []
+      end
+    end
+  end
+end

data/lib/miteru/feeds/phishtank.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Miteru
+  class Feeds
+    class PhishTank < Feed
+      ENDPOINT = "http://data.phishtank.com"
+
+      def urls
+        res = get("#{ENDPOINT}/data/online-valid.csv")
+        table = CSV.parse(res, headers: true)
+        table.map { |row| row["url"] }
+      rescue HTTPResponseError => e
+        puts "Failed to load PhishTank feed (#{e})"
+        []
+      end
+    end
+  end
+end

data/lib/miteru/feeds/urlscan.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Miteru
+  class Feeds
+    class UrlScan < Feed
+      ENDPOINT = "https://urlscan.io/api/v1"
+
+      attr_reader :size
+      def initialize(size = 100)
+        @size = size
+        raise ArgumentError, "size must be less than 10,000" if size > 10_000
+      end
+
+      def urls
+        url = "#{ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
+        res = JSON.parse(get(url))
+        res["results"].map { |result| result.dig("task", "url") }
+      rescue HTTPResponseError => e
+        puts "Failed to load urlscan.io feed (#{e})"
+        []
+      end
+    end
+  end
+end

data/lib/miteru/http_client.rb

@@ -25,12 +25,21 @@ module Miteru
       new.download(url, base_dir)
     end
 
-    def get(url)
-      HTTP.timeout(write: 2, connect: 5, read: 10).headers(default_headers).get(url, default_options)
+    def get(url, options = {})
+      options = options.merge default_options
+      HTTP.follow.timeout(write: 2, connect: 5, read: 10).headers(default_headers).get(url, options)
     end
 
-    def self.get(url)
-      new.get url
+    def self.get(url, options = {})
+      new.get url, options
+    end
+
+    def post(url, options = {})
+      HTTP.post url, options
+    end
+
+    def self.post(url, options = {})
+      new.post url, options
     end
 
     private

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.9.2"
+  VERSION = "0.9.3"
 end

data/miteru.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "bundler", "~> 1.17"
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 12.3"
@@ -33,8 +33,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.4"
 
   spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "down", "~> 4.5"
-  spec.add_dependency "http", "~> 3.3"
+  spec.add_dependency "down", "~> 4.6"
+  spec.add_dependency "http", "~> 4.0"
   spec.add_dependency "oga", "~> 2.15"
   spec.add_dependency "parallel", "~> 1.12"
   spec.add_dependency "thor", "~> 0.19"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-21 00:00:00.000000000 Z
+date: 2018-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -128,28 +128,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.5'
+        version: '4.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.5'
+        version: '4.6'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
@@ -215,6 +215,11 @@ files:
 - lib/miteru/crawler.rb
 - lib/miteru/downloader.rb
 - lib/miteru/error.rb
+- lib/miteru/feeds.rb
+- lib/miteru/feeds/feed.rb
+- lib/miteru/feeds/openphish.rb
+- lib/miteru/feeds/phishtank.rb
+- lib/miteru/feeds/urlscan.rb
 - lib/miteru/http_client.rb
 - lib/miteru/version.rb
 - lib/miteru/website.rb