miteru 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7802cd8b2ddf76ba7b2401a1be5d7c1855a49d7a15e7ae95134e717bf111611
-  data.tar.gz: b782d4432a8f3d9fe2074a758c05828878501b1fb02a8ccfee8c15737eac68bd
+  metadata.gz: b0961934a27b02e1571c875e3a5a1bed37e7069d5333b45ced0f2d20250c9423
+  data.tar.gz: 2172a57faed6cd18e7b00665f481821486606ab1407746bcf683bdea5e0829a4
 SHA512:
-  metadata.gz: 7ec5226ab2cbaacc91a40a646c60e50370dd716010bdf6e3c76381213b0fb8d0b2936340c53b714c94c3f8e23ca8382566e8057724e218b407008fed66034cbb
-  data.tar.gz: b18ddbb18399f5cef0c68d16268ff36159787c4478f79e76a864cb782c6715272c799a04c5716dbf431de210d15db426f6abd6301e9b267db5755b1ef266998e
+  metadata.gz: c100e58f709309e9d1dc1ed33cb660bcf7622bf4c8b5ec6186403571968d91a91e50bc22fb2087f662fa5dfb51e46b9c0688ed66d5758da69348a9955da0acec
+  data.tar.gz: 9efad43fa5a0e528d59c6b1c4ee16dd336b5378baf0c3bb21f6a9acafde84d16d806785968d7a5849e229a567c9e1e3f7c057cfd233a4ed40d0aa9ec67499a60

data/README.md

@@ -25,3 +25,25 @@ Commands:
   miteru execute         # Execute the crawler
   miteru help [COMMAND]  # Describe available commands or one specific command
 ```
+
+```sh
+$ miteru help execute
+Usage:
+  miteru execute
+
+Options:
+  [--verbose], [--no-verbose]
+                                           # Default: true
+  [--post-to-slack], [--no-post-to-slack]
+
+Execute the crawler
+```
+
+```sh
+$ miteru execute
+...
+https://dummy1.com: it doesn't contain a phishing kit.
+https://dummy2.com: it doesn't contain a phishing kit.
+https://dummy3.com: it doesn't contain a phishing kit.
+https://dummy4.com: it might contain a phishing kit (dummy.zip).
+```

data/lib/miteru/cli.rb

@@ -1,18 +1,21 @@
 # frozen_string_literal: true
 
+require "colorize"
 require "http"
 require "thor"
 
 module Miteru
   class CLI < Thor
+    method_option :verbose, type: :boolean, default: true
     method_option :post_to_slack, type: :boolean, default: false
     desc "execute", "Execute the crawler"
     def execute
-      results = Crawler.execute
-      results.each do |result|
-        message = "#{result} might contain a phishing kit."
-        puts message
-        post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
+      websites = Crawler.execute(options[:verbose])
+      websites.each do |website|
+        if website.has_kit?
+          puts "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')}).".colorize(:light_red)
+          post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
+        end
       end
     end
 

data/lib/miteru/crawler.rb

@@ -16,31 +16,24 @@ module Miteru
       res["results"].map { |result| result.dig("task", "url") }
     end
 
-    def has_kit?(url)
-      begin
-        res = get(url)
-      rescue HTTPResponseError => _
-        false
-      end
-
-      rules = ["Index of", ".zip"]
-      rules.all? { |rule| res.include? rule }
-    end
-
-    def execute
+    def execute(verbose = false)
       pool = Thread.pool(threads)
-      results = []
+      websites = []
 
       suspicous_urls.each do |url|
-        pool.process { results << url if has_kit?(url) }
+        pool.process do
+          website = Website.new(url)
+          puts "#{website.url}: it doesn't contain a phishing kit." if verbose && !website.has_kit?
+          websites << website
+        end
       end
       pool.shutdown
 
-      results
+      websites
     end
 
-    def self.execute
-      new.execute
+    def self.execute(verbose = false)
+      new.execute(verbose)
     end
 
     private

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/miteru/website.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "http"
+require "oga"
+
+module Miteru
+  class Website
+    attr_reader :url
+    def initialize(url)
+      @url = url
+      build
+    end
+
+    def title
+      doc.at_css("title")&.text
+    end
+
+    def zip_files
+      @zip_files ||= doc.css("a").map do |a|
+        href = a.get("href")
+        href&.end_with?(".zip") ? href : nil
+      end.compact
+    end
+
+    def ok?
+      response.code == 200
+    end
+
+    def index?
+      title == "Index of /"
+    end
+
+    def zip_files?
+      !zip_files.empty?
+    end
+
+    def has_kit?
+      ok? && index? && zip_files?
+    end
+
+    def build
+      doc
+    end
+
+    private
+
+    def response
+      @response ||= get
+    end
+
+    def get
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      HTTP.get(url, ssl_context: ctx)
+    end
+
+    def doc
+      @doc ||= Oga.parse_html(response.body.to_s)
+    end
+  end
+end

data/lib/miteru.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "miteru/error"
+require "miteru/website"
 require "miteru/crawler"
 require "miteru/cli"
 require "miteru/version"

data/miteru.gemspec

@@ -26,12 +26,15 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 12.3"
   spec.add_development_dependency "rspec", "~> 3.8"
   spec.add_development_dependency "vcr", "~> 4.0"
   spec.add_development_dependency "webmock", "~> 3.4"
 
+  spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "http", "~> 3.3"
+  spec.add_dependency "oga", "~> 2.15"
   spec.add_dependency "thor", "~> 0.19"
   spec.add_dependency "thread", "~> 0.2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-17 00:00:00.000000000 Z
+date: 2018-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: glint
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.15'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -159,6 +201,7 @@ files:
 - lib/miteru/crawler.rb
 - lib/miteru/error.rb
 - lib/miteru/version.rb
+- lib/miteru/website.rb
 - miteru.gemspec
 homepage: https://github.com/ninoseki/miteru
 licenses: