miteru 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0961934a27b02e1571c875e3a5a1bed37e7069d5333b45ced0f2d20250c9423
-  data.tar.gz: 2172a57faed6cd18e7b00665f481821486606ab1407746bcf683bdea5e0829a4
+  metadata.gz: 415304dc7c41fb9f40f68e7b8a59f0c03ce74c963020ecd2300b4b3a1ed3c058
+  data.tar.gz: dcef1c8fe00901e5d97f7e4a12c3b63227e63a11e4c1a3498f768fbb7dc7ddd3
 SHA512:
-  metadata.gz: c100e58f709309e9d1dc1ed33cb660bcf7622bf4c8b5ec6186403571968d91a91e50bc22fb2087f662fa5dfb51e46b9c0688ed66d5758da69348a9955da0acec
-  data.tar.gz: 9efad43fa5a0e528d59c6b1c4ee16dd336b5378baf0c3bb21f6a9acafde84d16d806785968d7a5849e229a567c9e1e3f7c057cfd233a4ed40d0aa9ec67499a60
+  metadata.gz: 686844a3d0e5016d66ced3d66a153777d56bc9d5751820b7e41d5a74e1d10ba6fb1d6fc334d69c66545c5e60782fb6df1baaf765bb5ac0ea1d9c48ff2d72b017
+  data.tar.gz: 146cb5832df24866984bf65b4074e1a362db98394aa4fe6150bd941c1f13d28eeb2b1afda088e0d39eb404ad3faa6fd363fdf789fd6d6052736e5ef3f1e1163f

data/README.md

@@ -32,9 +32,12 @@ Usage:
   miteru execute
 
 Options:
+  [--auto-download], [--no-auto-download]
+  [--download-to=DOWNLOAD_TO]
+                                           # Default: /tmp
+  [--post-to-slack], [--no-post-to-slack]
   [--verbose], [--no-verbose]
                                            # Default: true
-  [--post-to-slack], [--no-post-to-slack]
 
 Execute the crawler
 ```

data/lib/miteru.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "miteru/error"
+require "miteru/downloader"
 require "miteru/website"
 require "miteru/crawler"
 require "miteru/cli"

data/lib/miteru/cli.rb

@@ -6,20 +6,40 @@ require "thor"
 
 module Miteru
   class CLI < Thor
-    method_option :verbose, type: :boolean, default: true
+    method_option :auto_download, type: :boolean, default: false
+    method_option :download_to, type: :string, default: "/tmp"
     method_option :post_to_slack, type: :boolean, default: false
+    method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
     def execute
       websites = Crawler.execute(options[:verbose])
       websites.each do |website|
-        if website.has_kit?
-          puts "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')}).".colorize(:light_red)
-          post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
+        next unless website.has_kit?
+
+        puts "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')}).".colorize(:light_red)
+        post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
+        begin
+          download_zip_files(website.url, website.zip_files, options[:download_to]) if options[:auto_download]
+        rescue DownloadError => e
+          puts e.to_s
         end
       end
     end
 
     no_commands do
+      def download_zip_files(url, zip_files, base_dir)
+        failed_urls = []
+        zip_files.each do |path|
+          target_url = "#{url}/#{path}"
+          begin
+            Downloader.download target_url, base_dir
+          rescue Down::Error => _
+            failed_urls << target_url
+          end
+        end
+        raise DownloadError, "Failed to download: #{failed_urls}.join(',')" unless failed_urls.empty?
+      end
+
       def valid_slack_setting?
         ENV["SLACK_WEBHOOK_URL"] != nil
       end

data/lib/miteru/downloader.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "down"
+require "uri"
+
+module Miteru
+  class Downloader
+    attr_reader :url, :base_dir
+    def initialize(url, base_dir = "/tmp")
+      @url = url
+      @base_dir = base_dir
+    end
+
+    def filename
+      uri = URI.parse(url)
+      File.basename(uri.path)
+    end
+
+    def destination
+      @destination ||= "#{base_dir}/#{filename}"
+    end
+
+    def download
+      Down.download(url, destination: destination)
+    end
+
+    def self.download(url, base_dir = "/tmp")
+      new(url, base_dir).download
+    end
+  end
+end

data/lib/miteru/error.rb

@@ -2,4 +2,5 @@
 
 module Miteru
   class HTTPResponseError < StandardError; end
+  class DownloadError < StandardError; end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/miteru.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.4"
 
   spec.add_dependency "colorize", "~> 0.8"
+  spec.add_dependency "down", "~> 4.5"
   spec.add_dependency "http", "~> 3.3"
   spec.add_dependency "oga", "~> 2.15"
   spec.add_dependency "thor", "~> 0.19"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Manabu Niseki
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: down
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
@@ -199,6 +213,7 @@ files:
 - lib/miteru.rb
 - lib/miteru/cli.rb
 - lib/miteru/crawler.rb
+- lib/miteru/downloader.rb
 - lib/miteru/error.rb
 - lib/miteru/version.rb
 - lib/miteru/website.rb