miteru 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b6ffb792e263519ab645ef14235c5fb6cfa08919375bbaf2064528515dacd43
-  data.tar.gz: a67289780e2d42dfbbcd0a4b8142487e3ba22c5aa5addeef5707fe107161ecb5
+  metadata.gz: aa876b402ad3fcfe44fb1a66035bdb7649db11383650faf572d94fb8b4c8f498
+  data.tar.gz: c8ffde9ad933fbe670900d8b6547f5ce35ba7f85bd0a0d1201d3c00518d612bb
 SHA512:
-  metadata.gz: 5e43966e7560b60cc4e09930ef9f390862747380419267b0135dc7b86edf3aec83f1fb18732e3553a8adb805fc08c70654327c89bf973c171d3aa1bc59c70b4b
-  data.tar.gz: 906207242b7cafbba681f19d73df74b4e3c5160d8982e0144cc5d2f30c53917593e3abde329b3666400f2f27db9055c55bef4f880df5f20cc973add7e077ea6c
+  metadata.gz: 15bc48dc53696348635d637674c5574d537b8cf220939dce354875706f1a58820b95ecab547da0106a72e76d026d3e067b14438bfbef7ce1ce971737491269e2
+  data.tar.gz: 86d0aad06740b2013c35ef3920fa27fc1ac811580c08103273f3c073362d37b0e2c8bd0e49c9d8aac63dd06004556b902c6bccaf57f7833670e01d8e907d72a1

data/README.md

@@ -36,6 +36,8 @@ Options:
   [--auto-download], [--no-auto-download]  # Enable or disable auto-download of *.zip file(s)
   [--download-to=DOWNLOAD_TO]              # Directory to download file(s)
                                            # Default: /tmp
+  [--size=N]                               # Number of urlscan.io's results. (Max: 100,000)
+                                           # Default: 100
   [--post-to-slack], [--no-post-to-slack]  # Post a message to Slack if it detects a phishing kit
   [--verbose], [--no-verbose]
                                            # Default: true

data/lib/miteru/cli.rb

@@ -8,15 +8,17 @@ module Miteru
   class CLI < Thor
     method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of *.zip file(s)"
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
+    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 100,000)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
     def execute
-      websites = Crawler.execute(options[:verbose])
+      websites = Crawler.execute(size: options[:size], verbose: options[:verbose])
       websites.each do |website|
         next unless website.has_kit?
 
-        puts "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')}).".colorize(:light_red)
+        message = "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')})."
+        puts message.colorize(:light_red)
         post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
         download_zip_files(website.url, website.zip_files, options[:download_to]) if options[:auto_download]
       end

data/lib/miteru/crawler.rb

@@ -6,17 +6,23 @@ require "http"
 module Miteru
   class Crawler
     attr_reader :threads
-    def initialize
+    attr_reader :size
+    attr_reader :verbose
+
+    def initialize(size: 100, verbose: false)
       @threads = 10
+      @size = size
+      @verbose = verbose
+      raise ArgumentError, "size must be less than 100,000" if size > 100_000
     end
 
     def suspicous_urls
-      url = "https://urlscan.io/api/v1/search/?q=certstream-suspicious"
+      url = "https://urlscan.io/api/v1/search/?q=certstream-suspicious&size=#{size}"
       res = JSON.parse(get(url))
       res["results"].map { |result| result.dig("task", "url") }
     end
 
-    def execute(verbose = false)
+    def execute
       pool = Thread.pool(threads)
       websites = []
 
@@ -32,8 +38,8 @@ module Miteru
       websites
     end
 
-    def self.execute(verbose = false)
-      new.execute(verbose)
+    def self.execute(size: 100, verbose: false)
+      new(size: size, verbose: verbose).execute
     end
 
     private

data/lib/miteru/http_client.rb

@@ -26,7 +26,7 @@ module Miteru
     end
 
     def get(url)
-      HTTP.headers(default_headers).get(url, default_options)
+      HTTP.timeout(write: 2, connect: 5, read: 10).headers(default_headers).get(url, default_options)
     end
 
     def self.get(url)

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-20 00:00:00.000000000 Z
+date: 2018-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler