domains_scanner 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3feae4927a04afcb710018b1823d23b7f0144137
-  data.tar.gz: c4299771c9de14510a584c2f3007fa4497249254
+  metadata.gz: 43825859b85042d6bfb4b4bb8add80e1566578b4
+  data.tar.gz: 24b7b4e631d9c236056f0df938a7f152da283727
 SHA512:
-  metadata.gz: cc7889eed1197a502b31dcd5d58041eac9d746fa88f584bb543ceb9f124d919eaa1440efb2346e1b6271fdf34f280154f2a4729a765736f5b0c5fe380f1df15c
-  data.tar.gz: 9f53a04ffc8ad5d2caba6216d789e5f9c0757c32ecfd612283c545585499598c27c576645ec8c4a71179a06795242ec57a5f415e4efa70ab066ef25dac02f77c
+  metadata.gz: 3358a63817ca881e798f3209d490242aa45ca60aa74cf266c48fe1c769b8b9064005169fa2db36c1ef290a37ec9b87093488c8ae35ba3eead4956dd50a79f7e8
+  data.tar.gz: 100ad14ca19fbc6b95902197a1f3d6431cf0f1c49273245d848bb9d4d4c8b9ea28f7feec07e90dadd50f1edb5ee3a5b0d7121462950b1409603424a07fb17804

data/bin/domains_scanner

@@ -2,7 +2,6 @@
 
 require 'optparse'
 require 'domains_scanner'
-require 'ansi_colors'
 
 PRINT_LINE_LENGTH=80
 def print_list(list)

data/domains_scanner.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = %q{search possible domains for specified keyword}
   spec.description   = %q{search possible domains by search engines and other tools}
-  spec.homepage      = "https://rubygems.org/gems/domains_scanner"
+  spec.homepage      = "https://github.com/hackerpie/domains_scanner"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }

data/lib/domains_scanner.rb

@@ -1,6 +1,8 @@
 require 'uri'
 require 'thread'
+require 'ansi_colors'
 require "domains_scanner/version"
+require "domains_scanner/printer"
 require "domains_scanner/crawlers"
 require "domains_scanner/runner"
 
@@ -25,7 +27,7 @@ module DomainsScanner
       end
       runners.each(&:run)
 
-      puts "Start analysis crawl results..." if DomainsScanner.verbose
+      DomainsScanner::Printer.puts { "Start analysis crawl results...".green }
       analysis_output_queue
     end
 

data/lib/domains_scanner/crawlers/base.rb

@@ -3,9 +3,6 @@ require 'domains_scanner/results'
 module DomainsScanner
   module Crawlers
     class Base
-      def initialize
-      end
-
       def agent
         @agent ||= Mechanize.new do |agent|
           agent.user_agent_alias = "Mac Safari"

data/lib/domains_scanner/printer.rb

@@ -0,0 +1,12 @@
+module DomainsScanner
+  class Printer
+    class << self
+      def puts(message = nil)
+        if DomainsScanner.verbose
+          message = yield if block_given?
+          Kernel.puts message if message
+        end
+      end
+    end
+  end
+end

data/lib/domains_scanner/runner.rb

@@ -9,7 +9,7 @@ module DomainsScanner
 
     def run
       domain = "#{@domain_word}.#{@top_level_domain}"
-      puts "Start scan #{domain}" if DomainsScanner.verbose
+      DomainsScanner::Printer.puts { "Start scan #{domain}".green }
       @workers = DomainsScanner.engines.map do |engine|
         crawler = DomainsScanner::Crawlers.build(engine)
         page = 1
@@ -17,25 +17,30 @@ module DomainsScanner
 
         Thread.new do
           loop do
-            puts "Scanning #{domain} with #{engine} on page: #{page}" if DomainsScanner.verbose
+            DomainsScanner::Printer.puts { "Scanning #{domain} with #{engine} on page: #{page}".yellow }
 
             begin
               if page == 1
-                puts "Search by form>>>>" if DomainsScanner.verbose
+                DomainsScanner::Printer.puts { "Search by form>>" }
                 results = crawler.search_by_form(@domain_word, @top_level_domain)
               else
-                puts "Search by link: #{next_page_link}>>>>" if DomainsScanner.verbose
+                DomainsScanner::Printer.puts { "Search by link: #{next_page_link}>>" }
                 results = crawler.search_by_link(next_page_link)
               end
               next_page_link = results.next_page_link
 
               results.items.each do |item|
-                DomainsScanner.output_queue.push({
-                    domain: item.host, top_level_domain: @top_level_domain, engine: engine
-                  })
+                # Avoid some unexpected hostname from ads
+                if item.host.to_s.include?(domain)
+                  DomainsScanner.output_queue.push({
+                      domain: item.host, top_level_domain: @top_level_domain, engine: engine
+                    })
+                else
+                  DomainsScanner::Printer.puts { "Abondon unexpected domain: #{item.host}".red }
+                end
               end
             rescue Mechanize::ResponseCodeError => e
-              puts "search in #{engine} error, skip now" if DomainsScanner.verbose
+              DomainsScanner::Printer.puts { "Search in #{engine} error, skip now! Reason: #{e.message}".red  }
             end
 
             break unless next_page_link && page < DomainsScanner.max_page

data/lib/domains_scanner/version.rb

@@ -1,3 +1,3 @@
 module DomainsScanner
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: domains_scanner
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Martin Hong
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-05 00:00:00.000000000 Z
+date: 2017-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,11 +94,12 @@ files:
 - lib/domains_scanner/crawlers/baidu.rb
 - lib/domains_scanner/crawlers/base.rb
 - lib/domains_scanner/crawlers/google.rb
+- lib/domains_scanner/printer.rb
 - lib/domains_scanner/result_item.rb
 - lib/domains_scanner/results.rb
 - lib/domains_scanner/runner.rb
 - lib/domains_scanner/version.rb
-homepage: https://rubygems.org/gems/domains_scanner
+homepage: https://github.com/hackerpie/domains_scanner
 licenses:
 - MIT
 metadata: {}