domains_scanner 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea761f85e758619f9d262dc93c6d28ed3b528788
-  data.tar.gz: 9cf162281540446d281a98ec9f7b596215fdba47
+  metadata.gz: 3feae4927a04afcb710018b1823d23b7f0144137
+  data.tar.gz: c4299771c9de14510a584c2f3007fa4497249254
 SHA512:
-  metadata.gz: 9a39d231cd4afaf155659605ee0f04faa920f52d2b1b4c0af5fb545b0e5fcb634b58911a0a01986d5ebbf6430a261c350b5921c798fe4789566de7c07ebe9729
-  data.tar.gz: aa736332989a71b16ed09465375d29906eee1ddd1f1219b1582e3cd3a1e2e26e39c2ab44de2b3d58cecee064e1447e10791cb121b02a83cc3f933d95c9ebbc13
+  metadata.gz: cc7889eed1197a502b31dcd5d58041eac9d746fa88f584bb543ceb9f124d919eaa1440efb2346e1b6271fdf34f280154f2a4729a765736f5b0c5fe380f1df15c
+  data.tar.gz: 9f53a04ffc8ad5d2caba6216d789e5f9c0757c32ecfd612283c545585499598c27c576645ec8c4a71179a06795242ec57a5f415e4efa70ab066ef25dac02f77c

data/.gitignore

@@ -3,7 +3,6 @@
 /Gemfile.lock
 /_yardoc/
 /coverage/
-/doc/
 /pkg/
 /spec/reports/
 /tmp/

data/README.md

@@ -1,8 +1,8 @@
 # DomainsScanner
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/domains_scanner`. To experiment with that code, run `bin/console` for an interactive prompt.
+Search possible sub domains according to specified domain. DomainsScanner default use `site:*.{domain}.{top_level_domain}` search syntax to search sites available from baidu and google searchs.
 
-TODO: Delete this and the text above, and describe your gem
+![Preview Screenshot](./doc/images/preview.png)
 
 ## Installation
 
@@ -22,11 +22,22 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```sh
+$ domains_scanner -h
+Usage: domains_scanner [options] domain_name
 
-## Development
+Specific options:
+    -v, --[no-]verbose               Run verbosely, default: false
+        --top-domains=               search top level domains, split by comma, default: ["com", "cn", "com.cn", "net", "org", "ltd", "cc", "mobi", "live", "io", "co", "me", "hk"]
+    -e, --engines=                   search engines, split by comma, default: [google, baidu]
+        --max-page=                  Maximum number of pages to scan, default: 20
+
+Common options:
+    -h, --help                       Show this message
+        --version                    Show version
+```
 
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+## Development
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/lib/domains_scanner/crawlers/baidu.rb

@@ -1,16 +1,12 @@
 module DomainsScanner
   module Crawlers
     class Baidu < Base
-      def search(domain_name, top_level_domain, page = 1)
-        set_user_agent
-        query = search_keyword(domain_name, top_level_domain)
-        start = (page - 1) * 10
-        doc = agent.get("https://www.baidu.com/s?wd=#{query}&pn=#{start}")
-
-        results = parse_results(doc)
-        have_next_page = have_next_page?(doc)
+      def host
+        "https://www.baidu.com"
+      end
 
-        DomainsScanner::Results.new(results, have_next_page)
+      def keyword_field_name
+        "wd"
       end
 
       # [{title: "xxx", url: "xxx"}, ...]
@@ -33,8 +29,8 @@ module DomainsScanner
         end
       end
 
-      def have_next_page?(doc)
-        doc.search("#page strong+a").any?
+      def next_page_link_selector
+        "#page strong+a"
       end
     end
   end

data/lib/domains_scanner/crawlers/base.rb

@@ -7,23 +7,55 @@ module DomainsScanner
       end
 
       def agent
-        @agent ||= Mechanize.new
+        @agent ||= Mechanize.new do |agent|
+          agent.user_agent_alias = "Mac Safari"
+        end
       end
 
-      def set_user_agent
-        agent.user_agent_alias = available_agent_alias.sample
+      def search_by_form(domain_name, top_level_domain)
+        doc = agent.get(host)
+
+        form = doc.forms.first
+        query = search_keyword(domain_name, top_level_domain)
+        form[keyword_field_name] = query
+        doc = form.submit
+
+        results = parse_results(doc)
+        next_page_link = parse_next_page_link(doc)
+
+        DomainsScanner::Results.new(results, next_page_link)
       end
 
-      def available_agent_alias
-        @available_agent_alias ||= Mechanize::AGENT_ALIASES.keys - ['Mechanize']
+      def search_by_link(link)
+        doc = agent.get(link)
+        results = parse_results(doc)
+        next_page_link = parse_next_page_link(doc)
+
+        DomainsScanner::Results.new(results, next_page_link)
+      end
+
+      def parse_next_page_link(doc)
+        next_page_tag = doc.search(next_page_link_selector).first
+        return unless next_page_tag
+
+        href = next_page_tag.attributes["href"]
+        "#{host}#{href}"
       end
 
       def search_keyword(domain_name, top_level_domain)
         "site:*.#{domain_name}.#{top_level_domain}"
       end
 
-      def search(domain_name, top_level_domain, page = 1)
-        raise NotImplementedError, "#{self.class.name}#search need to be implmented in sub class"
+      def keyword_field_name
+        raise NotImplementedError
+      end
+
+      def parse_results(doc)
+        raise NotImplementedError
+      end
+
+      def have_next_page?(doc)
+        raise NotImplementedError
       end
     end
   end

data/lib/domains_scanner/crawlers/google.rb

@@ -1,16 +1,12 @@
 module DomainsScanner
   module Crawlers
     class Google < Base
-      def search(domain_name, top_level_domain, page = 1)
-        set_user_agent
-        query = search_keyword(domain_name, top_level_domain)
-        start = (page - 1) * 10
-        doc = agent.get("https://google.com/search?q=#{query}&start=#{start}")
-
-        results = parse_results(doc)
-        have_next_page = have_next_page?(doc)
+      def host
+        "https://google.com"
+      end
 
-        DomainsScanner::Results.new(results, have_next_page)
+      def keyword_field_name
+        "q"
       end
 
       # [{title: "xxx", url: "xxx"}, ...]
@@ -28,8 +24,8 @@ module DomainsScanner
         end
       end
 
-      def have_next_page?(doc)
-        doc.search("div#foot .cur+td").any?
+      def next_page_link_selector
+        "div#foot .cur+td>a"
       end
     end
   end

data/lib/domains_scanner/results.rb

@@ -2,13 +2,13 @@ require 'domains_scanner/result_item'
 
 module DomainsScanner
   class Results
-    attr_reader :items, :have_next_page
+    attr_reader :items, :next_page_link
 
-    def initialize(results, have_next_page)
+    def initialize(results, next_page_link)
       @items = results.map do |result|
         ResultItem.new(title: result[:title], url: result[:url])
       end
-      @have_next_page = have_next_page
+      @next_page_link = next_page_link
     end
   end
 end

data/lib/domains_scanner/runner.rb

@@ -13,24 +13,32 @@ module DomainsScanner
       @workers = DomainsScanner.engines.map do |engine|
         crawler = DomainsScanner::Crawlers.build(engine)
         page = 1
+        next_page_link = nil
 
         Thread.new do
           loop do
             puts "Scanning #{domain} with #{engine} on page: #{page}" if DomainsScanner.verbose
 
             begin
-              results = crawler.search(@domain_word, @top_level_domain, page)
+              if page == 1
+                puts "Search by form>>>>" if DomainsScanner.verbose
+                results = crawler.search_by_form(@domain_word, @top_level_domain)
+              else
+                puts "Search by link: #{next_page_link}>>>>" if DomainsScanner.verbose
+                results = crawler.search_by_link(next_page_link)
+              end
+              next_page_link = results.next_page_link
+
               results.items.each do |item|
-              DomainsScanner.output_queue.push({
-                  domain: item.host, top_level_domain: @top_level_domain, engine: engine
-                })
+                DomainsScanner.output_queue.push({
+                    domain: item.host, top_level_domain: @top_level_domain, engine: engine
+                  })
               end
-              break unless results.have_next_page
             rescue Mechanize::ResponseCodeError => e
               puts "search in #{engine} error, skip now" if DomainsScanner.verbose
             end
 
-            break unless page < DomainsScanner.max_page
+            break unless next_page_link && page < DomainsScanner.max_page
             page += 1
           end
         end

data/lib/domains_scanner/version.rb

@@ -1,3 +1,3 @@
 module DomainsScanner
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: domains_scanner
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Martin Hong
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-25 00:00:00.000000000 Z
+date: 2017-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,6 +86,7 @@ files:
 - README.md
 - Rakefile
 - bin/domains_scanner
+- doc/images/preview.png
 - domains_scanner.gemspec
 - lib/ansi_colors.rb
 - lib/domains_scanner.rb