domains_scanner 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ea761f85e758619f9d262dc93c6d28ed3b528788
+  data.tar.gz: 9cf162281540446d281a98ec9f7b596215fdba47
+SHA512:
+  metadata.gz: 9a39d231cd4afaf155659605ee0f04faa920f52d2b1b4c0af5fb545b0e5fcb634b58911a0a01986d5ebbf6430a261c350b5921c798fe4789566de7c07ebe9729
+  data.tar.gz: aa736332989a71b16ed09465375d29906eee1ddd1f1219b1582e3cd3a1e2e26e39c2ab44de2b3d58cecee064e1447e10791cb121b02a83cc3f933d95c9ebbc13

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.byebug_history
+/domains_scanner-*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in domains_scanner.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Martin Hong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# DomainsScanner
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/domains_scanner`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'domains_scanner'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install domains_scanner
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/HackerPie/domains_scanner.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/domains_scanner

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'domains_scanner'
+require 'ansi_colors'
+
+PRINT_LINE_LENGTH=80
+def print_list(list)
+  list.each do |group_name, group|
+    puts "Found domains for .#{group_name}\n#{"-" * 50}".green
+    group.each do |domain, engines|
+      tips_para = "Search Engine: #{engines.join(", ")}"
+      points_count = [PRINT_LINE_LENGTH - domain.length - tips_para.length - 2, 0].max
+      puts "#{domain.dup.red} #{'·' * points_count} #{tips_para.red}"
+    end
+    puts ""
+  end;nil
+end
+
+top_level_domains = nil
+OptionParser.new do |opts|
+  opts.banner = "Usage: domains_scanner [options] domain_name"
+  opts.separator ""
+  opts.separator "Specific options:"
+
+  opts.on("-v", "--[no-]verbose", "Run verbosely, default: false") do |v|
+    DomainsScanner.verbose = true
+  end
+
+  opts.on("--top-domains=", "search top level domains, split by comma, default: #{DomainsScanner::TOP_LEVEL_DOMAINS}") do |t|
+    puts "Manually configured top_level_domains=#{t.split(",")}"
+    top_level_domains = t.split(",")
+  end
+
+  opts.on("-e", "--engines=", "search engines, split by comma, default: [google, baidu]") do |engines|
+    puts "Manually configured engines=#{engines.split(",")}"
+    DomainsScanner.engines = engines.split(",")
+  end
+
+  opts.on("--max-page=", "Maximum number of pages to scan, default: 20") do |p|
+    puts "Manually configured max_page=#{p}"
+    DomainsScanner.max_page = p.to_i
+  end
+
+  opts.separator ""
+  opts.separator "Common options:"
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+  # Another typical switch to print the version.
+  opts.on_tail("--version", "Show version") do
+    puts DomainsScanner::VERSION
+    exit
+  end
+end.parse!
+domain_word = ARGV.pop
+
+list = DomainsScanner.scan(domain_word: domain_word, top_level_domains: top_level_domains)
+print_list(list)

data/domains_scanner.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'domains_scanner/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "domains_scanner"
+  spec.version       = DomainsScanner::VERSION
+  spec.authors       = ["Martin Hong"]
+  spec.email         = ["hongzeqin@gmail.com"]
+
+  spec.summary       = %q{search possible domains for specified keyword}
+  spec.description   = %q{search possible domains by search engines and other tools}
+  spec.homepage      = "https://rubygems.org/gems/domains_scanner"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   << 'domains_scanner'
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency 'byebug', '~> 9.1'
+
+  spec.add_dependency 'mechanize', '~> 2.7', '>= 2.7.5'
+end

data/lib/ansi_colors.rb

@@ -0,0 +1,59 @@
+# https://gist.githubusercontent.com/Martin91/3e21412209f8ee5e6112/raw/deb59949a92c400b04c61f53913970f7e91e614a/ansi_colors.rb
+# ansi_colors.rb
+# Implement add ansi color escape code to normal string
+#
+# Created by Martin Hong on 1/17/2015
+# Copyright (c) 2015 Martin Hong. All rights reserved.
+#
+
+String.class_eval do
+  # ==== Start colors with ANSI escape code ====
+  #
+  # ANSI colors excape code format:
+  # => Color=\033[code;前景;背景m
+  #
+  # More details can be retrieved from: http://en.wikipedia.org/wiki/ANSI_escape_code
+  #
+  # ANSI
+  #   Foreground        Background       Color
+  #   ---------------------------------------
+  #   30                40               Black
+  #   31                41               Red
+  #   32                42               Green
+  #   33                43               Yellow
+  #   34                44               Blue
+  #   35                45               Purple
+  #   36                46               Blue-Green
+  #   37                47               White
+  #   1                 1                Transparent
+  #
+  #   Code              Function
+  #   -------------------------
+  #   0                 OFF
+  #   1                 Highline
+  #   4                 Underline
+  #   5                 Flash
+  #   7                 Reverse Display
+  #   8                 Invisiable
+  #
+  RED = "\033[0;31;1m"            # Dangerous Color
+  YELLOW = "\033[0;33;1m"         # Warning Color
+  GREEN = "\033[0;32;1m"          # Infos Color
+  DEFAULT = "\033[0;37;1m"        # RESET COLOR
+  # ====  End colors with ANSI escape code  ====
+
+  # Usages:
+  #
+  #   "Hello world".red
+  #   "Hello world".green
+  #   "Hello world".yellow
+  #
+  %w(red yellow green).each do |method|
+    unless method_defined?(method)
+      define_method method do
+        prepend self.class.const_get(method.upcase)
+        concat DEFAULT
+      end
+    end
+  end
+end

data/lib/domains_scanner/crawlers/baidu.rb

@@ -0,0 +1,41 @@
+module DomainsScanner
+  module Crawlers
+    class Baidu < Base
+      def search(domain_name, top_level_domain, page = 1)
+        set_user_agent
+        query = search_keyword(domain_name, top_level_domain)
+        start = (page - 1) * 10
+        doc = agent.get("https://www.baidu.com/s?wd=#{query}&pn=#{start}")
+
+        results = parse_results(doc)
+        have_next_page = have_next_page?(doc)
+
+        DomainsScanner::Results.new(results, have_next_page)
+      end
+
+      # [{title: "xxx", url: "xxx"}, ...]
+      def parse_results(doc)
+        items = doc.search(".result")
+        items.map do |i|
+          title = i.search("h3.t > a").text
+          # Baidu encrypted the target url, so we can use show url only, but it is enough!
+          # bbs.abc.net/for...php?...
+          show_url = i.search("div:last-child > a.c-showurl")
+          url = if show_url
+            if show_url.text.start_with?("http")
+              show_url.text
+            else
+              "http://#{show_url.text}"
+            end
+          end
+
+          { title: i.text, url: URI.encode(url) }
+        end
+      end
+
+      def have_next_page?(doc)
+        doc.search("#page strong+a").any?
+      end
+    end
+  end
+end

data/lib/domains_scanner/crawlers/base.rb

@@ -0,0 +1,30 @@
+require 'domains_scanner/results'
+
+module DomainsScanner
+  module Crawlers
+    class Base
+      def initialize
+      end
+
+      def agent
+        @agent ||= Mechanize.new
+      end
+
+      def set_user_agent
+        agent.user_agent_alias = available_agent_alias.sample
+      end
+
+      def available_agent_alias
+        @available_agent_alias ||= Mechanize::AGENT_ALIASES.keys - ['Mechanize']
+      end
+
+      def search_keyword(domain_name, top_level_domain)
+        "site:*.#{domain_name}.#{top_level_domain}"
+      end
+
+      def search(domain_name, top_level_domain, page = 1)
+        raise NotImplementedError, "#{self.class.name}#search need to be implmented in sub class"
+      end
+    end
+  end
+end

data/lib/domains_scanner/crawlers/google.rb

@@ -0,0 +1,36 @@
+module DomainsScanner
+  module Crawlers
+    class Google < Base
+      def search(domain_name, top_level_domain, page = 1)
+        set_user_agent
+        query = search_keyword(domain_name, top_level_domain)
+        start = (page - 1) * 10
+        doc = agent.get("https://google.com/search?q=#{query}&start=#{start}")
+
+        results = parse_results(doc)
+        have_next_page = have_next_page?(doc)
+
+        DomainsScanner::Results.new(results, have_next_page)
+      end
+
+      # [{title: "xxx", url: "xxx"}, ...]
+      def parse_results(doc)
+        items = doc.search(".g h3.r a")
+        items.map do |i|
+          title = i.text
+          href = i.attributes["href"] && i.attributes["href"].value
+          # https://bbs.abc.net/thread-144889-1-1.html&sa=U&ved=0ahUKEwjpmNT0ltnXAhXMxLwKHQJIAmE4ChAWCBQwAA&usg=AOvVaw31kkGPP7ZVlFGlAby9OkzE
+          url = if href
+            href.sub("/url?q=", "")
+          end
+
+          { title: i.text, url: url }
+        end
+      end
+
+      def have_next_page?(doc)
+        doc.search("div#foot .cur+td").any?
+      end
+    end
+  end
+end

data/lib/domains_scanner/crawlers.rb

@@ -0,0 +1,17 @@
+require 'mechanize'
+require 'domains_scanner/crawlers/base'
+require 'domains_scanner/crawlers/baidu'
+require 'domains_scanner/crawlers/google'
+
+module DomainsScanner
+  module Crawlers
+    def self.build(engine)
+      case engine.to_s
+      when "google"
+        Google.new
+      when "baidu"
+        Baidu.new
+      end
+    end
+  end
+end

data/lib/domains_scanner/result_item.rb

@@ -0,0 +1,18 @@
+module DomainsScanner
+  class ResultItem
+    attr_reader :title, :url, :uri
+
+    def initialize(title:, url:)
+      @title = title
+      @url = url
+
+      if url
+        @uri = URI(@url)
+      end
+    end
+
+    def host
+      @uri && @uri.host
+    end
+  end
+end

data/lib/domains_scanner/results.rb

@@ -0,0 +1,14 @@
+require 'domains_scanner/result_item'
+
+module DomainsScanner
+  class Results
+    attr_reader :items, :have_next_page
+
+    def initialize(results, have_next_page)
+      @items = results.map do |result|
+        ResultItem.new(title: result[:title], url: result[:url])
+      end
+      @have_next_page = have_next_page
+    end
+  end
+end

data/lib/domains_scanner/runner.rb

@@ -0,0 +1,42 @@
+module DomainsScanner
+  class Runner
+    attr_reader :workers
+
+    def initialize(domain_word:, top_level_domain:)
+      @domain_word = domain_word
+      @top_level_domain = top_level_domain
+    end
+
+    def run
+      domain = "#{@domain_word}.#{@top_level_domain}"
+      puts "Start scan #{domain}" if DomainsScanner.verbose
+      @workers = DomainsScanner.engines.map do |engine|
+        crawler = DomainsScanner::Crawlers.build(engine)
+        page = 1
+
+        Thread.new do
+          loop do
+            puts "Scanning #{domain} with #{engine} on page: #{page}" if DomainsScanner.verbose
+
+            begin
+              results = crawler.search(@domain_word, @top_level_domain, page)
+              results.items.each do |item|
+              DomainsScanner.output_queue.push({
+                  domain: item.host, top_level_domain: @top_level_domain, engine: engine
+                })
+              end
+              break unless results.have_next_page
+            rescue Mechanize::ResponseCodeError => e
+              puts "search in #{engine} error, skip now" if DomainsScanner.verbose
+            end
+
+            break unless page < DomainsScanner.max_page
+            page += 1
+          end
+        end
+      end
+
+      @workers.each(&:join)
+    end
+  end
+end

data/lib/domains_scanner/version.rb

@@ -0,0 +1,3 @@
+module DomainsScanner
+  VERSION = "0.0.1"
+end

data/lib/domains_scanner.rb

@@ -0,0 +1,50 @@
+require 'uri'
+require 'thread'
+require "domains_scanner/version"
+require "domains_scanner/crawlers"
+require "domains_scanner/runner"
+
+module DomainsScanner
+  TOP_LEVEL_DOMAINS = %w(com cn com.cn net org ltd cc mobi live io co me hk).freeze
+
+  class << self
+    attr_accessor :verbose, :max_page, :engines
+    DomainsScanner.verbose = false
+    DomainsScanner.max_page = 20
+    DomainsScanner.engines = %w(baidu google)
+
+    def output_queue
+      @output_queue ||= Queue.new
+    end
+
+    def scan(domain_word:, top_level_domains: TOP_LEVEL_DOMAINS)
+      top_level_domains ||= TOP_LEVEL_DOMAINS
+
+      runners = top_level_domains.map do |top_level_domain|
+        Runner.new(domain_word: domain_word, top_level_domain: top_level_domain)
+      end
+      runners.each(&:run)
+
+      puts "Start analysis crawl results..." if DomainsScanner.verbose
+      analysis_output_queue
+    end
+
+    def analysis_output_queue
+      list = {}
+      begin
+        while(result = output_queue.pop(non_block = true)) do
+          domain, top_level_domain, engine = result.values_at(:domain, :top_level_domain, :engine)
+          next unless domain
+
+          group = list[top_level_domain] ||= {}
+          group[domain] ||= []
+
+          group[domain] << engine unless group[domain].include?(engine)
+        end
+      rescue ThreadError # queue is empty
+      end
+
+      list
+    end
+  end
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: domains_scanner
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Martin Hong
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-11-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.1'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.5
+description: search possible domains by search engines and other tools
+email:
+- hongzeqin@gmail.com
+executables:
+- domains_scanner
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/domains_scanner
+- domains_scanner.gemspec
+- lib/ansi_colors.rb
+- lib/domains_scanner.rb
+- lib/domains_scanner/crawlers.rb
+- lib/domains_scanner/crawlers/baidu.rb
+- lib/domains_scanner/crawlers/base.rb
+- lib/domains_scanner/crawlers/google.rb
+- lib/domains_scanner/result_item.rb
+- lib/domains_scanner/results.rb
+- lib/domains_scanner/runner.rb
+- lib/domains_scanner/version.rb
+homepage: https://rubygems.org/gems/domains_scanner
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.6
+signing_key: 
+specification_version: 4
+summary: search possible domains for specified keyword
+test_files: []