gcrawler 0.1.0

data/lib/gcrawler/search.rb

@@ -0,0 +1,122 @@
+# Crawl google result by keywords
+#
+# Dependences:
+#   gem 'wombat'
+#
+
+require 'wombat'
+require 'uri'
+require_relative './utils'
+
+# Crawl action
+class Crawler
+  include Wombat::Crawler
+
+  def query_str(val)
+    setting
+    path "/search?#{val}"
+  end
+
+  base_url "https://#{Utils.random_domain}/"
+
+  private
+
+  def setting
+    proxy = Utils.random_proxy
+    user_agent = Utils.random_user_agent
+    mechanize.set_proxy(*proxy) if proxy.length == 2
+    mechanize.user_agent = user_agent
+
+    pp "proxy: #{proxy}, user_agent: #{user_agent}"
+  end
+end
+
+# Google crawler
+class GoogleCrawler
+  attr_accessor :exclude_hosts
+
+  def initialize(proxies: [], black_domains: [], exclude_hosts: [])
+    @exclude_hosts = exclude_hosts
+    Utils.proxies = proxies
+    Utils.black_domains = black_domains
+
+    @crawler = Crawler.new
+  end
+
+  # search as url
+  def search_as_url(*keywords, language: nil, num: nil, country: nil, start: 0)
+    search_as_page(*keywords, language: language, num: num, country: country, start: start)
+
+    filter_urls
+  end
+
+  # search as object with keys {'text', 'url'}
+  def search_as_object(*keywords, language: nil, num: nil, country: nil, start: 0)
+    search_as_page(*keywords, language: language, num: num, country: country, start: start)
+
+    generate_objects
+  end
+
+  # search as page
+  # Args:
+  #   keywords (varargs): kw1, kw2, kw3, ...
+  #   language (str, optional): Query language. Defaults to nil.
+  #   num (uint, optional): Number of results per page(default is 10 per page). Defaults to nil.
+  #   start (int, optional): Offset. Defaults to 0.
+  #   country (str, optional): Query country, Defaults to None, example: countryCN or cn or CN
+  #
+  # Return:
+  #   Mechanize::Page, see https://github.com/sparklemotion/mechanize
+  #
+  def search_as_page(*keywords, language: nil, num: nil, country: nil, start: 0)
+    return if keywords.empty?
+
+    query_str = "q=#{keywords.join('+')}&btnG=Search&gbv=1&safe=active&start=0"
+    query_str += "&ln=#{language}" unless language.blank?
+    query_str += "&num=#{num.to_i}" unless num.blank?
+    query_str += "&cr=#{country}" unless country.blank?
+    query_str.gsub!(/start=0/, "start=#{start}") unless start == 0
+
+    @crawler.query_str(query_str)
+
+    seconds = Utils.random_interval_time
+    pp "Crawling query string is #{query_str}, will be crawling after #{seconds} seconds..."
+    sleep(seconds)
+
+    @crawler.crawl
+
+    raise "Fetch on Google failed with code #{@crawler.response_code}" unless @crawler.response_code == 200
+
+    pp 'Crawl on Google successfully...'
+  end
+
+  private
+
+  def filter_urls
+    urls = @crawler.page&.links&.map do |link_node|
+      uri_str = link_node.uri.to_s
+
+      if uri_str.start_with?(%r{/url\?q=}) && !@exclude_hosts.include?(URI.parse(uri_str.split(%r{/url\?q=})[1])&.host)
+        real_uri = uri_str.split(%r{/url\?q=})[1]
+      end
+      real_uri
+    end
+
+    urls.compact
+  end
+
+  def generate_objects
+    objects = @crawler.page&.links&.map do |link_node|
+      uri_str = link_node.uri.to_s
+
+      if uri_str.start_with?(%r{/url\?q=}) && !@exclude_hosts.include?(URI.parse(uri_str.split(%r{/url\?q=})[1])&.host)
+        node = { text: link_node.text,
+                 url: uri_str.split(%r{/url\?q=})[1] }
+      end
+
+      node
+    end
+
+    objects.compact
+  end
+end

data/lib/gcrawler/utils.rb

@@ -0,0 +1,37 @@
+module Utils
+  @domains = File.readlines(File.join(File.dirname(__FILE__), './data/domains.txt'))
+  @user_agents = File.readlines(File.join(File.dirname(__FILE__), './data/user_agents.txt'))
+  @proxies = []
+  @black_domains = []
+
+  class << self
+    attr_reader :proxies, :black_domains
+
+    def proxies=(proxies)
+      @proxies = proxies unless proxies.empty?
+    end
+
+    def black_domains=(black_domains)
+      @black_domains = black_domains unless black_domains.empty?
+    end
+
+    def random_domain
+      (@domains - @black_domains).sample.gsub(/[\r\n]+/, '')
+    end
+
+    def random_user_agent
+      @user_agents.sample.gsub(/[\r\n]+/, '')
+    end
+
+    def random_interval_time
+      (20..59).to_a.sample
+    end
+
+    def random_proxy
+      proxy = @proxies.sample
+      return [proxy[:ip], proxy[:port].to_i] unless proxy.nil?
+
+      []
+    end
+  end
+end

data/lib/gcrawler/version.rb

@@ -0,0 +1,3 @@
+module Gcrawler
+  VERSION = "0.1.0"
+end

data/lib/gcrawler.rb

@@ -0,0 +1,2 @@
+require_relative 'gcrawler/version'
+require_relative 'gcrawler/search'

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: gcrawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- rogerluo410
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2022-09-23 00:00:00.000000000 Z
+dependencies: []
+description: Crawling link text and link url by keywords on Google.com.
+email:
+- rogerluo410@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/example.rb
+- gcrawler.gemspec
+- lib/gcrawler.rb
+- lib/gcrawler/data/domains.txt
+- lib/gcrawler/data/user_agents.txt
+- lib/gcrawler/search.rb
+- lib/gcrawler/utils.rb
+- lib/gcrawler/version.rb
+homepage: https://github.com/rogerluo410/gcrawler
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org/
+  homepage_uri: https://github.com/rogerluo410/gcrawler
+  source_code_uri: https://github.com/rogerluo410/gcrawler
+  changelog_uri: https://github.com/rogerluo410/gcrawler/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key: 
+specification_version: 4
+summary: Google search crawler for Ruby version.
+test_files: []