capybara_crawler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 12bca6bf174ac78ecc5a66200ea7c282a084ce8a
+  data.tar.gz: 583e3743cbbce4ea35b9ed8a0d8005e73851915b
+SHA512:
+  metadata.gz: f07c9b1cc4062a8ef90a6c927f9b24f6a82fea50fe7d50c5d473db3bff231f84a9fb9c05d1a960438c8efd176e4590e0d59e3d2c863085900a703079873b303b
+  data.tar.gz: cf53828a6354c39edb2695e0f01052b478e6f7e6216cdcb8c8d7517f1f03f74c1fcd7c81d412c16b744ccfb02898dc67c89520761b1e5834f6d466fe47ab28a0

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/tips/
+/pkg/
+/spec/reports/
+/tmp/
+.DS_Store
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in capybara_crawler.gemspec
+gemspec

data/README.md

@@ -0,0 +1,53 @@
+# CapybaraCrawler
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/capybara_crawler`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'capybara_crawler'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install capybara_crawler
+
+## Usage
+
+see `examples` and `spec/capybara_crawler_spec.rb`
+
+## More
+
+### Capybara cheatsheet
+
+read `doc/capybara-cheatsheet.md`
+
+## Dependencies
+
+### Capybara Driver: poltergeist
+
+Poltergeist is a driver for Capybara that allows you to run your tests on a headless WebKit browser, provided by PhantomJS.
+https://github.com/teampoltergeist/poltergeist
+
+### PhantomJS
+PhantomJS is a headless WebKit scriptable with a JavaScript API.
+http://phantomjs.org/
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rainchen/capybara_crawler.
+

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+namespace :tmp do
+  desc "Clear capybara files from tmp/"
+  task clear: ["tmp:capybara:clear"]
+
+  namespace :capybara do
+    # desc "Clears all files and directories in tmp/capybara"
+    task :clear do
+      rm_rf Dir["tmp/capybara/[^.]*"], verbose: false
+    end
+  end
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "capybara_crawler"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/capybara_crawler.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'capybara_crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "capybara_crawler"
+  spec.version       = CapybaraCrawler::VERSION
+  spec.authors       = ["RainChen"]
+  spec.email         = ["hirainchen@gmail.com"]
+
+  spec.summary       = %q{a simple web crawler using DSL of capybara}
+  spec.description   = %q{a simple web crawler using DSL of capybara}
+  spec.homepage      = "https://github.com/rainchen/capybara_crawler"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "capybara"
+  spec.add_dependency "poltergeist"
+  spec.add_dependency "poltergeist-suppressor"
+  spec.add_dependency "capybara-user_agent"
+  spec.add_dependency "capybara-mechanize"
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "launchy"
+end

data/doc/capybara-cheatsheet.md

@@ -0,0 +1,204 @@
+# Capybara cheatsheet
+
+basing on https://gist.github.com/tomas-stefano/6652111
+
+## Capybara Actions
+
+```ruby
+# Anchor
+click_link 'Save'
+
+# Button
+click_button 'awesome'
+
+# Both above
+click_link_or_button 'Save'
+
+# `click_on` alias for click_link_or_button, shorter yet unambiguous
+click_on('Link Text') # clicks on either links or buttons
+click_on('Button Value')
+
+# Text (area) field
+fill_in 'Name or id', with: 'Content'
+
+# Checkbox
+check 'Content'
+uncheck 'Content'
+
+# Radio button
+choose 'Content'
+
+# Select option from select tag
+select 'Option', from: 'Label'
+
+# File input
+attach_file Rails.root.join('spec/fixture/some_file.png')
+```
+
+## Capybara Finders
+
+```ruby
+page.all(:xpath, '//a')
+
+page.first(:xpath, '//a')
+
+page.find('//textarea[@id="additional_newline"]')
+
+page.find(:xpath, "//input[@id='form_pets_dog']")['checked']
+# => true
+
+page.find(:css, '#with_focus_event').trigger(:focus)
+
+page.find(:css,'.wrapper').hover
+
+page.find_field("test_field").value
+# => 'blah'
+
+page.find_by_id('red').tag_name
+# => 'a'
+
+# finds invisible elements when false
+page.find_by_id("hidden_via_ancestor", visible: false)
+
+page.find_button('What an Awesome')[:value]
+# => 'awesome'
+
+page.find_link('abo').text
+# => 'labore'
+
+page.find_link('other')[:href]
+# => '/some_uri'
+```
+
+**Note:** `find` will wait for an element to appear on the page, as explained in the Ajax section. If the element does not appear it will raise an error.
+
+**Note:** In XPath the expression `//` means something very specific, and it might not be what you think. Contrary to common belief, `//` means "anywhere in the document" not "anywhere in the current context".
+
+
+## Capybara Scoped Finder `within`
+
+```ruby
+within(search_form) do
+  fill_in 'Name', with: 'iOS 7'
+  click_button 'Search'
+end
+
+def search_form
+  '.search_form'
+end
+
+within_fieldset("villain_fieldset") do
+  # ...
+end
+
+within_table("some_table") do
+  # ...
+end
+
+# Execute the given block within the given iframe using given frame name or index.
+#
+within_frame('some_frame') do
+end
+
+save_page
+
+# You need to install launchy gem.
+save_and_open_page
+```
+
+## Capybara Common
+
+```ruby
+visit("http://google.com")
+
+page.current_url
+
+# Execute the given script, not returning a result. This is useful for scripts that return
+# complex objects, such as jQuery statements. +execute_script+ should be used over
+# +evaluate_script+ whenever possible.
+#
+page.execute_script("$('#change').text('Funky Doodle')")
+
+# Evaluate the given JavaScript and return the result. Be careful when using this with
+# scripts that return complex objects, such as jQuery statements. +execute_script+ might
+# be a better alternative.
+#
+page.evaluate_script("1+3")
+# => 4
+
+using_wait_time 6 do
+  # ... Changed Capybara.default_wait_time in this block scope.
+end
+```
+
+## Capybara Matchers
+
+```ruby
+expect(page).to have_content("Some Content")
+expect(page).to have_no_content("Some Content")
+
+# True if there is a anchor tag with text matching regex
+expect(page).to have_xpath("//a")
+expect(page).to have_xpath("//a",:href => "google.com")
+expect(page).to have_xpath("//a[@href => 'google.com']")
+expect(page).to have_xpath("//a[contains(.,'some string')]")
+expect(page).to have_xpath("//p//a", :text => /re[dab]i/i, :count => 1)
+
+ # can take both xpath and css as input and can take arguments similar to both have_css and have_xpath
+ expect(page).to have_selector(:xpath, "//p/h1")
+ expect(page).to have_selector(:css, "p a#post_edit_path")
+
+ expect(page).to have_css("input#post_title")
+ expect(page).to have_css("input#post_title", :value => "Capybara cheatsheet")
+
+ # True if there are 3 input tags in response
+ expect(page).to have_css("input", :count => 3)
+
+ # True if there or fewer or equal to 3 input tags
+ expect(page).to have_css("input", :maximum => 3)
+
+ # True if there are minimum of 3 input tags
+ expect(page).to have_css("input", :minimum => 3)
+
+ # True if there 1 to 3 input tags
+ expect(page).to have_css("input", :between => 1..3)
+
+ # True if there is a anchor tag with text hello
+ expect(page).to have_css("p a", :text => "hello")
+ expect(page).to have_css("p a", :text => /[hH]ello(.+)/i)
+
+# For making capybara to take css as default selector
+Capybara.default_selector = :css
+
+# checks for the presence of the input tag
+expect(page).to have_selector("input")
+
+# checks for input tag with value
+expect(page).to have_selector("input", :value =>"Post Title")
+
+expect(page).to have_no_selector("input")
+
+# For making capybara to take css as default selector
+Capybara.default_selector = :xpath
+# checks for the presence of the input tag
+expect(page).to have_selector("//input")
+
+# checks for input tag with value
+expect(page).to have_selector("//input", :value =>"Post Title")
+
+# checks for presence of a input field named FirstName in a form
+expect(page).to have_field("FirstName")
+
+expect(page).to have_field("FirstName", :value => "Rambo")
+expect(page).to have_field("FirstName", :with => "Rambo")
+
+expect(page).to have_link("Foo")
+expect(page).to have_link("Foo", :href=>"googl.com")
+expect(page).to have_no_link("Foo", :href=>"google.com")
+
+
+# check page content
+# have_title matcher now supports :wait option
+expect(page).to have_title("changed title")
+expect(page).to have_text("text in page.text")
+```

data/examples/custom_user_agent.rb

@@ -0,0 +1,15 @@
+# an example for how to custom user agent for capybara_crawler
+class CustomUserAgent < CapybaraCrawler::Crawler
+  def check_user_agent
+    visit "http://www.useragentstring.com/" # or using "http://www.whatsmyua.info"
+    page.find_field("uas").value
+  end
+
+  def as_iphone
+    set_user_agent :iphone
+  end
+
+  def custom_user_agent
+    set_custom_user_agent "CapybaraCrawler/1.0"
+  end
+end

data/examples/google_search.rb

@@ -0,0 +1,41 @@
+# an example for how to use capybara_crawler to do google search
+class GoogleSearch < CapybaraCrawler::Crawler
+  GOOGLE_SEARCH_URL = "https://www.google.com/ncr"
+  GOOGLE_ADVANCED_SEARCH_URL = "https://www.google.com/advanced_search?hl=en&fg=1"
+
+  # search using Google Instant
+  def instant_search(query)
+    visit GOOGLE_SEARCH_URL
+    fill_in "q", with: query
+    press_return_on "q"
+    page
+  end
+
+  # search without js driver
+  def quick_search(query)
+    disable_js do
+      visit GOOGLE_SEARCH_URL
+      fill_in "q", with: query
+      click_button "Google Search"
+      page
+    end
+  end
+
+  # search using Google advanced search
+  # options could be :exact, :any, :none, :site
+  # select language as "English"
+  def advanced_search(query, options = {})
+    visit GOOGLE_ADVANCED_SEARCH_URL
+    fill_in "as_q"          , with: query # all these words
+    fill_in "as_epq"        , with: options[:exact] if options[:exact] # exact
+    fill_in "as_oq"         , with: options[:any] if options[:any] # any
+    fill_in "as_eq"         , with: options[:none] if options[:none] # none
+    fill_in "as_sitesearch" , with: options[:site] if options[:site] # site or domain:
+    # select language as "English"
+    find('.goog-select', text: "any language").click
+    within("#lr_menu") { find('li.goog-menuitem', :text => "English").click } # select "English"
+    click_button "Advanced Search"
+    page
+  end
+end
+

data/lib/capybara_crawler.rb

@@ -0,0 +1,6 @@
+require "capybara_crawler/version"
+require "capybara_crawler/crawler"
+
+module CapybaraCrawler
+
+end

data/lib/capybara_crawler/crawler.rb

@@ -0,0 +1,123 @@
+require "capybara/dsl"
+require 'capybara/poltergeist'
+require 'poltergeist/suppressor'
+require 'capybara/user_agent'
+require 'capybara/mechanize'
+
+module CapybaraCrawler
+  # Crawler is a class can use DSL of Capybara
+  class Crawler
+    include Capybara::DSL
+    include Capybara::UserAgent::DSL
+
+    class << self
+      def init_driver
+        register_driver_poltergeist_crawler
+        register_driver_mechanize_crawler
+        Capybara.default_max_wait_time = 3
+        Capybara.run_server = false
+      end
+
+      def register_driver_poltergeist_crawler
+        Capybara.register_driver :poltergeist_crawler do |app|
+          Capybara::Poltergeist::Driver.new(app, {
+            js_errors: false, # when false, JavaScript errors do not get re-raised in Ruby.
+            inspector: false, # when false, remote debugging will be disabled
+            debug: false, # output log messages like {"id":"68d74fdd-adab-4331-ab8e-48153b9a3176","name":"set_js_errors","args":[false]}
+            phantomjs_logger: Capybara::Poltergeist::Suppressor.new # silences noisy phantomjs warnings such as 'CoreText performance note...'.
+          })
+        end
+      end
+
+      def register_driver_mechanize_crawler
+        Capybara.register_driver :mechanize_crawler do |app|
+          driver = Capybara::Mechanize::Driver.new(app || {})
+          driver.configure do |agent|
+            # Configure other Mechanize options here.
+            agent.log = Logger.new "tmp/mechanize_crawler.log"
+          end
+          driver
+        end
+      end
+
+      def config_users_agents
+        Capybara::UserAgent.add_user_agents(
+          :phantomjs        => "Mozilla/5.0 (Macintosh; Intel Mac OS X) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1",
+          :capybara_crawler => "Mozilla/5.0 (Macintosh; Intel Mac OS X) AppleWebKit/538.1 (KHTML, like Gecko) CapybaraCrawler/#{CapybaraCrawler::VERSION} Safari/538.1",
+          :windows_firefox  => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20120427 Firefox/15.0a1",
+          :mac_safari       => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/601.5.17 (KHTML, like Gecko) Version/9.1 Safari/537.86.5",
+          :default          => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/538.1 (KHTML, like Gecko) Version/9.1 Safari/538.1",
+        )
+      end
+
+      def init
+        if !@initialized # make sure init once
+          init_driver
+          config_users_agents
+          @initialized = true
+        end
+        Capybara.default_driver = :poltergeist_crawler
+      end
+    end
+
+    # when a new class inherited
+    def self.inherited(subclass)
+      init
+      subclass.send :include, MoreDSL
+    end
+
+    # add more DSL methods
+    module MoreDSL
+      # press a key on a field
+      # usage:
+      #   press_key :return on: 'field-id'
+      # http://www.rubydoc.info/github/jnicklas/capybara/Capybara%2FNode%2FElement%3Asend_keys
+      def press_key(key, on:)
+        field = find_field(on)
+        field.send_keys key
+      end
+
+      # press "return" key on a field
+      def press_return_on(field_name_or_id)
+        press_key :return, on: field_name_or_id
+      end
+
+      # disable running javascript to crawl faster
+      def disable_js(&block)
+        Capybara.current_driver = :mechanize_crawler
+        result = yield
+        Capybara.use_default_driver # switch back to default driver
+        result
+      end
+
+      # override Capybara::UserAgent::DSL#set_custom_user_agent
+      def set_custom_user_agent(user_agent)
+        driver = Capybara.current_session.driver
+        if driver.is_a?(Capybara::Mechanize::Driver)
+          driver.configure do |agent|
+            agent.user_agent = user_agent
+          end
+        else
+          super
+        end
+      end
+    end
+
+    def initialize
+      config_user_agent
+    end
+
+    # use a user agent which is similar as :mac_safari to make it easy to check target page on Mac using Safari by default
+    # but using "Safari/538.1" which is extracted from default user agent of PhantomJS/2.1.1
+    def default_user_agent
+      Capybara::UserAgent.user_agents[:default]
+    end
+
+    protected
+      # set default user_agent as firefox
+      # more agent: http://whatsmyuseragent.com/
+      def config_user_agent
+        set_user_agent :default
+      end
+  end
+end

data/lib/capybara_crawler/version.rb

@@ -0,0 +1,3 @@
+module CapybaraCrawler
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,199 @@
+--- !ruby/object:Gem::Specification
+name: capybara_crawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- RainChen
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-03-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poltergeist-suppressor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara-user_agent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara-mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: launchy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: a simple web crawler using DSL of capybara
+email:
+- hirainchen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- capybara_crawler.gemspec
+- doc/capybara-cheatsheet.md
+- examples/custom_user_agent.rb
+- examples/google_search.rb
+- lib/capybara_crawler.rb
+- lib/capybara_crawler/crawler.rb
+- lib/capybara_crawler/version.rb
+homepage: https://github.com/rainchen/capybara_crawler
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: a simple web crawler using DSL of capybara
+test_files: []
+has_rdoc: