rcrawler 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06cb51462ed7694164d649bc33f28102ae5ed66a
-  data.tar.gz: fc4017fd01b7140517b1e13bcbda221ac672eb77
+  metadata.gz: ff1eee821a0d9e416d60245d49c28fb8a871b331
+  data.tar.gz: 74e6cd410b5c632ccc4249f61564843579654adc
 SHA512:
-  metadata.gz: 2408c89221e3c7d4b9e6f11f1d2015d4471efe3492369ded48ba4a202fce1c30b74894201ccdd7af5cd6e94cfe8ea9090a1a11c38f3352f7929804884a7eb2a2
-  data.tar.gz: 6c77a1d1fab7789c3920524c746e8f1f7070a78df0b27f02a6db059eb1695a4c616389eb40939b9d6dc88d95e3ff4c7dd065b2aa01cb097ec326143612275cbe
+  metadata.gz: d6db4bf8b44933b5dc99d0932dee1d6a19180ade6751ad5689f6aef3868d26cc0cbb0320b18b5b41c34641c376483ac4c413d7380a68b19f990f84e8e50827be
+  data.tar.gz: f6f7b46570e5af00e157cf2fb5ad1a81db2f09b2097e8db4ca79e681b849a24d73055c24f1b94c202be61b9e3e704cf63a068214066e70d2190250000c6d9c99

data/.coveralls.yml

@@ -0,0 +1 @@
+repo_token: OFZ4IKBbbaOGDyQwEB3LNlGDoV9dQearl

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+gemfile:
+  - Gemfile
+script: bundle exec rake spec
+branches:
+  only:
+    - master
+notifications:
+  mails:
+    - i2bskn@gmail.com

data/README.md

@@ -1,7 +1,17 @@
-# Rcrawler
+# RCrawler
+
+[![Gem Version](https://badge.fury.io/rb/rcrawler.png)](http://badge.fury.io/rb/rcrawler)
+[![Build Status](https://travis-ci.org/i2bskn/rcrawler.png?branch=master)](https://travis-ci.org/i2bskn/rcrawler)
+[![Coverage Status](https://coveralls.io/repos/i2bskn/rcrawler/badge.png)](https://coveralls.io/r/i2bskn/rcrawler)
+[![Code Climate](https://codeclimate.com/github/i2bskn/rcrawler.png)](https://codeclimate.com/github/i2bskn/rcrawler)
 
 The wrapper of capybara for crawler.
 
+## Dependencies
+
+* nokogiri requires libxml2.
+* capybara-webkit requires qt. [capybara-webkit wiki](https://github.com/thoughtbot/capybara-webkit/wiki/Installing-Qt-and-compiling-capybara-webkit)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -18,6 +28,8 @@ Or install it yourself as:
 
 ## Usage
 
+#### Crawl
+
 ```ruby
 require "rcrawler"
 
@@ -40,6 +52,33 @@ RCrawler.crawl do
 end
 ```
 
+#### Configuration
+
+```ruby
+RCrawler.configure do |c|
+  c.threads = 10 # => default is 8
+  c.timeout = 20 # => default is 10
+  c.timeout_proc = :ignore # => default is :raise
+end
+```
+
+#### Async processing
+
+```ruby
+RCrawler.async do
+  crawl do
+    # do something
+  end
+
+  crawl do
+    # do something
+  end
+
+  crawl do
+    # do something
+  end
+end
+```
 ## Contributing
 
 1. Fork it

data/lib/rcrawler.rb

@@ -7,14 +7,35 @@ require "nokogiri"
 require "headless"
 
 require "rcrawler/version"
+require "rcrawler/configuration"
 require "rcrawler/driver"
 require "rcrawler/crawl"
+require "rcrawler/async"
 
 module RCrawler
+  @config = Configuration.instance
+
   class << self
     def crawl(&block)
-      c = Crawl.new
-      c.instance_eval &block
+      begin
+        Timeout::timeout(@config.timeout) {Crawl.new.instance_eval &block}
+      rescue Timeout::Error => e
+        raise if @config.timeout_proc == :raise
+      end
+    end
+
+    def configure(&block)
+      if block_given?
+        @config.configure &block
+      else
+        @config
+      end
+    end
+
+    def async(&block)
+      async_threads = Async.new
+      async_threads.instance_eval &block
+      async_threads.execute
     end
   end
 end

data/lib/rcrawler/async.rb

@@ -0,0 +1,44 @@
+# coding: utf-8
+
+require "thread"
+require "timeout"
+
+module RCrawler
+  class Async
+    def initialize
+      @queue = ::Queue.new
+      @config = ::RCrawler::Configuration.instance
+    end
+
+    def crawl(&block)
+      raise ArgumentError, "crawl method is required block" unless block_given?
+      @queue.push block
+    end
+
+    def execute
+      threads = []
+      @config.threads.times do
+        threads << create_thread
+      end
+      threads.each {|thread| thread.join}
+    end
+
+    private
+    def create_thread
+      ::Thread.start do
+        while !@queue.empty?
+          begin
+            Timeout::timeout(@config.timeout) {exec_crawl(@queue.pop)}
+          rescue Timeout::Error => e
+            raise if @config.timeout_proc == :raise
+          end
+        end
+      end
+    end
+
+    def exec_crawl(block)
+      crwl = Crawl.new
+      crwl.instance_eval &block
+    end
+  end
+end

data/lib/rcrawler/configuration.rb

@@ -0,0 +1,31 @@
+# coding: utf-8
+
+require "singleton"
+
+module RCrawler
+  class Configuration
+    VALID_OPTIONS_KEYS = [
+      :threads,
+      :timeout,
+      :timeout_proc
+    ].freeze
+
+    attr_accessor *VALID_OPTIONS_KEYS
+
+    include Singleton
+
+    def initialize
+      reset
+    end
+
+    def configure
+      yield self
+    end
+
+    def reset
+      self.threads = 8
+      self.timeout = 10
+      self.timeout_proc = :raise
+    end
+  end
+end

data/lib/rcrawler/version.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
 
 module RCrawler
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/rcrawler/async_spec.rb

@@ -0,0 +1,65 @@
+# coding: utf-8
+
+require "spec_helper"
+
+describe RCrawler::Async do
+  let(:async) {RCrawler::Async.new}
+
+  describe "#initialize" do
+    it "Queue should be created" do
+      mock = double("queue mock")
+      Queue.should_receive(:new).and_return(mock)
+      expect(async.instance_eval {@queue}).to eq(mock)
+    end
+
+    it "Configuration should be setted" do
+      mock = double("configuration mock").as_null_object
+      RCrawler::Configuration.should_receive(:instance).exactly(2).and_return(mock)
+      expect(async.instance_eval {@config}).to eq(mock)
+    end
+  end
+
+  describe "#crawl" do
+    it "crawl task should be registered" do
+      async
+      expect {
+        async.crawl {visit "http://example.com"}
+      }.to change(async.instance_eval {@queue}, :size).by(1)
+    end
+
+    it "exception should be generated if not argument" do
+      expect {async.crawl}.to raise_error(ArgumentError)
+    end
+  end
+
+  describe "#execute" do
+    it "thread should be created 8" do
+      mock = double("thread mock")
+      mock.should_receive(:join).exactly(8)
+      RCrawler::Async.any_instance.should_receive(:create_thread).exactly(8).and_return(mock)
+      expect{async.execute}.not_to raise_error
+    end
+  end
+
+  describe "#create_thread" do
+    before {RCrawler::Crawl.stub(:new).and_return(double.as_null_object)}
+
+    it "Thread object should be returned" do
+      expect(async.send(:create_thread).is_a? Thread).to be_true
+    end
+
+    it "exec_crawl method should be called" do
+      async.crawl {}
+      expect{async.send(:create_thread)}.not_to raise_error
+    end
+  end
+
+  describe "#exec_crawl" do
+    it "Crawl object should be created" do
+      mock = double("crawl mock")
+      mock.should_receive(:instance_eval)
+      RCrawler::Crawl.should_receive(:new).and_return(mock)
+      expect(async.send(:exec_crawl, Proc.new{})).not_to raise_error
+    end
+  end
+end

data/spec/rcrawler/configuration_spec.rb

@@ -0,0 +1,88 @@
+# coding: utf-8
+
+require "spec_helper"
+
+describe RCrawler::Configuration do
+  let(:config) {RCrawler::Configuration.instance}
+
+  describe "defined accessors" do
+    it "defined threads" do
+      expect(config.respond_to? :threads).to be_true
+    end
+
+    it "defined timeout" do
+      expect(config.respond_to? :timeout).to be_true
+    end
+
+    it "defined timeout_proc" do
+      expect(config.respond_to? :timeout_proc).to be_true
+    end
+  end
+
+  describe "#initialize" do
+    it "threads should be a default" do
+      expect(config.threads).to eq(8)
+    end
+
+    it "timeout should be a default" do
+      expect(config.timeout).to eq(10)
+    end
+
+    it "timeout_proc should be a default" do
+      expect(config.timeout_proc).to eq(:raise)
+    end
+  end
+
+  describe "#configure" do
+    before do
+      config.configure do |c|
+        c.threads = 20
+        c.timeout = 30
+        c.timeout_proc = :ignore
+      end
+    end
+
+    it "threads should be a specified parameter" do
+      expect(config.threads).to eq(20)
+    end
+
+    it "timeout should be a specified parameter" do
+      expect(config.timeout).to eq(30)
+    end
+
+    it "timeout_proc should be a specified parameter" do
+      expect(config.timeout_proc).to eq(:ignore)
+    end
+
+    it "raise error unknown setting" do
+      expect {
+        config.configure do |c|
+          c.unknown = true
+        end
+      }.to raise_error
+    end
+  end
+
+  describe "#reset" do
+    before do
+      config.configure do |c|
+        c.threads = 20
+        c.timeout = 30
+        c.timeout_proc = :ignore
+      end
+      config.reset
+    end
+
+    it "threads should be a default" do
+      expect(config.threads).to eq(8)
+    end
+
+    it "timeout should be a default" do
+      expect(config.timeout).to eq(10)
+    end
+
+    it "timeout_proc should be a default" do
+      expect(config.timeout_proc).to eq(:raise)
+    end
+  end
+end

data/spec/rcrawler/crawl_spec.rb

@@ -0,0 +1,31 @@
+# coding: utf-8
+
+require "spec_helper"
+
+describe RCrawler::Crawl do
+  describe "#initialize" do
+    it "Driver should be created" do
+      mock = double("driver mock")
+      RCrawler::Driver.should_receive(:new).and_return(mock)
+      c = RCrawler::Crawl.new
+      expect(c.instance_eval {@driver}).to eq(mock)
+    end
+  end
+
+  describe "#method_missing" do
+    it "method in Driver should be called" do
+      RCrawler::Driver.any_instance.should_receive(:visit).and_return(true)
+      expect {
+        c = RCrawler::Crawl.new
+        c.instance_eval {visit "http://example.com"}
+      }.not_to raise_error
+    end
+
+    it "exception should be thrown if method is not defined in the Driver" do
+      expect {
+        c = RCrawler::Crawl.new
+        c.instance_eval {unknown_method "arg"}
+      }.to raise_error
+    end
+  end
+end

data/spec/rcrawler/driver_spec.rb

@@ -0,0 +1,56 @@
+# coding: utf-8
+
+require "spec_helper"
+
+describe RCrawler::Driver do
+  it "default_driver should be a webkit" do
+    d = RCrawler::Driver.new
+    expect(d.instance_eval {Capybara.default_driver}).to eq(:webkit)
+  end
+
+  it "should include Capybara::DSL" do
+    d = RCrawler::Driver.new
+    expect(d.respond_to? :visit).to be_true
+  end
+
+  describe "#initialize" do
+    it "Headless object should be created if no argument" do
+      mock = double("Headless mock")
+      mock.should_receive(:start)
+      Headless.should_receive(:new).and_return(mock)
+      RCrawler::Driver.new
+    end
+
+    it "Headless should not start if headless is false" do
+      Headless.any_instance.should_not_receive(:start)
+      RCrawler::Driver.new(headless: false)
+    end
+
+    it "@headless should be specified object" do
+      mock = double("Headless mock")
+      mock.should_receive(:start)
+      RCrawler::Driver.new(headless: mock)
+    end
+  end
+
+  describe "#screenshot" do
+    it "exception should be thrown if no argument" do
+      expect{RCrawler::Driver.new.screenshot}.to raise_error(ArgumentError)
+    end
+
+    it "visit and save_screenshot method should be called" do
+      d = RCrawler::Driver.new
+      d.should_receive(:visit)
+      d.should_receive(:page).and_return(double("page mock").as_null_object)
+      d.screenshot("http://example.com", "/tmp/example.png")
+    end
+  end
+
+  describe "#doc" do
+    it "should return Nokogiri::HTML::Document object" do
+      d = RCrawler::Driver.new
+      d.page.should_receive(:html).and_return("<html></html>")
+      expect(d.doc.is_a? Nokogiri::HTML::Document).to be_true
+    end
+  end
+end

data/spec/rcrawler_spec.rb

@@ -0,0 +1,54 @@
+# coding: utf-8
+
+require "spec_helper"
+
+describe RCrawler do
+  describe ".crawl" do
+    it "method should be defined" do
+      expect(RCrawler.respond_to? :crawl).to be_true
+    end
+
+    it "exception should be thrown if no argument" do
+      expect{RCrawler.crawl}.to raise_error(ArgumentError)
+    end
+
+    it "Crawl object should be created" do
+      mock = double("Crawl mock")
+      RCrawler::Crawl.should_receive(:new).and_return(mock)
+      mock.should_receive(:instance_eval)
+      RCrawler.crawl {}
+    end
+  end
+
+  describe ".async" do
+    it "method should be defined" do
+      expect(RCrawler.respond_to? :async).to be_true
+    end
+
+    it "exception should be thrown if no argument" do
+      expect{RCrawler.async}.to raise_error(ArgumentError)
+    end
+
+    it "Async object should be created" do
+      mock = double("Async mock")
+      RCrawler::Async.should_receive(:new).and_return(mock)
+      mock.should_receive(:instance_eval)
+      mock.should_receive(:execute)
+      RCrawler.async {}
+    end
+  end
+
+  describe ".configure" do
+    it "return configuration object" do
+      expect(RCrawler.configure.is_a? RCrawler::Configuration).to be_true
+    end
+
+    it "set config value from block" do
+      RCrawler.configure do |c|
+        c.threads = 10
+      end
+      expect(RCrawler.configure.threads).not_to eq(8)
+      expect(RCrawler.configure.threads).to eq(10)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,15 +1,16 @@
-# require "simplecov"
-# require "coveralls"
-# Coveralls.wear!
+require "simplecov"
+require "coveralls"
+Coveralls.wear!
 
 # SimpleCov.formatter = SimpleCov::Formatter::HTMLFormatter
-# SimpleCov.start do
-#   add_filter "spec"
-#   add_filter ".bundle"
-# end
+SimpleCov.start do
+  add_filter "spec"
+  add_filter ".bundle"
+end
 
 require "rcrawler"
 
 RSpec.configure do |config|
   config.order = "random"
+  config.after {RCrawler::Configuration.instance.reset}
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rcrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - i2bskn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-02 00:00:00.000000000 Z
+date: 2013-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -115,16 +115,25 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .coveralls.yml
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - lib/rcrawler.rb
+- lib/rcrawler/async.rb
+- lib/rcrawler/configuration.rb
 - lib/rcrawler/crawl.rb
 - lib/rcrawler/driver.rb
 - lib/rcrawler/version.rb
 - rcrawler.gemspec
+- spec/rcrawler/async_spec.rb
+- spec/rcrawler/configuration_spec.rb
+- spec/rcrawler/crawl_spec.rb
+- spec/rcrawler/driver_spec.rb
+- spec/rcrawler_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/i2bskn/rcrawler
 licenses:
@@ -146,9 +155,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
 summary: The wrapper of capybara for crawler
 test_files:
+- spec/rcrawler/async_spec.rb
+- spec/rcrawler/configuration_spec.rb
+- spec/rcrawler/crawl_spec.rb
+- spec/rcrawler/driver_spec.rb
+- spec/rcrawler_spec.rb
 - spec/spec_helper.rb