RubyGems - klepto - Versions diffs - 0.1.0 - Mend

klepto 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (32) hide show

data/.gitignore +21 -0
data/.rspec +2 -0
data/.rvmrc +1 -0
data/Gemfile +18 -0
data/Guardfile +11 -0
data/LICENSE.txt +22 -0
data/README.md +129 -0
data/Rakefile +7 -0
data/klepto.gemspec +26 -0
data/lib/klepto.rb +26 -0
data/lib/klepto/bot.rb +59 -0
data/lib/klepto/browser.rb +18 -0
data/lib/klepto/crawler.rb +72 -0
data/lib/klepto/tasks.rb +15 -0
data/lib/klepto/version.rb +3 -0
data/samples/example.rb +49 -0
data/spec/cassettes/Klepto_Crawler/dsl_interaction/should_crawl_the_resource.yml +1960 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_designate_scraping_of_a_set_of_nodes.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_designate_scraping_of_a_single_node_with_a_block.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_designate_scraping_of_a_single_node_with_a_symbol.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_scrape_the_node_that_the_crawler_is_scoped_to.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_CSS_scope.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_desired_syntax.yml +114 -0
data/spec/lib/klepto/bot_spec.rb +40 -0
data/spec/lib/klepto/browser_spec.rb +15 -0
data/spec/lib/klepto/crawler_spec.rb +88 -0
data/spec/lib/klepto/dsl_spec.rb +6 -0
data/spec/lib/klepto_spec.rb +64 -0
data/spec/orm/active_record.rb +36 -0
data/spec/orm/database.example.yml +15 -0
data/spec/spec_helper.rb +32 -0
metadata +157 -0

data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_CSS_scope.yml ADDED

@@ -0,0 +1,114 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://example.com/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: Found
+    headers:
+      location:
+      - http://www.iana.org/domains/example/
+      server:
+      - BigIP
+      connection:
+      - Keep-Alive
+      content-length:
+      - '0'
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.0'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: FOUND
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:03 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      location:
+      - http://www.iana.org/domains/example
+      content-length:
+      - '0'
+      connection:
+      - close
+      content-type:
+      - text/html; charset=utf-8
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:04 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      last-modified:
+      - Fri, 04 Jan 2013 01:17:22 GMT
+      vary:
+      - Accept-Encoding
+      connection:
+      - close
+      transfer-encoding:
+      - chunked
+      content-type:
+      - text/html; charset=UTF-8
+    body:
+      encoding: US-ASCII
+      string: ! "<!doctype html>\n<html>\n<head>\n\t<title>Example Domain</title>\n\n\t<meta
+        charset=\"utf-8\" />\n\t<meta http-equiv=\"Content-type\" content=\"text/html;
+        charset=utf-8\" />\n\t<meta name=\"viewport\" content=\"width=device-width,
+        initial-scale=1\" />\n\t<style type=\"text/css\">\n\tbody {\n\t\tbackground-color:
+        #f0f0f2;\n\t\tmargin: 0;\n\t\tpadding: 0;\n\t\tfont-family: \"Open Sans\",
+        \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n\t\t\n\t}\n\tdiv {\n\t\twidth:
+        600px;\n\t\tmargin: 5em auto;\n\t\tpadding: 3em;\n\t\tbackground-color: #fff;\n\t\tborder-radius:
+        1em;\n\t}\n\ta:link, a:visited {\n\t\tcolor: #38488f;\n\t\ttext-decoration:
+        none;\n\t}\n\t@media (max-width: 600px) {\n\t\tbody {\n\t\t\tbackground-color:
+        #fff;\n\t\t}\n\t\tdiv {\n\t\t\twidth: auto;\n\t\t\tmargin: 0 auto;\n\t\t\tborder-radius:
+        0;\n\t\t\tpadding: 1em;\n\t\t}\n\t}\n\t</style>\t\n</head>\n\n<body>\n<div>\n\t<h1>Example
+        Domain</h1>\n\t<p>This domain is established to be used for illustrative examples
+        in documents. You do not need to\n\t\tcoordinate or ask for permission to
+        use this domain in examples, and it is not available for\n\t\tregistration.</p>\n\t<p><a
+        href=\"http://www.iana.org/domains/special\">More information...</a></p>\n</div>\n</body>\n</html>\n"
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+recorded_with: VCR 2.4.0

data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_desired_syntax.yml ADDED

@@ -0,0 +1,114 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://example.com/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: Found
+    headers:
+      location:
+      - http://www.iana.org/domains/example/
+      server:
+      - BigIP
+      connection:
+      - Keep-Alive
+      content-length:
+      - '0'
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.0'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: FOUND
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:04 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      location:
+      - http://www.iana.org/domains/example
+      content-length:
+      - '0'
+      connection:
+      - close
+      content-type:
+      - text/html; charset=utf-8
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:04 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      last-modified:
+      - Fri, 04 Jan 2013 01:17:22 GMT
+      vary:
+      - Accept-Encoding
+      connection:
+      - close
+      transfer-encoding:
+      - chunked
+      content-type:
+      - text/html; charset=UTF-8
+    body:
+      encoding: US-ASCII
+      string: ! "<!doctype html>\n<html>\n<head>\n\t<title>Example Domain</title>\n\n\t<meta
+        charset=\"utf-8\" />\n\t<meta http-equiv=\"Content-type\" content=\"text/html;
+        charset=utf-8\" />\n\t<meta name=\"viewport\" content=\"width=device-width,
+        initial-scale=1\" />\n\t<style type=\"text/css\">\n\tbody {\n\t\tbackground-color:
+        #f0f0f2;\n\t\tmargin: 0;\n\t\tpadding: 0;\n\t\tfont-family: \"Open Sans\",
+        \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n\t\t\n\t}\n\tdiv {\n\t\twidth:
+        600px;\n\t\tmargin: 5em auto;\n\t\tpadding: 3em;\n\t\tbackground-color: #fff;\n\t\tborder-radius:
+        1em;\n\t}\n\ta:link, a:visited {\n\t\tcolor: #38488f;\n\t\ttext-decoration:
+        none;\n\t}\n\t@media (max-width: 600px) {\n\t\tbody {\n\t\t\tbackground-color:
+        #fff;\n\t\t}\n\t\tdiv {\n\t\t\twidth: auto;\n\t\t\tmargin: 0 auto;\n\t\t\tborder-radius:
+        0;\n\t\t\tpadding: 1em;\n\t\t}\n\t}\n\t</style>\t\n</head>\n\n<body>\n<div>\n\t<h1>Example
+        Domain</h1>\n\t<p>This domain is established to be used for illustrative examples
+        in documents. You do not need to\n\t\tcoordinate or ask for permission to
+        use this domain in examples, and it is not available for\n\t\tregistration.</p>\n\t<p><a
+        href=\"http://www.iana.org/domains/special\">More information...</a></p>\n</div>\n</body>\n</html>\n"
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:04 GMT
+recorded_with: VCR 2.4.0

data/spec/lib/klepto/bot_spec.rb ADDED

@@ -0,0 +1,40 @@
+require 'spec_helper'
+describe Klepto::Bot, :vcr => {:record => :new_episodes} do
+  before(:each) do
+    @bot = Klepto::Bot.new
+  end
+  it 'should know if it is a dry run' do
+    @bot.dry_run?.should be false
+    @bot.dry_run!
+    @bot.dry_run?.should be true
+  end
+  it 'should be able to set the selection syntax' do
+    @bot.syntax(:xpath)
+    @bot.syntax.should be(:xpath)
+  end
+  it 'should be able to read the selection syntax' do
+    @bot.syntax.should be(:css)
+  end
+  it 'should be able to set request headers' do
+    @bot.should respond_to(:headers)
+  end
+  it 'should be able to set a list of URLs to crawl' do
+    @bot.url 'http://www.google.com'
+    @bot.urls.should include('http://www.google.com')
+    @bot.urls 'http://twitter.com', 'http://facebook.com'
+    @bot.urls.should include('http://twitter.com')
+    @bot.urls.should include('http://facebook.com')
+  end
+  it 'should be able to add crawlers' do
+    @bot.crawl('div'){}
+    @bot.instance_variable_get("@crawlers").should have(1).crawler
+    @bot.instance_variable_get("@crawlers").first.should be_kind_of(Klepto::Crawler)
+  end
+end

data/spec/lib/klepto/browser_spec.rb ADDED

@@ -0,0 +1,15 @@
+require 'spec_helper'
+describe Klepto::Browser, :vcr => {:record => :new_episodes} do
+  before(:each) do
+    @browser = Klepto::Browser.new
+    @browser.set_headers({
+      'Referer'     => 'http://www.example.com'
+    })
+  end
+  it 'should be able to fetch a page' do
+    @page = @browser.fetch! 'http://www.example.com'
+    @page.status_code.should be(200)
+  end
+end

data/spec/lib/klepto/crawler_spec.rb ADDED

@@ -0,0 +1,88 @@
+require 'spec_helper'
+require 'open-uri'
+describe Klepto::Crawler, :vcr => {:record => :new_episodes} do
+  describe 'dsl interaction' do
+    before(:each) do
+      @page = page("http://www.iana.org")
+      @crawler = Klepto::Crawler.new('body',{:syntax => :css}) do
+        scrape 'h1', :title
+        scrape '#intro p' do |node|
+          {description: node.text}
+        end
+        scrape_all '.home-panel h2' do |nodes|
+          { sections: nodes.map{|n| n.text} }
+        end
+      end
+      @resources = @crawler.crawl @page
+    end #end before
+    it 'should crawl the resource' do
+      @resources.should have(1).resource
+      @resources.first[:title].should match('Internet Assigned Numbers Authority')
+      @resources.first[:description].should match(/^The Internet Assigned Numbers Authority/i)
+      @resources.first[:sections].should have(3).sections
+    end
+  end
+  describe 'standard interaction' do
+    before(:each) do
+      @page     = page()
+      @crawler  = Klepto::Crawler.new 'body', {:syntax => :css}
+    end
+    it 'should have a CSS scope' do
+      @crawler.scope.should eq 'body'
+    end
+    it 'should have a desired syntax' do
+      @crawler.syntax.should == :css
+    end
+    it 'should be able to scrape the node that the crawler is scoped to' do
+      @crawler.scrape do |node|
+        {:name => node.native.name}
+      end
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:name].should eq('body')
+    end
+    it 'should be able to designate scraping of a single node with a symbol' do
+      @crawler.scrape 'h1', :title
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:title].should eq('Example Domain')
+    end
+    it 'should be able to designate scraping of a single node with a block' do
+      @crawler.scrape 'h1' do |node|
+        {title: node.text}
+      end
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:title].should eq('Example Domain')
+    end
+    it 'should be able to designate scraping of a set of nodes' do
+      @crawler.scrape_all 'p' do |nodes|
+        {
+          paragraphs: [
+            nodes.first.text,
+            nodes.last.text
+          ]
+        }
+      end
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:paragraphs].should be_kind_of(Array)
+      resources.first[:paragraphs].last.should eq("More information...")
+    end
+    pending 'should be able to save a set of resources'
+    pending 'should be able to specify a limit'
+    pending 'should be able to specify a skip'
+  end
+end

data/spec/lib/klepto/dsl_spec.rb ADDED

@@ -0,0 +1,6 @@
+require 'spec_helper'
+describe 'Klepto DSL' do
+end

data/spec/lib/klepto_spec.rb ADDED

@@ -0,0 +1,64 @@
+require 'spec_helper'
+describe Klepto do
+  it "should have a version" do
+    Klepto::VERSION.should_not be_nil
+  end
+end
+describe 'Scraping pages', :skip => false do
+  before(:each) do
+    @bot = Klepto::Bot.new do
+      syntax :css
+      headers({
+        'Referer'     => 'https://twitter.com',
+        'User-Agent'  => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22"
+      })
+      urls  'https://twitter.com/justinbieber'
+      crawl 'body' do
+        scrape "h1.fullname", :name
+        scrape '.username span.screen-name', :username
+        save do |params|
+          user = User.find_by_name(params[:username]) || User.new
+          user.update_attributes params
+        end
+      end
+      crawl 'li.stream-item' do
+        scrape do |node|
+          {:twitter_id => node['data-item-id']}
+        end
+        scrape '.content p', :content
+        scrape '._timestamp' do |node|
+          {timestamp: node['data-time']}
+        end
+        scrape '.time a' do |node|
+          {permalink: node[:href]}
+        end
+        save do |params|
+          tweet = Tweet.find_by_twitter_id(params[:twitter_id]) || Tweet.new
+          tweet.update_attributes params
+        end
+      end
+    end
+    @bot.start!
+  end
+  it 'should have collected some resources' do
+    @bot.crawlers.should have(2).crawlers
+    @bot.crawlers.first.resources.should have(1).user
+  end
+  it 'should persist resources' do
+    User.count.should be(1)
+    Tweet.count.should_not be(0)
+  end
+end