RubyGems - klepto - Versions diffs - 0.1.0 - Mend

klepto 0.1.0

Files changed (32) hide show

data/.gitignore +21 -0
data/.rspec +2 -0
data/.rvmrc +1 -0
data/Gemfile +18 -0
data/Guardfile +11 -0
data/LICENSE.txt +22 -0
data/README.md +129 -0
data/Rakefile +7 -0
data/klepto.gemspec +26 -0
data/lib/klepto.rb +26 -0
data/lib/klepto/bot.rb +59 -0
data/lib/klepto/browser.rb +18 -0
data/lib/klepto/crawler.rb +72 -0
data/lib/klepto/tasks.rb +15 -0
data/lib/klepto/version.rb +3 -0
data/samples/example.rb +49 -0
data/spec/cassettes/Klepto_Crawler/dsl_interaction/should_crawl_the_resource.yml +1960 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_designate_scraping_of_a_set_of_nodes.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_designate_scraping_of_a_single_node_with_a_block.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_designate_scraping_of_a_single_node_with_a_symbol.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_be_able_to_scrape_the_node_that_the_crawler_is_scoped_to.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_CSS_scope.yml +114 -0
data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_desired_syntax.yml +114 -0
data/spec/lib/klepto/bot_spec.rb +40 -0
data/spec/lib/klepto/browser_spec.rb +15 -0
data/spec/lib/klepto/crawler_spec.rb +88 -0
data/spec/lib/klepto/dsl_spec.rb +6 -0
data/spec/lib/klepto_spec.rb +64 -0
data/spec/orm/active_record.rb +36 -0
data/spec/orm/database.example.yml +15 -0
data/spec/spec_helper.rb +32 -0
metadata +157 -0

data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_CSS_scope.yml ADDED

@@ -0,0 +1,114 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://example.com/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: Found
+    headers:
+      location:
+      - http://www.iana.org/domains/example/
+      server:
+      - BigIP
+      connection:
+      - Keep-Alive
+      content-length:
+      - '0'
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.0'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: FOUND
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:03 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      location:
+      - http://www.iana.org/domains/example
+      content-length:
+      - '0'
+      connection:
+      - close
+      content-type:
+      - text/html; charset=utf-8
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:04 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      last-modified:
+      - Fri, 04 Jan 2013 01:17:22 GMT
+      vary:
+      - Accept-Encoding
+      connection:
+      - close
+      transfer-encoding:
+      - chunked
+      content-type:
+      - text/html; charset=UTF-8
+    body:
+      encoding: US-ASCII
+      string: ! "<!doctype html>\n<html>\n<head>\n\t<title>Example Domain</title>\n\n\t<meta
+        charset=\"utf-8\" />\n\t<meta http-equiv=\"Content-type\" content=\"text/html;
+        charset=utf-8\" />\n\t<meta name=\"viewport\" content=\"width=device-width,
+        initial-scale=1\" />\n\t<style type=\"text/css\">\n\tbody {\n\t\tbackground-color:
+        #f0f0f2;\n\t\tmargin: 0;\n\t\tpadding: 0;\n\t\tfont-family: \"Open Sans\",
+        \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n\t\t\n\t}\n\tdiv {\n\t\twidth:
+        600px;\n\t\tmargin: 5em auto;\n\t\tpadding: 3em;\n\t\tbackground-color: #fff;\n\t\tborder-radius:
+        1em;\n\t}\n\ta:link, a:visited {\n\t\tcolor: #38488f;\n\t\ttext-decoration:
+        none;\n\t}\n\t@media (max-width: 600px) {\n\t\tbody {\n\t\t\tbackground-color:
+        #fff;\n\t\t}\n\t\tdiv {\n\t\t\twidth: auto;\n\t\t\tmargin: 0 auto;\n\t\t\tborder-radius:
+        0;\n\t\t\tpadding: 1em;\n\t\t}\n\t}\n\t</style>\t\n</head>\n\n<body>\n<div>\n\t<h1>Example
+        Domain</h1>\n\t<p>This domain is established to be used for illustrative examples
+        in documents. You do not need to\n\t\tcoordinate or ask for permission to
+        use this domain in examples, and it is not available for\n\t\tregistration.</p>\n\t<p><a
+        href=\"http://www.iana.org/domains/special\">More information...</a></p>\n</div>\n</body>\n</html>\n"
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+recorded_with: VCR 2.4.0

data/spec/cassettes/Klepto_Crawler/standard_interaction/should_have_a_desired_syntax.yml ADDED

@@ -0,0 +1,114 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://example.com/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: Found
+    headers:
+      location:
+      - http://www.iana.org/domains/example/
+      server:
+      - BigIP
+      connection:
+      - Keep-Alive
+      content-length:
+      - '0'
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.0'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 302
+      message: FOUND
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:04 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      location:
+      - http://www.iana.org/domains/example
+      content-length:
+      - '0'
+      connection:
+      - close
+      content-type:
+      - text/html; charset=utf-8
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:03 GMT
+- request:
+    method: get
+    uri: http://www.iana.org/domains/example
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      accept:
+      - ! '*/*'
+      user-agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Wed, 10 Apr 2013 06:51:04 GMT
+      server:
+      - Apache/2.2.3 (CentOS)
+      last-modified:
+      - Fri, 04 Jan 2013 01:17:22 GMT
+      vary:
+      - Accept-Encoding
+      connection:
+      - close
+      transfer-encoding:
+      - chunked
+      content-type:
+      - text/html; charset=UTF-8
+    body:
+      encoding: US-ASCII
+      string: ! "<!doctype html>\n<html>\n<head>\n\t<title>Example Domain</title>\n\n\t<meta
+        charset=\"utf-8\" />\n\t<meta http-equiv=\"Content-type\" content=\"text/html;
+        charset=utf-8\" />\n\t<meta name=\"viewport\" content=\"width=device-width,
+        initial-scale=1\" />\n\t<style type=\"text/css\">\n\tbody {\n\t\tbackground-color:
+        #f0f0f2;\n\t\tmargin: 0;\n\t\tpadding: 0;\n\t\tfont-family: \"Open Sans\",
+        \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n\t\t\n\t}\n\tdiv {\n\t\twidth:
+        600px;\n\t\tmargin: 5em auto;\n\t\tpadding: 3em;\n\t\tbackground-color: #fff;\n\t\tborder-radius:
+        1em;\n\t}\n\ta:link, a:visited {\n\t\tcolor: #38488f;\n\t\ttext-decoration:
+        none;\n\t}\n\t@media (max-width: 600px) {\n\t\tbody {\n\t\t\tbackground-color:
+        #fff;\n\t\t}\n\t\tdiv {\n\t\t\twidth: auto;\n\t\t\tmargin: 0 auto;\n\t\t\tborder-radius:
+        0;\n\t\t\tpadding: 1em;\n\t\t}\n\t}\n\t</style>\t\n</head>\n\n<body>\n<div>\n\t<h1>Example
+        Domain</h1>\n\t<p>This domain is established to be used for illustrative examples
+        in documents. You do not need to\n\t\tcoordinate or ask for permission to
+        use this domain in examples, and it is not available for\n\t\tregistration.</p>\n\t<p><a
+        href=\"http://www.iana.org/domains/special\">More information...</a></p>\n</div>\n</body>\n</html>\n"
+    http_version: '1.1'
+  recorded_at: Wed, 10 Apr 2013 06:51:04 GMT
+recorded_with: VCR 2.4.0

data/spec/lib/klepto/bot_spec.rb ADDED

@@ -0,0 +1,40 @@
+require 'spec_helper'
+describe Klepto::Bot, :vcr => {:record => :new_episodes} do
+  before(:each) do
+    @bot = Klepto::Bot.new
+  end
+  it 'should know if it is a dry run' do
+    @bot.dry_run?.should be false
+    @bot.dry_run!
+    @bot.dry_run?.should be true
+  end
+  it 'should be able to set the selection syntax' do
+    @bot.syntax(:xpath)
+    @bot.syntax.should be(:xpath)
+  end
+  it 'should be able to read the selection syntax' do
+    @bot.syntax.should be(:css)
+  end
+  it 'should be able to set request headers' do
+    @bot.should respond_to(:headers)
+  end
+  it 'should be able to set a list of URLs to crawl' do
+    @bot.url 'http://www.google.com'
+    @bot.urls.should include('http://www.google.com')
+    @bot.urls 'http://twitter.com', 'http://facebook.com'
+    @bot.urls.should include('http://twitter.com')
+    @bot.urls.should include('http://facebook.com')
+  end
+  it 'should be able to add crawlers' do
+    @bot.crawl('div'){}
+    @bot.instance_variable_get("@crawlers").should have(1).crawler
+    @bot.instance_variable_get("@crawlers").first.should be_kind_of(Klepto::Crawler)
+  end
+end

data/spec/lib/klepto/browser_spec.rb ADDED

@@ -0,0 +1,15 @@
+require 'spec_helper'
+describe Klepto::Browser, :vcr => {:record => :new_episodes} do
+  before(:each) do
+    @browser = Klepto::Browser.new
+    @browser.set_headers({
+      'Referer'     => 'http://www.example.com'
+    })
+  end
+  it 'should be able to fetch a page' do
+    @page = @browser.fetch! 'http://www.example.com'
+    @page.status_code.should be(200)
+  end
+end

data/spec/lib/klepto/crawler_spec.rb ADDED

@@ -0,0 +1,88 @@
+require 'spec_helper'
+require 'open-uri'
+describe Klepto::Crawler, :vcr => {:record => :new_episodes} do
+  describe 'dsl interaction' do
+    before(:each) do
+      @page = page("http://www.iana.org")
+      @crawler = Klepto::Crawler.new('body',{:syntax => :css}) do
+        scrape 'h1', :title
+        scrape '#intro p' do |node|
+          {description: node.text}
+        end
+        scrape_all '.home-panel h2' do |nodes|
+          { sections: nodes.map{|n| n.text} }
+        end
+      end
+      @resources = @crawler.crawl @page
+    end #end before
+    it 'should crawl the resource' do
+      @resources.should have(1).resource
+      @resources.first[:title].should match('Internet Assigned Numbers Authority')
+      @resources.first[:description].should match(/^The Internet Assigned Numbers Authority/i)
+      @resources.first[:sections].should have(3).sections
+    end
+  end
+  describe 'standard interaction' do
+    before(:each) do
+      @page     = page()
+      @crawler  = Klepto::Crawler.new 'body', {:syntax => :css}
+    end
+    it 'should have a CSS scope' do
+      @crawler.scope.should eq 'body'
+    end
+    it 'should have a desired syntax' do
+      @crawler.syntax.should == :css
+    end
+    it 'should be able to scrape the node that the crawler is scoped to' do
+      @crawler.scrape do |node|
+        {:name => node.native.name}
+      end
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:name].should eq('body')
+    end
+    it 'should be able to designate scraping of a single node with a symbol' do
+      @crawler.scrape 'h1', :title
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:title].should eq('Example Domain')
+    end
+    it 'should be able to designate scraping of a single node with a block' do
+      @crawler.scrape 'h1' do |node|
+        {title: node.text}
+      end
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:title].should eq('Example Domain')
+    end
+    it 'should be able to designate scraping of a set of nodes' do
+      @crawler.scrape_all 'p' do |nodes|
+        {
+          paragraphs: [
+            nodes.first.text,
+            nodes.last.text
+          ]
+        }
+      end
+      resources = @crawler.crawl( @page )
+      resources.should have(1).resource
+      resources.first[:paragraphs].should be_kind_of(Array)
+      resources.first[:paragraphs].last.should eq("More information...")
+    end
+    pending 'should be able to save a set of resources'
+    pending 'should be able to specify a limit'
+    pending 'should be able to specify a skip'
+  end
+end

data/spec/lib/klepto/dsl_spec.rb ADDED

@@ -0,0 +1,6 @@
+require 'spec_helper'
+describe 'Klepto DSL' do
+end

data/spec/lib/klepto_spec.rb ADDED

@@ -0,0 +1,64 @@
+require 'spec_helper'
+describe Klepto do
+  it "should have a version" do
+    Klepto::VERSION.should_not be_nil
+  end
+end
+describe 'Scraping pages', :skip => false do
+  before(:each) do
+    @bot = Klepto::Bot.new do
+      syntax :css
+      headers({
+        'Referer'     => 'https://twitter.com',
+        'User-Agent'  => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22"
+      })
+      urls  'https://twitter.com/justinbieber'
+      crawl 'body' do
+        scrape "h1.fullname", :name
+        scrape '.username span.screen-name', :username
+        save do |params|
+          user = User.find_by_name(params[:username]) || User.new
+          user.update_attributes params
+        end
+      end
+      crawl 'li.stream-item' do
+        scrape do |node|
+          {:twitter_id => node['data-item-id']}
+        end
+        scrape '.content p', :content
+        scrape '._timestamp' do |node|
+          {timestamp: node['data-time']}
+        end
+        scrape '.time a' do |node|
+          {permalink: node[:href]}
+        end
+        save do |params|
+          tweet = Tweet.find_by_twitter_id(params[:twitter_id]) || Tweet.new
+          tweet.update_attributes params
+        end
+      end
+    end
+    @bot.start!
+  end
+  it 'should have collected some resources' do
+    @bot.crawlers.should have(2).crawlers
+    @bot.crawlers.first.resources.should have(1).user
+  end
+  it 'should persist resources' do
+    User.count.should be(1)
+    Tweet.count.should_not be(0)
+  end
+end