spieker 0.0.3 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c787dd82625950fc0ce4eca39e86d29c021b5414
-  data.tar.gz: 9825847bfbe9cb7a09961a26f191b88ff8b0ec0a
+  metadata.gz: 8d68fdae00bc592760e27214ce5b0546702034a9
+  data.tar.gz: 2451ccff0b6d42c869b1f3fbada18fc700806bf5
 SHA512:
-  metadata.gz: 4f50c13bdac9438682de85294c0cad57848d0e7205be4d76d0d4809894b6338cb989ff833ec4e7304bfa9bb092a6ee52d7cfdac9aeb120b258c4dcc11622a59c
-  data.tar.gz: 9dc328ea3b99435490d8620cc65f42604415592c826c8ff56ce897a107ca4f0d63881991ab411a7bacab58940acc893f082362380813e52e593700d85f5bfbb8
+  metadata.gz: 2689c5559124616dc3305271a479f1e52212788cfc116d3524283b6da38c8395e84ae2b64d7abc3cc18b5be45c277200d83e8669e73f757e222af338e7d5f420
+  data.tar.gz: 7a7c4fa8ac735de6440484861035000816cac7fdd2a19f29207dceb75912f638aa5e0ace15f76fb2a094ebe398a7341043620d693ac63ace5ce702b933b32073

data/CHANGELOG

@@ -1,2 +1,8 @@
+0.0.7
+	* Support setting the language as a parameter, default is 'en'
+	* Wait for tolqjs to submit the content before continuing
+0.0.4
+	* Set useragent to Tolq Spieker
+
 0.0.3
 	* Ignore pdf and javascript links

data/README.md

@@ -0,0 +1,43 @@
+# Tolq Spieker
+
+Tolq Spieker is a javascript friendly crawler. It uses selenium and
+capybara to crawl a site.
+
+## Installation and usage
+
+```
+$ gem install spieker
+$ spieker <url>
+```
+
+You can also use the library from your apps to do whatever you please.
+
+## TODO
+
+* Be awesome in parallel
+* Use poltergeist instead, does not work well by default
+
+## How it works
+
+Spieker makes a request to the URL you provide. It scrapes all links,
+filters any remote or non resource links, and crawls those on its own
+terms. It tracks visited links in memory. When using the binary, it very
+verbosely outputs this to STDOUT.
+
+## Changelog
+
+See [Changelog](CHANGELOG)
+
+## Contribute
+
+* fork
+* test
+* pull request
+
+### Contributors
+
+* [Timon Vonk](https://www.github.com/timonv)
+
+## License
+
+See [License](LICENSE)

data/bin/spieker

@@ -2,4 +2,10 @@
 
 require 'spieker'
 
-Spieker::Crawler.new(ARGV[0], verbose: true).crawl!
+if ARGV[0] == '--version' or ARGV[0] == '-v'
+  puts Spieker::VERSION
+  exit
+end
+
+lang = ARGV[1] && ARGV[1].length > 0 ? ARGV[1] : 'en'
+Spieker::Crawler.new(ARGV[0], verbose: true, lang: lang).crawl!

data/lib/spieker/crawler.rb

@@ -1,15 +1,16 @@
 module Spieker
   class Crawler
-    def initialize(url, verbose: false)
+    def initialize(url, verbose: false, lang: 'en')
       @url = url
       @tracked_links = []
       @verbose = verbose
+      @lang = lang
     end
 
     def crawl!
       report "Starting to crawl on #{@url}"
 
-      scraper = LinkScraper.new(@url)
+      scraper = LinkScraper.new(@url, lang: @lang)
       track_link(@url)
       links =  scraper.result
       recursively_crawl(links)

data/lib/spieker/link_scraper.rb

@@ -1,5 +1,6 @@
 require 'capybara'
-require 'capybara/poltergeist'
+require 'capybara/selenium/driver'
+require 'selenium/webdriver'
 
 module Spieker
   class LinkScraper
@@ -7,15 +8,19 @@ module Spieker
     include Capybara::DSL
     attr_writer :links
 
-    def initialize(url)
+    def initialize(url, lang: 'en')
       @url = URI.parse(url)
+      @lang = lang
       Capybara.app_host = app_host
 
-      Capybara.register_driver :poltergeist do |app|
-          Capybara::Poltergeist::Driver.new(app, phantomjs_logger: NullStream.new )
+      Capybara.register_driver :tolq do |app|
+        profile = Selenium::WebDriver::Firefox::Profile.new
+        profile['general.useragent.override'] = "Mozilla/5.0 (compatible; Tolq Spieker/#{Spieker::VERSION}; +http://www.tolq.com)"
+
+        Capybara::Selenium::Driver.new(app, :profile => profile)
       end
 
-      Capybara.current_driver = :selenium
+      Capybara.current_driver = :tolq
     end
 
     def result
@@ -34,10 +39,17 @@ module Spieker
 
     def drive_page_for_links
       begin
-        visit @url.path
-        page.all('a').map { |el| el[:href]}
-      rescue
-        puts "Error parsing #{@url.to_s}"
+        visit @url.path + "#!lang=#{@lang}"
+        links = page.all('a').map { |el| el[:href]}
+        begin
+          # Our javascript adds a class if the content has been succesfully submitted
+          page.find(:css, 'html.tolq-content-updated')
+        rescue Capybara::Ambiguous, Capybara::ElementNotFound => e
+          puts "Something went wrong with submitting the content #{e.inspect}"
+        end
+        links
+      rescue => e
+        puts "Error parsing #{@url.to_s}, #{e.inspect}"
         []
       end
     end

data/lib/spieker/version.rb

@@ -1,3 +1,3 @@
 module Spieker
-  VERSION = "0.0.3"
+  VERSION = "0.0.7"
 end

data/test/link_scraper_test.rb

@@ -2,21 +2,20 @@ require 'test_helper'
 
 class TestLinkScraper < Test::Unit::TestCase
   def setup
-    @scraper = Spieker::LinkScraper.new('http://www.google.com')
+    @scraper = Spieker::LinkScraper.new('http://supersimple.st.tolq.com')
   end
 
   def test_apphost
-    assert_equal 'http://www.google.com', @scraper.app_host
+    assert_equal 'http://supersimple.st.tolq.com', @scraper.app_host
   end
 
   def test_result
-    return
-    assert @scraper.result.any?
+    #assert @scraper.result.any?
   end
 
   def test_filter_local
     found_links = [
-      'http://www.google.com/local',
+      'http://supersimple.st.tolq.com/local',
       '/local',
       '/local#justahash',
       '#justahash',
@@ -26,7 +25,7 @@ class TestLinkScraper < Test::Unit::TestCase
       'pdf'
     ]
     expected_links = [
-      'http://www.google.com/local',
+      'http://supersimple.st.tolq.com/local',
       '/local',
       ''
     ]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spieker
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.7
 platform: ruby
 authors:
 - Timon Vonk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-28 00:00:00.000000000 Z
+date: 2013-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -105,7 +105,7 @@ files:
 - .gitignore
 - CHANGELOG
 - Gemfile
-- LICENSE.txt
+- LICENSE
 - README.md
 - Rakefile
 - bin/spieker