spieker 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8d68fdae00bc592760e27214ce5b0546702034a9
-  data.tar.gz: 2451ccff0b6d42c869b1f3fbada18fc700806bf5
+  metadata.gz: 1efbda5b0c8acddf71421f0a398fe81147bba156
+  data.tar.gz: b8d089ba60167155d2f502e6bbe6f06110d55a58
 SHA512:
-  metadata.gz: 2689c5559124616dc3305271a479f1e52212788cfc116d3524283b6da38c8395e84ae2b64d7abc3cc18b5be45c277200d83e8669e73f757e222af338e7d5f420
-  data.tar.gz: 7a7c4fa8ac735de6440484861035000816cac7fdd2a19f29207dceb75912f638aa5e0ace15f76fb2a094ebe398a7341043620d693ac63ace5ce702b933b32073
+  metadata.gz: 33cbdaa59a122b8572c84e6697dd8f2e3c066248a0a770bee2766ec8a0de261eda9f90591f4ae3fc2539a39e402bac78286bec9e5bc0df4f77ac224f4fc61e3a
+  data.tar.gz: 68074bacc9d188afcaf4ce2ac896b6684c3dcea36034b16760a53a564262999628bdf2fedc31a1e2ff3c90e415e7fb286a3f6ace046ecd4e392eff64a4584da1

data/CHANGELOG

@@ -1,6 +1,11 @@
+0.0.8
+	* Handle query params like a boss
+	* Fix small bug with language not always being set
+
 0.0.7
 	* Support setting the language as a parameter, default is 'en'
 	* Wait for tolqjs to submit the content before continuing
+
 0.0.4
 	* Set useragent to Tolq Spieker
 

data/lib/spieker/crawler.rb

@@ -28,7 +28,7 @@ module Spieker
       links.each do |link|
         report "Crawling page #{link}"
 
-        scraper = LinkScraper.new(link)
+        scraper = LinkScraper.new(link, lang: @lang)
         new_links_threaded[link] = scraper.result
         track_link(link)
 

data/lib/spieker/link_scraper.rb

@@ -39,17 +39,24 @@ module Spieker
 
     def drive_page_for_links
       begin
-        visit @url.path + "#!lang=#{@lang}"
-        links = page.all('a').map { |el| el[:href]}
+        query = if @url.query
+                  "?#{@url.query}"
+                else
+                  ""
+                end
+        visit @url.path + query + "#!lang=#{@lang}"
+        # Capybara + selenium causes some links not to be found. There doesn't seem to be any method to that.
+        # Cool is tho, in JS it's a lot faster as well
+        links = page.evaluate_script('document.getElementsByTagName(\'a\')').map { |el| el['href'] }
         begin
           # Our javascript adds a class if the content has been succesfully submitted
           page.find(:css, 'html.tolq-content-updated')
         rescue Capybara::Ambiguous, Capybara::ElementNotFound => e
-          puts "Something went wrong with submitting the content #{e.inspect}"
+          puts "Something went wrong with submitting the content: #{e.message}"
         end
         links
       rescue => e
-        puts "Error parsing #{@url.to_s}, #{e.inspect}"
+        puts "Error parsing #{@url.to_s}, #{e.message}"
         []
       end
     end

data/lib/spieker/version.rb

@@ -1,3 +1,3 @@
 module Spieker
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spieker
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Timon Vonk