blinkr 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f3dba6e6e0d389b3760f1f8e5fed683f52f93bd
-  data.tar.gz: bf28ba4b5e000d6492429e92409a97434e628106
+  metadata.gz: e6d0248a7f8b5ffd6fb131c9bfd554d542fcf4e2
+  data.tar.gz: cc36b93643ea59c8b2874f31fbec41f0460a353b
 SHA512:
-  metadata.gz: 3ade22d86f835fe6addb8262bd416d3f8cd797ba342aa447b7bd6ff76080229a81189b56538c789de67e1bd1461a57657c6c11075571ef0820bec19bc746b670
-  data.tar.gz: 819b85bfc6f8fd85ccd13040a0cb3832444a7019f97f5b16f4fa402f26694f996dbe3bfdd414127e002d996837b809b40a9f9434c4d44ef4062c994070605f02
+  metadata.gz: a48ff2eb5cc0d25694bc17f6d34c7443199cf4521bd6564e21f197e99bc0d39d6ced7a4169049ff32f6f72c163b771914bd60f298b491b8d873f169cbb70f0ae
+  data.tar.gz: 521419f058076d99248ed4bf0642a5be3152ad7d698f0089e0d5bf6031d1e8b7f1544b2279712437f2c36ce91008591978aa7d81bf5099ea41e507d77afeb9d6

data/lib/blinkr/extensions/links.rb

@@ -59,7 +59,8 @@ module Blinkr
         end
         external_links.each do |url, metadata|
           # if link start_with? @config.base_url check to see if it's in the sitemap.xml
-          browser.process(url, @config.max_retrys, :method => :get, :followlocation => true) do |resp|
+          browser.process(url, @config.max_retrys, :method => :get, :followlocation => true, :timeout => 30,
+                                                   :connecttimeout => 10, :maxredirs => 3) do |resp|
             puts "Loaded #{url} via #{browser.name} #{'(cached)' if resp.cached?}" if @config.verbose
 
             if resp.code.to_i < 200 || resp.code.to_i > 300

data/lib/blinkr/phantomjs_wrapper.rb

@@ -4,6 +4,7 @@ require 'tempfile'
 require 'blinkr/http_utils'
 require 'blinkr/cache'
 require 'parallel'
+require 'open3'
 
 module Blinkr
   class PhantomJSWrapper 
@@ -39,38 +40,39 @@ module Blinkr
     def _process url, limit, max, opts = {}, &block
       raise "limit must be set. url: #{url}, limit: #{limit}, max: #{max}" if limit.nil?
       unless @config.skipped? url
-        Tempfile.open('blinkr') do|f|
-          # Try and sidestep any unnecessary requests by checking the cache
-          request = Typhoeus::Request.new(url)
-          if @cache.get(request)
-            return block.call response, @cache.get("#{url}-resourceErrors"), @cache.get("#{url}-javascriptErrors")
-          end
+        #Tempfile.open('blinkr') do|f|
+        # Try and sidestep any unnecessary requests by checking the cache
+        request = Typhoeus::Request.new(url)
+        if @cache.get(request)
+          return block.call response, @cache.get("#{url}-resourceErrors"), @cache.get("#{url}-javascriptErrors")
+        end
 
-          if system "phantomjs #{SNAP_JS} #{url} #{@config.viewport} #{f.path}"
-            json = JSON.load(File.read(f.path))
-            response = Typhoeus::Response.new(:code => 200, :body => json['content'], :effective_url => json['url'],
+        output, status = Open3.capture2("phantomjs #{SNAP_JS} #{url} #{@config.viewport}")
+        if status == 0
+          json = JSON.load(output)
+          response = Typhoeus::Response.new(:code => 200, :body => json['content'], :effective_url => json['url'],
+                                            :mock => true)
+          response.request = Typhoeus::Request.new(url)
+          Typhoeus.stub(url).and_return(response)
+          @cache.set(response.request, response)
+          @cache.set("#{url}-resourceErrors", json['resourceErrors'])
+          @cache.set("#{url}-javascriptErrors", json['javascriptErrors'])
+          block.call response, json['resourceErrors'], json['javascriptErrors']
+        else
+          if limit > 1
+            puts "Loading #{url} via phantomjs (attempt #{max - limit + 2} of #{max})" if @config.verbose
+            _process url, limit - 1, max, &block
+          else
+            puts "Loading #{url} via phantomjs failed" if @config.verbose
+            response = Typhoeus::Response.new(:code => 0, :status_message => "Server timed out after #{max} retries",
                                               :mock => true)
             response.request = Typhoeus::Request.new(url)
             Typhoeus.stub(url).and_return(response)
             @cache.set(response.request, response)
-            @cache.set("#{url}-resourceErrors", json['resourceErrors'])
-            @cache.set("#{url}-javascriptErrors", json['javascriptErrors'])
-            block.call response, json['resourceErrors'], json['javascriptErrors']
-          else
-            if limit > 1
-              puts "Loading #{url} via phantomjs (attempt #{max - limit + 2} of #{max})" if @config.verbose
-              _process url, limit - 1, max, &block
-            else
-              puts "Loading #{url} via phantomjs failed" if @config.verbose
-              response = Typhoeus::Response.new(:code => 0, :status_message => "Server timed out after #{max} retries",
-                                                :mock => true)
-              response.request = Typhoeus::Request.new(url)
-              Typhoeus.stub(url).and_return(response)
-              @cache.set(response.request, response)
-              block.call response, nil, nil
-            end
+            block.call response, nil, nil
           end
-        end
+          end
+        #end
         @count += 1
       end
     end

data/lib/blinkr/snap.js

@@ -2,8 +2,8 @@ var system = require('system');
 var page = require('webpage').create();
 var fs = require('fs');
 
-if (system.args.length === 3) {
-    console.log('Usage: snap.js <URL> <view port width> <out>');
+if (system.args.length === 2) {
+    console.log('Usage: snap.js <URL> <view port width>');
     phantom.exit();
 }
 
@@ -13,7 +13,6 @@ info.javascriptErrors = [];
 info.url = system.args[1];
 info.view_port_width = system.args[2];
 
-var out = system.args[3]
 var current_requests = 0;
 var last_request_timeout;
 var final_timeout;
@@ -75,7 +74,7 @@ var exit = function exit(err_code) {
   info.content = page.content;
   // Re-read the URL in case we've been redirected
   info.url = page.url;
-  fs.write(out, JSON.stringify(info), 'w');
+  system.stdout.write(JSON.stringify(info));
   if (err_code === undefined) {
     err_code = 0;
   }

data/lib/blinkr/typhoeus_wrapper.rb

@@ -2,6 +2,7 @@ require 'typhoeus'
 require 'blinkr/cache'
 require 'blinkr/http_utils'
 require 'net/http'
+require 'logger'
 
 module Blinkr
   class TyphoeusWrapper
@@ -16,7 +17,10 @@ module Blinkr
       Typhoeus::Config.cache = Blinkr::Cache.new
       @hydra = Typhoeus::Hydra.new(:maxconnects => (@config.maxconnects || 30),
                                    :max_total_connections => (@config.maxconnects || 30),
+                                   :pipelining => false,
                                    :max_concurrency => (@config.maxconnects || 30))
+      Ethon.logger = Logger.new STDOUT if config.vverbose
+      Ethon::Curl.set_option(:max_host_connections, 5, @hydra.multi.handle, :multi)
       @count = 0
       @context = context
     end

data/lib/blinkr/version.rb

@@ -1,4 +1,4 @@
 module Blinkr
-  VERSION='0.3.2'
+  VERSION='0.3.3'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: blinkr
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Pete Muir
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2015-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler