phantom_mechanize 0.0.3 → 0.0.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZDExZGVjMDRhZDFlZTQzMmFhMmY4Mzc4YmVmNzc3MGJkZTljMmI3Yw==
+    ODBjYTZjMzJkNGFiNTZiYWFiMzQ0NzA4Njk2MTRlNTFiN2FkZjRmOA==
   data.tar.gz: !binary |-
-    MGFmY2VkY2Q0YTg4MDk5N2EyNzlmYWViMzc1NWJmYjcyNTZhMzE4Mg==
+    N2FiOWEyZWZjNDhiNjg0ZjQ1NzY0NGQ0MzJiNGM2MzJkZmJjNGNhNw==
 SHA512:
   metadata.gz: !binary |-
-    NmIyNjk1ZTZhYjc1MjkxOTJmMGU1MTRhOTYwMTYzOWExY2RjMGQ1OTRhMzEx
-    OGM2OTM5YjU0M2IyMGQ3NTliOWRjZjQ5ZDA3ZGQ1YzdiMTZhODZjODVkOGQ4
-    ODEzOGJlNjhiNWNkYmFkODI4ZTFmZDE3YWI3MjUxM2M5NjMyMTY=
+    ZWNlZDFiODljMTU3NWY0OTNkZmU3NTAzZjBkOGM4MTc1YjczMjE0NGRhYWE2
+    ZmMyZTc2YmJiYjBiYTI3ZWE5ZjVhZThhMjhmNDNhN2Q0NDhlOWIyZWM3ODhm
+    YTBjMjhjZmVjMDliMzQ0OTJkMDA3YWIyZTE3NDkxN2RlY2QwNjU=
   data.tar.gz: !binary |-
-    ZDhhNTUzMjNjMjNmODk0YWQ3MjRlZWE2N2MyNTk1NzhjODk2YzQwM2RkYWMx
-    NjkxNjk5NDg4MjYwYjhmMWEzOTZlN2RmM2JiMWY3NzMxZDI0OTc0Y2RkZTJm
-    MDNhMmQyOTRjODZiZDY5NGIxYjZhNmRiZjhmOTEwODViNjM5MDI=
+    NGFlMmViZTZhZTczOTNjMjJmYWY3YWU2NzIxNTUwYjIxMmJjNTUwZjUzY2Yx
+    MDUzZmVkY2Y3Yjk3YWFjNDM4N2ZkYTA3Yjg1ODRiYmI4ZTYzZTI5NjhkYmM3
+    OTgyNDlkNWM4MjE4NzA2ZmM0Nzg3NjE5NGU5Y2RmZDI2YWI4NDQ=

data/README.md

@@ -27,11 +27,15 @@ page = agent.phget 'http://www.google.com', :wait => 10000, :selector => '[name=
 # use some js to submit a form or interact with the dom:
 page = agent.phget 'http://www.google.com', :selector => ['[name=q]', 'h3 a'], :js => "$('[name=q]').val('phantom_mechanize');$('form').submit()"
 
+# return page after scrolling has completed
+page = agent.phget 'http://www.somescrollingpage.com', :scroll => true
+
 
 ```
 ## Options
 * :selector - return once this selector is located (jquery)
 * :wait - wait this many milliseconds (default 10,000)
+* :scroll - scroll to the bottom until no more results show up (infinite scrolling)
 
 ## Faq
 > What about cookies?

data/js/phget.js

@@ -8,6 +8,7 @@ var selector = selectors[0];
 var cookies = JSON.parse(args[4]);
 var user_agent = args[5];
 var jss = JSON.parse(args[6]);
+var scroll = JSON.parse(args[7]);
 
 // var date = 
 for(i in cookies){
@@ -31,13 +32,15 @@ function output(html, page){
 
 var page = require('webpage').create();
 
-setInterval(function() {
-  var html = page.evaluate(function() {
-    return document.documentElement.outerHTML;
-  });
-  output(html, page);
-  output(page.content, page);
-}, timeout);
+if(!scroll){
+  setInterval(function() {
+    var html = page.evaluate(function() {
+      return document.documentElement.outerHTML;
+    });
+    output(html, page);
+    output(page.content, page);
+  }, timeout);
+}
 
 setInterval(function() {
   page.render('phantomjs.png');
@@ -119,4 +122,23 @@ page.onError = function(msg, trace) {
 
 page.open(url, function() {
 
+  if(scroll){
+    var num = 0;
+    window.setInterval(function() {
+      num2 = page.evaluate(function() {
+          // Scrolls to the bottom of page
+          window.document.body.scrollTop = document.body.scrollHeight;
+          return document.body.scrollHeight;
+      });
+      if (num2 == num){
+        var html = page.evaluate(function() {
+          return document.documentElement.outerHTML;
+        });
+        output(html, page);
+      }
+      num = num2;
+      // console.log(num);
+    }, timeout); // Number of milliseconds to wait between scrolls
+  }
+
 });

data/lib/phantom_mechanize/ext/mechanize.rb

@@ -4,6 +4,7 @@ class Mechanize
     args = args[0] || {}
     wait = args[:wait] || 10000
     selector = args[:selector] || ""
+    scroll = args[:scroll] ? 1 : 0
     selector = [selector] if selector.is_a?(String)
     js = args[:js] || ""
     js = [js] if js.is_a?(String)
@@ -12,9 +13,10 @@ class Mechanize
 
     ph_args = ['--ssl-protocol=any', '--web-security=false']
     ph_args << "--proxy=#{proxy_addr}:#{proxy_port}" if proxy_port && proxy_addr
-    puts "phantomjs #{ph_args.join(' ')} \"#{PhantomMechanize::JS_FOLDER}/phget.js\" \"#{url}\" \"#{wait}\" \"#{selector.to_json.gsub('"', '\"')}\" \"#{pc.gsub('"', '\"')}\" \"#{user_agent.gsub('"', '\"')}\" \"#{js.to_json.gsub('"', '\"')}\""
 
-    response = `phantomjs #{ph_args.join(' ')} "#{PhantomMechanize::JS_FOLDER}/phget.js" "#{url}" "#{wait}" "#{selector.to_json.gsub('"', '\"')}" "#{pc.gsub('"', '\"')}" "#{user_agent.gsub('"', '\"')}" "#{js.to_json.gsub('"', '\"')}"`
+    cmd = "phantomjs #{ph_args.join(' ')} \"#{PhantomMechanize::JS_FOLDER}/phget.js\" \"#{url}\" \"#{wait}\" \"#{selector.to_json.gsub('\"', '\\"')}\" \"#{pc.gsub('\"', '\\"')}\" \"#{user_agent.gsub('\"', '\\"')}\" \"#{js.to_json.gsub('\"', '\\"')}\" \"#{scroll.to_json}\""
+    response = %x[#{cmd}]
+
     raise 'bad response' if response == ''
 
     mcs, html = response.split '<<<phget_separator>>>'

data/lib/phantom_mechanize/version.rb

@@ -1,3 +1,3 @@
 module PhantomMechanize
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: phantom_mechanize
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - P Guardiario
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-08 00:00:00.000000000 Z
+date: 2015-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler