phantom_mechanize 0.0.2 → 0.0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZWJlNzExOGNkYmFjMjBhNjg5Yjg0MWI1NTNkZTMxM2ViZTkyY2MzZA==
+    ZDExZGVjMDRhZDFlZTQzMmFhMmY4Mzc4YmVmNzc3MGJkZTljMmI3Yw==
   data.tar.gz: !binary |-
-    OTFlYjdlZWNkNjI4ZTg0YzMzNmE1ODZmOGE4YmZhNmZlZDQ5OWM3Mw==
+    MGFmY2VkY2Q0YTg4MDk5N2EyNzlmYWViMzc1NWJmYjcyNTZhMzE4Mg==
 SHA512:
   metadata.gz: !binary |-
-    MTRmZWIzNDFkYTNiYjlmZTRmZTc1MmI5ZGU2OGU5ZGFkYjVkY2NkMzUyMTM0
-    ZWQwMDRjNGU5OWQxNTkxYjA0MjlhYWFkZTBmY2FlMGI0MTcyZTk0YjMxNzZl
-    NWYxZWU0MDg4NDU1MzIzZGI2NDBhZTkyNzMwMDc0ZDJiODc1YWE=
+    NmIyNjk1ZTZhYjc1MjkxOTJmMGU1MTRhOTYwMTYzOWExY2RjMGQ1OTRhMzEx
+    OGM2OTM5YjU0M2IyMGQ3NTliOWRjZjQ5ZDA3ZGQ1YzdiMTZhODZjODVkOGQ4
+    ODEzOGJlNjhiNWNkYmFkODI4ZTFmZDE3YWI3MjUxM2M5NjMyMTY=
   data.tar.gz: !binary |-
-    ZTY3NWQzYjcxZTA1N2QzM2E3MzliNDE4ZGUwNjEyNGY1OGI3N2M2ZDg2Zjlk
-    NjY3MTI1N2FmODU5Nzg4ZGQ0ODU2Y2M0ODBjODBhNmJmZWU1ODNlNDVmYjRi
-    NTY3NTYwNmJkYjU4MjAyYTEzNTY5OTdiYTI1NWJlMmZkNGEwNjM=
+    ZDhhNTUzMjNjMjNmODk0YWQ3MjRlZWE2N2MyNTk1NzhjODk2YzQwM2RkYWMx
+    NjkxNjk5NDg4MjYwYjhmMWEzOTZlN2RmM2JiMWY3NzMxZDI0OTc0Y2RkZTJm
+    MDNhMmQyOTRjODZiZDY5NGIxYjZhNmRiZjhmOTEwODViNjM5MDI=

data/.gitignore

@@ -1,14 +1 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-*.bundle
-*.so
-*.o
-*.a
-mkmf.log
+pkg

data/js/phget.js

@@ -23,27 +23,54 @@ for(i in cookies){
   });
 }
 
+function output(html, page){
+  if(!html.match(/<html/i)) return;
+  console.log(JSON.stringify(phantom.cookies, null, 2) + '<<<phget_separator>>>' + html);
+  phantom.exit();
+}
+
 var page = require('webpage').create();
 
 setInterval(function() {
-  page.includeJs('//ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function() {
-    page.evaluate(function() {
-      console.log($(':root').html());
-    });
+  var html = page.evaluate(function() {
+    return document.documentElement.outerHTML;
   });
+  output(html, page);
+  output(page.content, page);
 }, timeout);
 
+setInterval(function() {
+  page.render('phantomjs.png');
+}, 500);
+
+
 page.settings.userAgent = user_agent;
 
 page.onConsoleMessage = function(msg, lineNum, sourceId) {
-  console.log(JSON.stringify(phantom.cookies, null, 2) + '<<<phget_separator>>>' + msg);
-  phantom.exit();
+  console.log(msg);
+};
+
+page.onInitialized = function() {
+  page.evaluate(function() {
+    // spoof plugins
+    window.navigator = {plugins: {length: 5}};
+    // remove stuff that will give us away
+    delete window.callPhantom;
+  });
 };
 
 page.onResourceRequested = function(requestData, networkRequest) {
-  if(requestData.url.match(/.(gif|jpe?g|png|css)/)) {
+  // skip loading images / styles
+  if(requestData.url.match(/\.(gif|jpe?g|png|css)\b/i)) {
     networkRequest.abort(); 
   }
+
+  host = networkRequest.setHeader('Host');
+  networkRequest.setHeader('Host', 0);
+  networkRequest.setHeader('Host', host);
+  networkRequest.setHeader('Connection', 'keep-alive');
+  //networkRequest.setHeader('Accept-Encoding', 'gzip, deflate');
+
 };
 
 page.onLoadFinished = function() {
@@ -54,27 +81,30 @@ page.onLoadFinished = function() {
     var js = jss.shift();
     var done = (selectors[0] == undefined && js == undefined);
 
+    if(selector !== undefined){
     page.includeJs('//ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function() {
-      page.evaluate(function(selector, page, js, done) {
+      html = page.evaluate(function(selector, page, js, done) {
         setInterval(function() {
           if($(selector)[0]){
-            if(js){
+            if(js !== undefined){
               eval(js);
             }
             if(done){
-              console.log($(':root').html());
-            }
-            
+              return document.documentElement.outerHTML;
+            }            
           }
         }, 500);
       }, selector, page, js, done);
-
+      output(html, page);
     });
+//    console.log(document.documentElement.outerHTML);
+    }
+
   }
 };
 
 page.onError = function(msg, trace) {
-  /*
+ /*
     var msgStack = ['ERROR: ' + msg];
     if (trace && trace.length) {
         msgStack.push('TRACE:');
@@ -82,9 +112,9 @@ page.onError = function(msg, trace) {
             msgStack.push(' -> ' + t.file + ': ' + t.line + (t.function ? ' (in function "' + t.function + '")' : ''));
         });
     }
-   */
-    // uncomment to log into the console 
-    // console.error(msgStack.join('\n'));
+   
+    console.error(msgStack.join('\n'));
+*/
 };
 
 page.open(url, function() {

data/lib/phantom_mechanize/ext/mechanize.rb

@@ -1,5 +1,6 @@
 class Mechanize
 	def phget url, *args
+    
     args = args[0] || {}
     wait = args[:wait] || 10000
     selector = args[:selector] || ""
@@ -9,11 +10,13 @@ class Mechanize
 
     pc = cookies.map{|c| [c.name, c.value, c.domain, c.path, c.httponly, c.secure, c.expires.to_i]}.to_json
 
-    ph_args = []
+    ph_args = ['--ssl-protocol=any', '--web-security=false']
     ph_args << "--proxy=#{proxy_addr}:#{proxy_port}" if proxy_port && proxy_addr
-    # puts "phantomjs #{ph_args.join(' ')} \"#{PhantomMechanize::JS_FOLDER}/phget.js\" \"#{url}\" \"#{wait}\" \"#{selector.to_json.gsub('"', '\"')}\" \"#{pc.gsub('"', '\"')}\" \"#{user_agent.gsub('"', '\"')}\" \"#{js.to_json.gsub('"', '\"')}\""
+    puts "phantomjs #{ph_args.join(' ')} \"#{PhantomMechanize::JS_FOLDER}/phget.js\" \"#{url}\" \"#{wait}\" \"#{selector.to_json.gsub('"', '\"')}\" \"#{pc.gsub('"', '\"')}\" \"#{user_agent.gsub('"', '\"')}\" \"#{js.to_json.gsub('"', '\"')}\""
 
     response = `phantomjs #{ph_args.join(' ')} "#{PhantomMechanize::JS_FOLDER}/phget.js" "#{url}" "#{wait}" "#{selector.to_json.gsub('"', '\"')}" "#{pc.gsub('"', '\"')}" "#{user_agent.gsub('"', '\"')}" "#{js.to_json.gsub('"', '\"')}"`
+    raise 'bad response' if response == ''
+
     mcs, html = response.split '<<<phget_separator>>>'
     JSON.parse(mcs).each do |mc|
       cookie = Cookie.new Hash[mc.map{|k, v| [k.to_sym, v]}]

data/lib/phantom_mechanize/version.rb

@@ -1,3 +1,3 @@
 module PhantomMechanize
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: phantom_mechanize
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - P Guardiario
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-14 00:00:00.000000000 Z
+date: 2015-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler