phantom_proxy 1.2.14 → 1.2.15

data/.gitignore

@@ -0,0 +1,12 @@
+phantom_proxy-*.gem
+.rvmrc
+log/
+
+#IDE
+.redcar
+.project
+.rbenv-gemsets
+
+doc
+!.keep
+Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+ree-1.8.7-2012.02

data/Gemfile

@@ -1,2 +1,5 @@
 source 'http://rubygems.org'
+
+gemspec
+
 gem 'rack'

data/lib/phantom_proxy/phantomjs.rb

@@ -68,7 +68,7 @@ module PhantomJSProxy
 			if /URL_ERROR_CODE/.match(@dom)
         puts("LOAD_ERROR_CODE")
 			  @ready = getHTTPCode @dom
-        puts("LOAD_ERROR_CODE_DONE")
+        puts("LOAD_ERROR_CODE_DONE: #{@ready}")
 			end
 			@dom = dom_text
 			return @dom

data/lib/phantom_proxy/phantomjsserver.rb

@@ -3,6 +3,20 @@ require 'hmac-md5'
 require 'base64'
 
 module PhantomJSProxy
+  class Options
+    def initialize(env)
+      @env = env
+    end
+
+    def picture?
+      @env['HTTP_GET_PAGE_AS_IMAGE']||false
+    end
+
+    def loadFrames?
+      @env['HTTP_GET_PAGE_WITH_IFRAMES']||true
+    end
+  end
+
 	class PhantomJSServer
 		def initialize
 		  @control_panel = PhantomJSProxy::PhantomJSControlPanel.new
@@ -78,28 +92,13 @@ module PhantomJSProxy
       return true
     end
 
-    def getOptions(env)
-    	if defined? env['HTTP_GET_PAGE_AS_IMAGE']
-    		picture = env['HTTP_GET_PAGE_AS_IMAGE']
-      else
-      	picture = true
-      end
-      
-      if defined? env['HTTP_GET_PAGE_WITH_IFRAMES']
-      	loadFrames = env['HTTP_GET_PAGE_WITH_IFRAMES']
-      else
-        loadFrames = false
-      end
-      
-      return picture,loadFrames
-    end
-
 		def prepareUrl(env, params, req, https_request, type)
 			if type == "none"
-				url = env['REQUEST_URI'];
+				url = req.url#env['REQUEST_URI']
+        puts "URL is: #{url}"
         if https_request
-          url['http'] = 'https'
-          url[':443'] = ''
+          url['http'] = 'https' if url['http']
+          url[':443'] = '' if url[':443']
         end
         
         if params.length > 0
@@ -108,7 +107,7 @@ module PhantomJSProxy
         return url
 			end
 			url = Base64.decode64(req.params["address"])
-			env['rack.errors'].write("After Base64 decoding: "+url)
+			env['rack.errors'].write("After Base64 decoding: "+url+"\n")
 			return url
 		end
 
@@ -138,7 +137,7 @@ module PhantomJSProxy
 			env['rack.errors'].write("Paramas: "+params+"\n")
       
       #this routes the request to the outgoing server incase its not html that we want to load
-      type = check_for_route(env['REQUEST_URI'])
+      type = check_for_route(req.url)#env['REQUEST_URI'])
       if type == "control_panel"
         return control_panel.show()
       elsif type != "none" and type != "base64"
@@ -149,15 +148,25 @@ module PhantomJSProxy
         phJS = PhantomJS.new
         
         env['rack.errors'].write("Extract the uri\n")
-        
-        picture,loadFrames = getOptions(env)
+
+        loadOptions = Options.new(env)
+
+        puts "Options: #{loadOptions.picture?}, #{loadOptions.loadFrames?}"
         
         url = prepareUrl(env, params, req, https_request, type)
         
-        phJS.getUrl(url, picture, loadFrames)
+        phJS.getUrl(url, loadOptions.picture?, loadOptions.loadFrames?)
 
         #Create the response
-        if phJS.ready != 200
+        if loadOptions.picture?
+          control_panel.add_special_request "@image_requests"
+          resp = Rack::Response.new([], 200,  {
+                                                  'Content-Type' => 'image/png'
+                                              }) { |r|
+            r.write(phJS.image)
+          }
+          resp.finish
+        elsif phJS.ready != 200
           if !/favicon\.ico/.match(req.url())
             env['rack.errors'].write("Request FAILED\n")
             control_panel.add_special_request "@failed_requests"
@@ -170,14 +179,6 @@ module PhantomJSProxy
             r.write(phJS.dom)
           }
           resp.finish
-        elsif picture
-          control_panel.add_special_request "@image_requests"
-          resp = Rack::Response.new([], 200,  {
-                                                  'Content-Type' => 'image/png'
-                                              }) { |r|
-            r.write(phJS.image)
-          }
-          resp.finish
         else
           control_panel.add_special_request "@html_requests"
           resp = Rack::Response.new([], 200,  {

data/lib/phantom_proxy/scripts/proxy.js

@@ -6,103 +6,72 @@ var frameContent = [];
 var masterURL = "";
 var masterPage = null;
 
-function evaluateWithVars(page, func, vars)
-{
-	var fstr = func.toString()
-	//console.log(fstr.replace("function () {", "function () {\n"+vstr))
-	var evalstr = fstr.replace(
-		new RegExp("function \((.*?)\) {"),
-		"function $1 {\n" + 
-			"var vars = JSON.parse('" + JSON.stringify(vars) + "')\n" +
-			"for (var v in vars) window[v] = vars[v]\n" +
-		"\n"
-	)
-	console.log(evalstr)
-	return page.evaluate(evalstr)
-}
-
-function insertFrames(url) {
-
-	var page = masterPage;
-	/*
-	var page = require('webpage').create();
-	page.onConsoleMessage = function (msg) { console.log(msg); };
-	page.onAlert = function(msg) { console.log(msg);};
-	page.onLoadStarted = function () {
-		console.log('Start loading final Page...'+url);
-	};
-	page.open(url, function (status) {
-		if (status !== 'success') {
-            console.log('FAILED_LOADING_URL: '+url);
-        } else {*/
-        	page.evaluate(function () {
-        		var framestmp = document.getElementsByTagName('IFRAME');
-		      	var frames = []
-		      	for (var i=0;i<framestmp.length;i++) {
-		      		frames.push(framestmp[i]);
-		      	}
-		      	//mark iframes
-		      	for (var i in frames) {
-		      		frames[i].innerHTML = "PHANTOMJS_PROXY_IFRAME"+i;
-		      	}
-			});
-
-			//replace iframes with their data
-			var content = new String(page.content);
-			for (var i in frameContent) {
-				content = content.replace("PHANTOMJS_PROXY_IFRAME"+i, "<phantomjsframe>"+frameContent[i]+"</phantomjsframe>");
+function newIFrameLoad(page, load_iframes) {
+	var frame_data = [];
+	if (load_iframes) {
+	  frame_data = page.evaluate(function () {
+	    var framestmp = document.getElementsByTagName('IFRAME');
+			var frames = [];
+			for (var i=0;i<framestmp.length;i++) {
+				frames.push(framestmp[i].contentWindow.document.body.innerHTML);
+				framestmp[i].outerHTML = "<phantomjsframe>PHANTOM_JS_FRAME_"+i+"</phantomjsframe>";
 			}
-			console.log("PHANTOMJS_DOMDATA_WRITE:"+content);
-			console.log('PHANTOMJS_DOMDATA_END');
-        //}
-        console.log('WHATEVER');
-        phantom.exit();
-  //});
-};
-
-function exit() {
-	framesWorked++;
-	if (framesWorked == frameCount)
-		insertFrames(masterURL);
+			return frames;
+		});
+	}
+	var content = new String(page.content);
+	for (var i=0;i<frame_data.length;i++) {
+		content = content.replace("PHANTOM_JS_FRAME_"+i, frame_data[i]);
+	}
+	console.log("PHANTOMJS_DOMDATA_WRITE:"+content);
+	console.log('PHANTOMJS_DOMDATA_END');
 }
 								
-var loadpage = function(url) {
+var loadpage = function(url, referer, success, failure, configure) {
+	var redirectURL = null;
+
 	var page = require('webpage').create();
+
+	page.settings.localToRemoteUrlAccessEnabled = true;
+
+	page.settings.userAgent = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/34.0.1847.116 Chrome/34.0.1847.116 Safari/537.36";
+
 	page.onConsoleMessage = function (msg) { console.log(msg); };
-	//page.onLoadFinished =
+
 	page.onAlert = function(msg) { console.log(msg);};
+
 	page.onLoadStarted = function () {
 		console.log('Start loading...'+url);
 	};
-	page.open(url, function (status) {
-		if (status !== 'success') {
-            console.log('FAILED_LOADING_URL: '+url);
-        } else {
-            console.log('LOADED PAGE CONTENT['+url+']\n');
-            frameContent.push(page.content);
-        }
-        console.log('WHATEVER');
-        exit();
-    });
-};
 
-function loadIFrames(page) {
-	var frames = page.evaluate(function () {
-		var framestmp = document.getElementsByTagName('IFRAME');
-		var frames = [];
-		for (var i=0;i<framestmp.length;i++) {
-			frames.push(framestmp[i].getAttribute('src'));
-		}
-		return frames;
-	});
+	page.onResourceReceived = function(resource) {
+    if (url == resource.url && resource.redirectURL && resource.stage == "end") {
+      redirectURL = resource.redirectURL;
+      console.log('FRAME_URL_ERROR_CODE: '+resource.status+'FRAME_URL_ERROR_CODE_END');
+    }
+  };
+
+	if (referer!=undefined) {
+		console.log("Set Referer: "+referer);
+		page.customHeaders = {
+			"REFERER": referer
+		};
+	}
 
-	frameCount = frames.length+1;
+	if (configure != undefined)
+		configure(page);
 
-	for (var i=0;i<frames.length;i++) {
-		console.log("Frame: "+i+" : "+frames[i]);
-		loadpage(frames[i]);
-	}
-}
+	page.open(url, function (status) {
+		console.log("Page Status: "+status);
+		if (redirectURL) {
+      loadpage(redirectURL, url, success, failure);
+    } else if (status !== 'success') {
+	    failure(page, url);
+    } else {
+    	success(page, url);
+    }
+  });
+};
 
 function main() {
 
@@ -118,49 +87,43 @@ function main() {
 
 		args = ""
 
-		for (var i=0;i<argCount;i++)
-			args += phantom.args[i+4]+'&';
+		for (var i=0;i<argCount;i++) {
+			args += phantom.args[i+4];
+			if (i<argCount-1) args += "&"
+		}
 		if (args.length > 0)
 			address += '?'+args;
 
 		console.log("Open page: "+address+", "+args+" END");
 
-		var page = require('webpage').create();
-
-		page.onConsoleMessage = function (msg) { console.log(msg); };
-
 		console.log('start openning page');
 
 		masterURL = address;
-		
-		masterPage = page;
-
-		//catches status != 200 and throws error immidiatly
-		page.onResourceReceived = function (response) {
-			if (response.stage == "end" && response.url == address && response.status != 200)
-			{
-				console.log('URL_ERROR_CODE: '+response.status+'URL_ERROR_CODE_END');
-				//phantom.exit();
-			}
-		};
 
-		page.open(address, function (status) {
-			if (status != 'success') {
-				console.log('FAILED_LOADING_URL');
-			} else {
-				console.log('DONE_LOADING_URL');
-				//load iframes into page
-				if (loadIframes) {
-					loadIFrames(page);
-				}
-				//evaluateWithVars(page, function(){}, phantom.args);
-				console.log('PHANTOMJS_MAINDOM_WRITE:'+page.content);
-				console.log('PHANTOMJS_MAINDOM_END');
+		loadpage(address, undefined, function(page) {
+			masterPage = page;
+			console.log('DONE_LOADING_URL');
+			//load iframes into page
+			newIFrameLoad(page, loadIframes);
+			if (file_name != null && file_name != "none") {
+				page.render(file_name);
 			}
+			phantom.exit();
+		}, function(page) {
+			masterPage = page;
+			console.log('FAILED_LOADING_URL: '+status+", "+address);
 			if (file_name != null && file_name != "none") {
 				page.render(file_name);
 			}
-			exit();
+			phantom.exit();
+		}, function(page) {
+			//page.customHeaders = {"Referer": "http://uk-amazon.icims.com/jobs/240290/account-representative---amazon-web-services---iberia/job"}
+			// page.onResourceReceived = function (response) {
+			// 	if (response.stage == "end" && response.url == address && response.status != 200)
+			// 	{
+			// 		console.log('URL_ERROR_CODE: '+response.status+'URL_ERROR_CODE_END');
+			// 	}
+			// };
 		});
 	}
 }

data/phantom_proxy.gemspec

@@ -0,0 +1,28 @@
+require 'rubygems'
+
+Gem::Specification.new do |s|
+  s.name        = 'phantom_proxy'
+  s.version     = '1.2.15'
+  s.summary     = "This is a phantomjs Proxy"
+  s.description = "This is a phyntonjs Proxy it allows you to fetch webpages and execute javascript in them."
+  s.authors     = ["Daniel Sudmann"]
+  s.email       = 'suddani@googlemail.com'
+  s.files       = `git ls-files`.split($\)
+=begin
+                  FileList['lib/**/*.rb',
+  										'lib/**/*.js',
+                      'lib/**/**/*.ru',
+                      'lib/**/**/*.html',
+                      'lib/**/**/*',
+                      'lib/**/**/**/*',
+                      'lib/phantom_proxy/install/**/*',
+                      'lib/phantom_proxy/install/**/**/*',
+                      'bin/*',
+                      '[A-Z]*',
+                      'test/**/*'].to_a
+=end
+  s.homepage    = 'http://experteer.com'
+  s.executables = ['phantom_proxy']
+  s.add_dependency('thin', '>= 1.3.1')
+  s.add_dependency('ruby-hmac', '>= 0.4.0')
+end

data/spec/test

@@ -0,0 +1,2 @@
+#bash
+bin/phantomjs lib/phantom_proxy/scripts/proxy.js tmp/picture.png true http://dl.dropbox.com/u/36978459/index.html > tmp/log.txt

data/tmp/run

@@ -0,0 +1,5 @@
+gem uninstall phantom_proxy -x
+rm phantom_proxy-*.*.*.gem
+gem build phantom_proxy.gemspec
+gem install phantom_proxy-*.*.*.gem
+phantom_proxy -p 3003

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: phantom_proxy
 version: !ruby/object:Gem::Version 
-  hash: 3
+  hash: 1
   prerelease: 
   segments: 
   - 1
   - 2
-  - 14
-  version: 1.2.14
+  - 15
+  version: 1.2.15
 platform: ruby
 authors: 
 - Daniel Sudmann
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-01-14 00:00:00 +01:00
+date: 2014-05-09 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -59,18 +59,24 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-- lib/phantom_proxy/phantomjsserver.rb
-- lib/phantom_proxy/phantomjs_control_panel.rb
-- lib/phantom_proxy/phantomjs.rb
+- .gitignore
+- .ruby-version
+- Gemfile
+- README.rdoc
+- bin/phantom_proxy
 - lib/phantom_proxy.rb
-- lib/phantom_proxy/scripts/proxy.js
 - lib/phantom_proxy/config.ru
-- lib/phantom_proxy/web/control_panel.html
 - lib/phantom_proxy/install/etc/init.d/phproxy
 - lib/phantom_proxy/install/etc/phantom_proxy/config/example_config
-- bin/phantom_proxy
-- README.rdoc
-- Gemfile
+- lib/phantom_proxy/phantomjs.rb
+- lib/phantom_proxy/phantomjs_control_panel.rb
+- lib/phantom_proxy/phantomjsserver.rb
+- lib/phantom_proxy/scripts/proxy.js
+- lib/phantom_proxy/web/control_panel.html
+- phantom_proxy.gemspec
+- spec/test
+- tmp/.keep
+- tmp/run
 has_rdoc: true
 homepage: http://experteer.com
 licenses: []
@@ -101,7 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.5.2
+rubygems_version: 1.5.3
 signing_key: 
 specification_version: 3
 summary: This is a phantomjs Proxy