phantom_proxy 1.0.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org'
+gem 'rack'

data/README.rdoc

@@ -0,0 +1,37 @@
+== Phantom Proxy - A webkit proxy
+The phantom proxy acts as a http proxy server. It fetches the remote webpages
+with the help phantomjs (see http://www.phantomjs.org/ ).
+
+You can use this to get a page after the javascipt execution. By setting some HTTP
+headers you can get the page with all iframes included or as an image. 
+
+== Installation
+Install phanotmjs (see: http://code.google.com/p/phantomjs/wiki/BuildInstructions)
+
+On Debian:
+
+ sudo apt-get install libqt4-dev libqtwebkit-dev qt4-qmake
+ cd phantom
+ git clone https://github.com/ariya/phantomjs.git
+ git checkout 1.2
+ qmake-qt4 && make
+
+ checkout phantom_proxy
+ 
+ gem build phantom_proxy.gemspec
+
+ gem install phantom_proxy-*.gem
+         
+== Usage
+Run
+ phantom_proxy 
+either with -self (ip, port) to not use the thin::runner framework
+or
+with any thin parameter you want (e.g. -p 8080).
+
+Point your browser's proxy to http://localhost:8080 for testting.
+
+You can use the Net::HTTP lib to fetch page or use the phantom_client 
+(see: https://github.com/experteer/phantom_client).
+ 
+== TODO

data/bin/phantom_proxy

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'thin'
+
+require 'fileutils'
+require 'timeout'
+require 'stringio'
+require 'time'
+require 'forwardable'
+require 'rack'
+require 'daemons'
+
+module PhantomJSProxy
+	CONFIG = File.expand_path(File.dirname(__FILE__))+"/../lib/phantom_proxy/config.ru"
+end
+
+require 'phantom_proxy'
+
+# Become a daemon
+options = {
+  :app_name		=> "phantom_proxy",
+  :backtrace  => true,
+  :ontop      => true,
+  :log_output => true
+}
+#Daemons.daemonize(options)
+phantom = false
+ARGV.each { |arg|
+    phantom = true if /-self/.match(arg)
+}
+
+if !phantom
+  startoptions = ["start", "-R", PhantomJSProxy::CONFIG, "-P", "/tmp/pids/phantom_proxy.pid", "--tag", "phantom_proxy"]+ARGV
+  Thin::Runner.new(startoptions).run!
+else
+  Thin::Server.start(PhantomJSProxy::PhantomJSServer.new, ARGV[0], ARGV[1], ARGV[2])
+end

data/lib/phantom_proxy/config.ru

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'phantom_proxy'
+
+# You can install Rack middlewares
+# to do some crazy stuff like logging,
+# filtering, auth or build your own.
+use Rack::CommonLogger
+
+run PhantomJSProxy::PhantomJSServer.new()
+

data/lib/phantom_proxy/phantomjs.rb

@@ -0,0 +1,84 @@
+require 'tempfile'
+
+module PhantomJSProxy
+	class PhantomJS
+		attr_accessor :dom
+		attr_accessor :image
+		attr_accessor :ready
+	
+		def initialize()
+			@ready = false
+		end
+		
+		def getUrl(url, pictureOnly=true, loadIFrames=true)
+			puts("PhantomJS: "+url)
+			@ready = false
+			
+			pictureFile = nil
+			picture = "none"
+      
+      loadFrames = "false"
+      
+      if loadIFrames
+        loadFrames = "true"
+      end
+			
+			if pictureOnly
+				if !File.directory?("/tmp/phantomjs_proxy")
+					Dir.mkdir("/tmp/phantomjs_proxy")
+				end
+				pictureFile = Tempfile.new(["phantomjs_proxy/page", ".png"])
+				picture = pictureFile.path
+			end
+			
+			url_args = ""
+			url_args_ = []
+      
+			if /\?/.match(url)
+				url_args = url.split('?')[1]
+				url = url.split('?')[0]
+				
+				if url_args
+					url_args_ = url_args.split('&')
+					url_args = url_args_.join(' ')
+				end
+			end
+			
+			@dom = invokePhantomJS(SCRIPT, [picture, loadFrames, url, url_args_.length, url_args])
+			
+			puts("Opened page: "+ /Open page: (.*?) END/.match(@dom)[1])
+			
+			if /DONE_LOADING_URL/.match(@dom)
+				@dom = @dom.split('PHANTOMJS_DOMDATA_WRITE:')[1];
+				@dom = @dom.split('PHANTOMJS_DOMDATA_END')[0]
+				if pictureOnly && File.exist?(picture) 
+					puts("File is there")
+					@image = IO::File.open(picture, "rb") {|f| f.read }
+					pictureFile.close!
+				else
+					puts("No file to load at: "+picture)
+					@image = ""
+				end
+				@ready = true
+			else
+				@dom = "Failed to load page"
+				puts("TOTAL FAIL")
+			end
+			puts("Return dom")
+			return @dom
+		end
+		
+		def getAsImageResponse(type='png')
+			return "HTTP/1.0 200 OK\r\nConnection: close\r\nContent-Type: image/"+type+"\r\n\r\n"+@image;
+		end
+	
+		def invokePhantomJS(script, args)
+			argString = " "+args.join(" ")
+			puts("Call phantomJS with: "+argString)
+			out = IO.popen(PHANTOMJS_BIN+" --cookies-file=/tmp/phantomjs_proxy/cookies.txt "+script+argString)
+			o = out.readlines.join
+      puts("PHANTOMJS_OUT: "+o)
+			return o
+		end
+	end
+end

data/lib/phantom_proxy/phantomjsserver.rb

@@ -0,0 +1,104 @@
+require 'net/http'
+
+module PhantomJSProxy
+	class PhantomJSServer
+		def initialize()
+		end
+    
+    def check_for_route(url)
+      if /\.js/i.match(url)
+        return 'text/html';
+      end
+      if /\.css/i.match(url)
+        return 'text/css'
+      end
+      if /\.png/i.match(url) or /\.jpg/i.match(url) or /\.jpeg/i.match(url) or /\.gif/i.match(url)
+        return 'image/*';
+      end
+      "none"
+    end
+    
+    def route(env, type)
+      _req = Net::HTTP::Get.new(env['REQUEST_URI'])
+      
+      _req['User-Agent'] = env['HTTP_USER_AGENT']
+      
+      _res = Net::HTTP.start(env['HTTP_HOST'], env['SERVER_PORT']) {|http|
+        #http.request(_req)
+        http.get(env['REQUEST_URI'])
+      }
+      
+      env['rack.errors'].write("Response is:"+_res.body+"\n")
+      
+      resp = Rack::Response.new([], 200, 	{'Content-Type' => type}) { |r|
+        r.write(_res.body)
+      }
+      resp.finish
+    end
+    
+		def call(env)
+			req = Rack::Request.new(env)
+			
+			haha = env.collect { |k, v| "#{k} : #{v}\n" }.join
+			env['rack.errors'].write("The request: "+req.url()+"\nGET: "+haha+"\n")
+			
+			params = req.params.collect { |k, v| "#{k}=#{v}&\n" }.join
+			env['rack.errors'].write("Paramas: "+params+"\n")
+      
+      #this routes the request to the outgoing server incase its not html that we want to load
+      type = check_for_route(env['REQUEST_URI'])
+      if type != "none"
+          return route(env, type)
+      end
+			
+				
+			#Fetch the Webpage with PhantomJS
+			phJS = PhantomJS.new
+			
+			env['rack.errors'].write("Extract the uri\n")
+			
+      if defined? env['HTTP_GET_PAGE_AS_IMAGE']
+			  picture = env['HTTP_GET_PAGE_AS_IMAGE']
+      else
+        picture = true
+      end
+      
+			if defined? env['HTTP_GET_PAGE_WITH_IFRAMES']
+        loadFrames = env['HTTP_GET_PAGE_WITH_IFRAMES']
+      else
+        loadFrames = false
+      end
+      
+      url = env['REQUEST_URI'];
+      if params.length > 0
+        url += '?'+params;
+      end
+			
+			phJS.getUrl(url, picture, loadFrames)
+				
+			#Create the response
+			if !phJS.ready
+				resp = Rack::Response.new([], 503, 	{
+																								'Content-Type' => 'text/html'
+																						}) { |r|
+					r.write(phJS.dom)
+				}
+				resp.finish
+			elsif picture
+				resp = Rack::Response.new([], 200, 	{
+																								'Content-Type' => 'image/png'
+																						}) { |r|
+					r.write(phJS.image)
+				}
+				resp.finish
+			else
+				resp = Rack::Response.new([], 200, 	{
+																								'Content-Type' => 'text/html'
+																						}) { |r|
+					r.write(phJS.dom)
+				}
+				resp.finish
+			end
+		end
+	end
+end

data/lib/phantom_proxy/scripts/proxy.js

@@ -0,0 +1,169 @@
+var fs = require('fs');
+
+var framesWorked = 0;
+var frameCount = 1;
+var frameContent = [];
+var masterURL = "";
+
+evaluateWithVars = function(page, func, vars)
+{
+	var fstr = func.toString()
+	//console.log(fstr.replace("function () {", "function () {\n"+vstr))
+	var evalstr = fstr.replace(
+		new RegExp("function \((.*?)\) {"),
+		"function $1 {\n" + 
+			"var vars = JSON.parse('" + JSON.stringify(vars) + "')\n" +
+			"for (var v in vars) window[v] = vars[v]\n" +
+		"\n"
+	)
+	console.log(evalstr)
+	return page.evaluate(evalstr)
+}
+
+var insertFrames = function(url) {
+	var page = require('webpage').create();
+	page.onConsoleMessage = function (msg) { console.log(msg); };
+  page.onAlert = function(msg) { console.log(msg);};
+  page.onLoadStarted = function () {
+    console.log('Start loading final Page...'+url);
+	};
+	page.open(url, function (status) {
+        if (status !== 'success') {
+            console.log('FAILED_LOADING_URL: '+url);
+        } else {
+        	page.evaluate(function () {
+        		var framestmp = document.getElementsByTagName('IFRAME');
+		      	var frames = []
+		      	for (var i=0;i<framestmp.length;i++) {
+		      		frames.push(framestmp[i]);
+		      	}
+		      	//mark iframes
+		      	for (var i in frames) {
+		      		frames[i].innerHTML = "PHANTOMJS_PROXY_IFRAME"+i;
+		      	}
+					});
+					//replace iframes with their data
+					var content = new String(page.content);
+					for (var i in frameContent) {
+						content = content.replace("PHANTOMJS_PROXY_IFRAME"+i, "<phantomjsframe>"+frameContent[i]+"</phantomjsframe>");
+		      }
+        	console.log("PHANTOMJS_DOMDATA_WRITE:"+content);
+		      console.log('PHANTOMJS_DOMDATA_END');
+        }
+        console.log('WHATEVER');
+        phantom.exit();
+  });
+};
+
+function exit() {
+  framesWorked++;
+	if (framesWorked == frameCount)
+		insertFrames(masterURL);
+}
+								
+var loadpage = function(url) {
+  var page = require('webpage').create();
+	page.onConsoleMessage = function (msg) { console.log(msg); };
+  //page.onLoadFinished = 
+  page.onAlert = function(msg) { console.log(msg);};
+  page.onLoadStarted = function () {
+    console.log('Start loading...'+url);
+	};
+	page.open(url, function (status) {
+        if (status !== 'success') {
+            console.log('FAILED_LOADING_URL: '+url);
+        } else {
+            console.log('LOADED PAGE CONTENT['+url+']\n');
+            frameContent.push(page.content);
+        }
+        console.log('WHATEVER');
+        exit();
+  });
+};
+
+function loadIFrames(page) {
+  var frames = page.evaluate(function () {
+    var framestmp = document.getElementsByTagName('IFRAME');
+    var frames = []
+    for (var i=0;i<framestmp.length;i++) {
+      frames.push(framestmp[i].getAttribute('src'));
+    }
+    return frames;
+  });
+  
+  for (var i=0;i<frames.length;i++) {
+    console.log("Frame: "+i+" : "+frames[i]);
+    loadpage(frames[i]);
+  }
+  
+  frameCount = frames.length+1;
+}
+
+function main() {
+
+	if (phantom.args.length < 2) {
+		  console.log('Usage: proxy.js <picture filename or none> <load iframe(true/false)> <URL> <url param count> <url params...>');
+		  phantom.exit();
+	} else {
+		  file_name = phantom.args[0];
+		  var loadIframes = phantom.args[1].match(/true/i) ? true : false;
+		  address = phantom.args[2];
+      
+      var argCount = phantom.args[3];
+		  
+		  args = ""
+		  for (var i=0;i<argCount;i++)
+		  	args += phantom.args[i+4]+'&';
+		  if (args.length > 0)
+		  	address += '?'+args;
+		  	
+		  console.log("Open page: "+address+", "+args+" END");
+		  
+		  var page = require('webpage').create();
+		  
+		  page.onConsoleMessage = function (msg) { console.log(msg); };
+		  
+		  console.log('start openning page');
+		  
+		  masterURL = address;
+		  
+		  page.open(address, function (status) {
+		      if (status !== 'success') {
+		          console.log('FAILED_LOADING_URL');
+		      } else {
+		      		console.log('DONE_LOADING_URL');
+		      		
+		      		//load iframes into page
+		      		if (loadIframes) {
+                loadIFrames(page);
+                /*
+				    		var frames = page.evaluate(function () {
+				    			var framestmp = document.getElementsByTagName('IFRAME');
+				    			var frames = []
+				    			for (var i=0;i<framestmp.length;i++) {
+				    				frames.push(framestmp[i].getAttribute('src'));
+				    			}
+				    			return frames;
+								});
+								
+								for (var i=0;i<frames.length;i++) {
+							
+									console.log("Frame: "+i+" : "+frames[i]);
+									loadpage(frames[i]);
+				    		}
+				    		frameCount = frames.length+1;
+                */
+							}
+		      		//evaluateWithVars(page, function(){}, phantom.args);
+		      		console.log('PHANTOMJS_MAINDOM_WRITE:'+page.content);
+		      		console.log('PHANTOMJS_MAINDOM_END');
+		      }
+		      if (file_name != null && file_name != "none") {
+		  			page.render(file_name);
+		  		}
+		  		exit();
+		  });
+	}
+}
+
+main();

data/lib/phantom_proxy.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+
+module PhantomJSProxy
+	ROOT = File.expand_path(File.dirname(__FILE__))
+	SCRIPT = ROOT+"/phantom_proxy/scripts/proxy.js"
+	PHANTOMJS_BIN = ROOT+'/../bin/phantomjs'
+end
+
+require PhantomJSProxy::ROOT+'/phantom_proxy/phantomjs.rb'
+require PhantomJSProxy::ROOT+'/phantom_proxy/phantomjsserver.rb'

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: phantom_proxy
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Daniel Sudmann
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+description: This is a phyntonjs Proxy it allows you to fetch webpages and execute
+  javascript in them.
+email: suddani@googlemail.com
+executables:
+- phantom_proxy
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/phantom_proxy/phantomjsserver.rb
+- lib/phantom_proxy/phantomjs.rb
+- lib/phantom_proxy.rb
+- lib/phantom_proxy/scripts/proxy.js
+- lib/phantom_proxy/config.ru
+- bin/phantom_proxy
+- bin/phantomjs
+- README.rdoc
+- Gemfile
+homepage: http://experteer.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.19
+signing_key: 
+specification_version: 3
+summary: This is a phantomjs Proxy
+test_files: []