crabfarm 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb3be13182df3bfb8b36dbf9561680f76b0a8c4a
-  data.tar.gz: f2dfc98fd27fc4523ea715ec8dd811a98977237d
+  metadata.gz: 1031e279aeab473f8e46469b3f91383a4dffbd70
+  data.tar.gz: f366d36831117570d65b999de947bb5d6af0d0d1
 SHA512:
-  metadata.gz: db1a6932c2ff5645f3dfe8c77c3e092a39fd9c43ad52293098180cff4f5732cc1b7cd71eb5460556fcb56de456d11592a2ba04d5681bad4e198552fab12aa71d
-  data.tar.gz: 81b972f5cec55ba61d90ec943944ad17524b20fc199dbab37fad746a26de62631f18680efcc6f67cc12c03f1b383ca9ce800aa91f8d44df300e4579a57e1bdb3
+  metadata.gz: a5e46aeefc0f4fe6c96ad2f5d02ee312c826a5a724e20b9797818ddb4c876245b6e58331a85feaafc48ab7b7f97f38c3dd1fd497705c72020559bf84ee8c22e4
+  data.tar.gz: df9dd5a3874ef38e462d1adfcf8c4ae92bf4b8f7ed0cfda6456d7e3ed6a1ceb2b791e1dd6f89988bec33bdf8d3fa9ef7db86c38515c43ae1de47aa314876063e

data/bin/crabtrap

@@ -0,0 +1,347 @@
+#!/usr/bin/env node
+
+var net = require('net'),
+	http = require('http'),
+	https = require('https'),
+	url = require('url'),
+	fs = require('fs'),
+	zlib = require('zlib');
+
+// Globals
+
+var HTTPS_OPTIONS = {
+	key: '-----BEGIN RSA PRIVATE KEY-----\nMIIBOQIBAAJBAK/L/lXb/kxUzve1olo71s6mQLvuQCm3z2wqClq71NLerFnaXpN+\nFrNPy7+R3gZ1hdWXqbN5NqpWDMM9fcbd7p0CAwEAAQJAUDImN3Lhgl7Z/+TLSJCt\nwJ3VQCZC/QUOSdCv4o53Wy5aL/n8ootYFC3eoFC2Nal5bnH6onP9YR+X9l3HKLaT\n3QIhANXwb5SvJ+Kewa8F5wNHo9LFjSbL7WSSb1MyvYnOeFlPAiEA0lvaLz6UXRDL\n6T6Z1fkF0exmQqVimeL5qjY5o9Gk5lMCH1A52Z3oEQzqe7cmf3q7YrOnYUcrMdqF\nDzojzO/gfUECIQCe9fImiW+r9CljFH9Dhm6zd6S+8CNWjoKD8X4VITMvKQIgb3sg\nq9gPVzXn/+f8Qcc2KILSh3ffkIpA8yJK9omUIxI=\n-----END RSA PRIVATE KEY-----\n',
+	cert: '-----BEGIN CERTIFICATE-----\nMIIBmDCCAUICCQDGtiGKgI9AXjANBgkqhkiG9w0BAQUFADBTMQswCQYDVQQGEwJD\nTDELMAkGA1UECBMCUk0xETAPBgNVBAcTCFNhbnRpYWdvMREwDwYDVQQKEwhQbGF0\nYW51czERMA8GA1UEAxMIQ3JhYnRyYXAwHhcNMTUwMTE1MjAxNzMzWhcNNDIwNjAx\nMjAxNzMzWjBTMQswCQYDVQQGEwJDTDELMAkGA1UECBMCUk0xETAPBgNVBAcTCFNh\nbnRpYWdvMREwDwYDVQQKEwhQbGF0YW51czERMA8GA1UEAxMIQ3JhYnRyYXAwXDAN\nBgkqhkiG9w0BAQEFAANLADBIAkEAr8v+Vdv+TFTO97WiWjvWzqZAu+5AKbfPbCoK\nWrvU0t6sWdpek34Ws0/Lv5HeBnWF1Zeps3k2qlYMwz19xt3unQIDAQABMA0GCSqG\nSIb3DQEBBQUAA0EAmecqIZqQ8OXSIj0V2VKaIXwz8RBnhLzU7BJwcsWJE/Bex7zB\nWP+vLv9ML5ZRLCsXjL5IOav8qAX/NZXjoN3e3Q==\n-----END CERTIFICATE-----\n'
+};
+
+var LOG = {
+	DEBUG: 0,
+	INFO: 1,
+	WARN: 2,
+	ERROR: 3
+};
+
+var STACK = [],
+	MODE = false,
+	SOURCE = null,
+	PORT = 4000,
+	LOG_LEVEL = LOG.WARN;
+
+(function() {
+	if(process.argv.length < 2) throw 'Must provide a proxy mode';
+	MODE = process.argv[2];
+	var i = 3;
+
+	if(MODE != 'pass') {
+		if(process.argv.length < 3) throw 'Must provide a bucket path';
+		SOURCE = process.argv[3];
+		i = 4;
+	}
+
+	for(; i < process.argv.length; i++) {
+		var parts = process.argv[i].split('=');
+		switch(parts[0]) {
+			case '--port': PORT = parseInt(parts[1], 10); break;
+			case '--quiet': PORT = parseInt(parts[1], 10); break;
+			default: throw 'Invalid option ' + parts[0];
+		}
+	}
+})();
+
+// Utility methods
+
+function log(_level, _message) {
+	if(_level == LOG.DEBUG) _message = '\t' + _message;
+	if(_level >= LOG_LEVEL) console.log(_message);
+}
+
+function forOwn(_obj, _cb) {
+	for(var key in _obj) {
+		if(_obj.hasOwnProperty(key)) {
+			_cb(key, _obj[key]);
+		}
+	}
+}
+
+function keysToLowerCase(_obj) {
+	var result = {};
+	forOwn(_obj, function(k,v) { result[k.toLowerCase()] = v; });
+	return result;
+}
+
+function pickRandomPort() {
+	return 0; // This could fail on Linux...
+}
+
+function matchRequestToResource(_req, _resource) {
+	return _resource.method.toLowerCase() == _req.method.toLowerCase() && _resource.url == _req.url;
+}
+
+function matchRequestToResourceWOQuery(_req, _resource) {
+	if(_resource.method.toLowerCase() == _req.method.toLowerCase()) return false;
+
+	var reqUrl = url.parse(_req.url, true),
+		resUrl = url.parse(_resource.url, true);
+
+	return reqUrl.hostname == resUrl.hostname && reqUrl.pathname == resUrl.pathname;
+}
+
+function findAndMoveLast(_req, _array, _matches) {
+	for(var i = 0, l = _array.length; i < l; i++) {
+		if(_matches(_req, _array[i])) {
+			var resource = _array.splice(i, 1)[0];
+			_array.push(resource);
+			return resource;
+		}
+	}
+
+	return null;
+}
+
+function loadStackFrom(_path, _then) {
+	var data = fs.readFileSync(_path);
+	zlib.gunzip(data, function(err, buffer) {
+		if (!err) STACK = JSON.parse(buffer.toString());
+		_then();
+	});
+}
+
+function saveStackTo(_path, _then) {
+	var data = JSON.stringify(STACK);
+	zlib.gzip(data, function(err, buffer) {
+		if (!err) fs.writeFileSync(_path, buffer);
+		_then();
+	});
+}
+
+function resolveAndServeResource(_req, _resp) {
+	var resource = findInStack(_req);
+	if(resource) {
+		log(LOG.INFO, "Serving: " + resource.method + ' ' + resource.url);
+		log(LOG.DEBUG, "HTTP " + resource.statusCode);
+		log(LOG.DEBUG, JSON.stringify(resource.headers));
+
+		serveResource(resource, _resp);
+	} else {
+		log(LOG.WARN, 'Not found: ' + _req.url);
+		_resp.statusCode = 404;
+		_resp.end();
+	}
+}
+
+function serveLastResource(_resp) {
+	serveResource(STACK[STACK.length-1], _resp);
+}
+
+function serveResource(_resource, _resp) {
+	_resp.statusCode = _resource.statusCode;
+
+	forOwn(_resource.headers, function(k, v) { _resp.setHeader(k, v); });
+
+	if(_resource.content) {
+		var buf = new Buffer(_resource.content, _resource.encoding);
+		_resp.end(buf);
+	} else {
+		_resp.end();
+	}
+}
+
+function findAndMoveLast(_req, _matches) {
+	for(var i = 0, l = STACK.length; i < l; i++) {
+		if(_matches(_req, STACK[i])) {
+			var resource = STACK.splice(i, 1)[0];
+			STACK.push(resource);
+			return resource;
+		}
+	}
+
+	return null;
+}
+
+function findInStack(_req, _partial) {
+	return findAndMoveLast(_req, matchRequestToResource) ||
+		findAndMoveLast(_req, matchRequestToResourceWOQuery);
+}
+
+function cacheResponse(_req, _resp, _cb) {
+
+	log(LOG.INFO, "Caching Response");
+	log(LOG.DEBUG, "HTTP " + _resp.statusCode);
+	log(LOG.DEBUG, JSON.stringify(keysToLowerCase(_resp.headers)));
+
+	var encoding = null,
+		// TODO: consider storing port and protocoll in the resource.
+		resource = {
+			url: _req.url,
+			statusCode: _resp.statusCode,
+			method: _req.method,
+			// inHeaders: req.headers, // store request headers to aid in recognition?
+			headers: keysToLowerCase(_resp.headers),
+			content: '',
+			encoding: 'base64'
+		},
+		contentEncoding = resource.headers['content-encoding'],
+		contentType = resource.headers['content-type'],
+		outStream = _resp;
+
+	// add decompression if supported encoding:
+	if(contentEncoding == 'gzip') {
+		outStream = _resp.pipe(zlib.createGunzip());
+		delete resource.headers['content-encoding'];
+		contentEncoding = null;
+	} else if(contentEncoding == 'deflate') {
+		outStream = _resp.pipe(zlib.createInflate());
+		delete resource.headers['content-encoding'];
+		contentEncoding = null;
+	}
+
+	// use utf8 encoding for uncompresed text:
+	if(!contentEncoding && contentType) {
+		contentType = contentType.match(/([^\/]+)\/([^\s]+)(?:\s+(.+))?/i);
+		if(contentType && (contentType[1] == 'text' || contentType[1] == 'application')) {
+			resource.encoding = 'utf-8';
+		}
+	}
+
+	// remove unwanted headers:
+	delete resource.headers['content-length'];
+
+	// start receiving data:
+	if(resource.encoding) outStream.setEncoding(resource.encoding);
+	outStream.on('data', function(_chunk) {
+		resource.content += _chunk;
+	});
+
+	// when all data is received, store resource (dont know how this will handle more than one request)
+	outStream.on('end', function() {
+		STACK.push(resource);
+		_cb();
+	});
+}
+
+function prepareForwardRequest(_req) {
+	var urlObj = url.parse(_req.url);
+
+	var options = {
+		method: _req.method,
+		host: urlObj.host,
+		path: urlObj.path,
+		rejectUnauthorized: false,
+		headers: keysToLowerCase(_req.headers)
+	};
+
+	// Rewrite headers
+	options.headers['accept-encoding'] = 'gzip,deflate';
+	return options;
+}
+
+function passRequest(_req, _resp) {
+	log(LOG.INFO, 'Passing through ' + _req.method + ' request for ' + _req.url);
+
+	var urlObj = url.parse(_req.url);
+	var forward = (urlObj.protocol == 'https:' ? https : http).request({
+		method: _req.method,
+		host: urlObj.host,
+		path: urlObj.path,
+		headers: _req.headers
+	}, function(_fw_resp) {
+		// pipe response back untouched
+		_resp.writeHead(_fw_resp.statusCode, _fw_resp.headers);
+		_fw_resp.pipe(_resp);
+	});
+
+	_req.pipe(forward);
+}
+
+function captureRequest(_req, _resp, _useSSL) {
+	log(LOG.INFO, 'Forwarding ' + _req.method + ' request for ' + _req.url);
+
+	var urlObj = url.parse(_req.url);
+	var options = {
+		method: _req.method,
+		host: urlObj.host,
+		path: urlObj.path,
+		rejectUnauthorized: false,
+		headers: keysToLowerCase(_req.headers)
+	};
+
+	// Rewrite headers
+	options.headers['accept-encoding'] = 'gzip,deflate';
+	log(LOG.DEBUG, JSON.stringify(options));
+
+	var forward = (urlObj.protocol == 'https:' ? https : http).request(options, function(_fw_resp) {
+		cacheResponse(_req, _fw_resp, function() {
+			serveLastResource(_resp);
+		});
+	});
+
+	_req.pipe(forward); // forward request data
+}
+
+function replayRequest(_req, _resp) {
+	log(LOG.INFO, 'Resolving ' + _req.method + ' request for ' + _req.url);
+	resolveAndServeResource(_req, _resp);
+}
+
+function selectProxy() {
+	switch(MODE) {
+		case 'pass': return passRequest;
+		case 'capture': return captureRequest;
+		case 'replay': return replayRequest;
+		default: throw 'Invalid proxy mode';
+	}
+}
+
+var PROXY_FUN = selectProxy(),
+	SERVER = http.createServer(PROXY_FUN);
+
+// Special handler for HTTPS request, creates a dedicated HTTPS proxy per connection,
+// that way the CONNECT tunnel can be intercepted, requires support for self signed
+// certificates in the client.
+SERVER.on('connect', function (_req, _sock, _head) {
+
+	var urlObj = url.parse('http://' + _req.url);
+	log(LOG.INFO, 'New HTTPS request: starting https intercept on ' + urlObj.hostname);
+
+	var httpsServ = https.createServer(HTTPS_OPTIONS, function(_req, _resp) {
+		_req.url = 'https://' + urlObj.hostname + _req.url;
+		PROXY_FUN(_req, _resp);
+	});
+
+	httpsServ.listen(pickRandomPort());
+
+	var tunnelSock = net.connect(httpsServ.address().port, function() {
+		_sock.write('HTTP/1.1 200 Connection Established\r\n' +
+			'Proxy-agent: Node-Proxy\r\n' +
+			'\r\n');
+		tunnelSock.write(_head);
+		tunnelSock.pipe(_sock);
+		_sock.pipe(tunnelSock);
+	});
+
+	_sock.on('close', function() {
+		httpsServ.close();
+	});
+});
+
+console.log("Starting crabtrap! mode: " + MODE);
+
+if(MODE == 'replay') {
+	loadStackFrom(SOURCE, SERVER.listen.bind(SERVER, PORT));
+} else {
+	SERVER.listen(PORT);
+}
+
+var EXITING = false;
+process.on('SIGINT', function() {
+	if(EXITING) return;
+	EXITING = true;
+
+	console.log("Shutting down crabtrap!");
+	SERVER.close();
+	if(MODE == 'capture') {
+		saveStackTo(SOURCE, process.exit.bind(process));
+	} else {
+		process.exit();
+	}
+});

data/lib/crabfarm/cli.rb

@@ -10,14 +10,28 @@ module Crabfarm
     desc "Starts the crawler in console mode"
     command [:console, :c] do |c|
 
+      c.desc "Capture to crabtrap file"
+      c.flag :capture
+
+      c.desc "Replay from crabtrap file"
+      c.flag :replay
+
       Support::GLI.generate_options c
 
       c.action do |global_options,options,args|
         next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
 
-        require "crabfarm/modes/console"
         Crabfarm.config.set Support::GLI.parse_options options
-        Crabfarm::Modes::Console.start
+
+        next puts "Cannot use --replay with --capture" if options[:capture] and options[:replay]
+
+        require 'crabfarm/crabtrap_context'
+        context = Crabfarm::CrabtrapContext.new
+        context.capture options[:capture] if options[:capture]
+        context.replay options[:replay] if options[:replay]
+
+        require "crabfarm/modes/console"
+        Crabfarm::Modes::Console.start context
       end
     end
 
@@ -84,9 +98,29 @@ module Crabfarm
       end
     end
 
+    desc "Perform an HTTP recording for use in tests"
+    command [:record, :r] do |c|
+      c.action do |global_options, options, args|
+        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
+
+        require "crabfarm/modes/recorder"
+        Crabfarm::Modes::Recorder.start args[0]
+      end
+    end
+
+    desc "Publish the crawler to a crabfarm cloud"
     command :publish do |c|
+      c.desc "Just list the files that are beign packaged"
+      c.switch :dry, :default_value => false
+
+      c.desc "Don't check for pending changes"
+      c.switch :unsafe, :default_value => false
+
       c.action do |global_options,options,args|
+        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
 
+        require "crabfarm/modes/publisher"
+        Crabfarm::Modes::Publisher.publish CF_PATH, options
       end
     end
 

data/lib/crabfarm/configuration.rb

@@ -9,6 +9,7 @@ module Crabfarm
       [:output_builder, :string, 'Default json output builder used by states'],
       [:driver_factory, :mixed, 'Driver factory, disabled if phantom_mode is used'],
       [:log_path, :string, 'Path where logs should be stored'],
+      [:proxy, :string, 'If given, a proxy is used to connect to the internet if driver supports it'],
 
       # Default driver configuration parameters
       [:driver, ['chrome', 'firefox', 'phantomjs', 'remote'], 'Webdriver to be user, common options: chrome, firefox, phantomjs, remote.'],
@@ -21,10 +22,14 @@ module Crabfarm
 
       # Phantom launcher configuration
       [:phantom_load_images, :boolean, 'Phantomjs image loading, only for phantomjs driver.'],
-      [:phantom_proxy, :string, 'Phantonjs proxy address, only for phantomjs driver.'],
       [:phantom_ssl, ['sslv3', 'sslv2', 'tlsv1', 'any'], 'Phantomjs ssl mode: sslv3, sslv2, tlsv1 or any, only for phantomjs driver.'],
       [:phantom_bin_path, :string, 'Phantomjs binary path, only for phantomjs driver.'],
-      [:phantom_lock_file, :string, 'Phantomjs lock file path, only for phantomjs driver.']
+      [:phantom_lock_file, :string, 'Phantomjs lock file path, only for phantomjs driver.'],
+
+      # Crabtrap launcher configuration
+      [:crabtrap_bin, :string, 'Crabtrap binary path.'],
+      [:crabtrap_port, :integer, 'Crabtrap port, defaults to 4000.'],
+      [:crabtrap_mode, ['capture', 'replay'], 'Crabtrap operation mode.']
     ]
     .map { |o| Option.new *o }
 
@@ -48,6 +53,7 @@ module Crabfarm
         output_builder: :hash,
         driver_factory: nil,
         log_path: 'logs',
+        proxy: nil,
 
         driver: 'phantomjs',
         driver_capabilities: Selenium::WebDriver::Remote::Capabilities.firefox,
@@ -58,10 +64,12 @@ module Crabfarm
         driver_window_height: 800,
 
         phantom_load_images: false,
-        phantom_proxy: nil,
         phantom_ssl: 'any',
         phantom_bin_path: 'phantomjs',
-        phantom_lock_file: nil
+        phantom_lock_file: nil,
+
+        crabtrap_bin: 'crabtrap',
+        crabtrap_port: 4000
       }
     end
 
@@ -79,6 +87,7 @@ module Crabfarm
     def driver_config
       {
         name: driver,
+        proxy: proxy,
         capabilities: driver_capabilities,
         remote_host: driver_remote_host,
         remote_timeout: driver_remote_timeout,
@@ -94,7 +103,7 @@ module Crabfarm
     def phantom_config
       {
         load_images: phantom_load_images,
-        proxy: phantom_proxy,
+        proxy: proxy,
         ssl: phantom_ssl,
         bin_path: phantom_bin_path,
         lock_file: phantom_lock_file,
@@ -102,9 +111,17 @@ module Crabfarm
       }
     end
 
-     # Add enviroment support (like a Gemfile)
-     # group :test { set_driver :phantom }
-     # set_driver :phantom, group: :test
+    def crabtrap_config
+      {
+        bin_path: crabtrap_bin,
+        port: crabtrap_port,
+        proxy: proxy
+      }
+    end
+
+    # Add enviroment support (like a Gemfile)
+    # group :test { set_driver :phantom }
+    # set_driver :phantom, group: :test
 
   end
 

data/lib/crabfarm/context.rb

@@ -7,23 +7,67 @@ module Crabfarm
     def_delegators :@pool, :driver
 
     def initialize
-      @pool = DriverBucketPool.new
       @store = StateStore.new
+      @loaded = false
+    end
+
+    def load
+      unless @loaded
+        init_phantom_if_required
+        @pool = DriverBucketPool.new build_driver_factory
+        @loaded = true
+      end
     end
 
     def run_state(_name, _params={})
+      load
       state = LoaderService.load_state(_name).new @pool, @store, _params
       state.crawl
       state
     end
 
     def reset
+      load
       @store.reset
       @pool.reset
     end
 
     def release
-      @pool.release
+      if @loaded
+        @pool.release
+        @phantom.stop unless @phantom.nil?
+        @loaded = false
+      end
+    end
+
+  private
+
+    def init_phantom_if_required
+      if config.phantom_mode_enabled?
+        @phantom = PhantomRunner.new phantom_config
+        @phantom.start
+      end
+    end
+
+    def build_driver_factory
+      if @phantom
+        PhantomDriverFactory.new @phantom, driver_config
+      else
+        return config.driver_factory if config.driver_factory
+        DefaultDriverFactory.new driver_config
+      end
+    end
+
+    def config
+      Crabfarm.config
+    end
+
+    def driver_config
+      config.driver_config
+    end
+
+    def phantom_config
+      config.phantom_config
     end
 
   end

data/lib/crabfarm/crabtrap_context.rb

@@ -0,0 +1,54 @@
+require 'active_support'
+require 'crabfarm/crabtrap_runner'
+
+module Crabfarm
+  class CrabtrapContext < Context
+
+    def load
+      pass_through if @runner.nil?
+      super
+    end
+
+    def pass_through
+      restart_with_options(mode: :pass) if @runner.nil? or @runner.mode != :pass
+    end
+
+    def capture(_path)
+      restart_with_options(mode: :capture, bucket_path: _path)
+    end
+
+    def replay(_path)
+      restart_with_options(mode: :replay, bucket_path: _path)
+    end
+
+    def release
+      super
+      stop_daemon
+    end
+
+  private
+
+    def restart_with_options(_options)
+      stop_daemon
+      @runner = CrabtrapRunner.new Crabfarm.config.crabtrap_config.merge(_options)
+      @runner.start
+    end
+
+    def stop_daemon
+      @runner.stop unless @runner.nil?
+    end
+
+    def driver_config
+      super.merge(proxy: proxy_address)
+    end
+
+    def phantom_config
+      super.merge(proxy: proxy_address)
+    end
+
+    def proxy_address
+      "127.0.0.1:#{@runner.port}"
+    end
+
+  end
+end

data/lib/crabfarm/crabtrap_runner.rb

@@ -0,0 +1,54 @@
+require 'net/http'
+
+module Crabfarm
+  class CrabtrapRunner
+
+    def initialize(_config={})
+      @config = _config;
+      @pid = nil
+    end
+
+    def port
+      @config[:port] # TODO: maybe select port dynamically...
+    end
+
+    def mode
+      @config.fetch(:mode, :pass).to_sym
+    end
+
+    def start
+      @pid = Process.spawn({}, crabtrap_cmd)
+      wait_for_server
+    end
+
+    def stop
+      unless @pid.nil?
+        Process.kill("INT", @pid)
+        Process.wait @pid
+        @pid = nil
+      end
+    end
+
+  private
+
+    def crabtrap_cmd
+      cmd = [@config[:bin_path]]
+      cmd << mode.to_s
+      cmd << @config[:bucket_path] if mode != :pass
+      cmd << "--port=#{port}"
+      cmd.join(' ')
+    end
+
+    def wait_for_server
+      loop do
+        begin
+          # TODO: improve waiting, making this kind of request could change crabtrap's stack
+          Net::HTTP.get_response(URI.parse("http://127.0.0.1:#{port}/status"))
+          break
+        rescue
+        end
+      end
+    end
+
+  end
+end

data/lib/crabfarm/default_driver_factory.rb

@@ -7,33 +7,79 @@ module Crabfarm
 
     def build_driver(_session_id)
 
-      driver_name = @config[:name]
-      raise ConfigurationError.new 'must provide a webdriver type' if driver_name.nil?
+      raise ConfigurationError.new 'must provide a webdriver type' unless config_present? :name
+      driver_name = @config[:name].to_sym
 
-      case driver_name
+      driver = case driver_name
       when :noop
         require "crabfarm/mocks/noop_driver"
         driver = Crabfarm::Mocks::NoopDriver.new # TODO: improve dummy driver...
       when :remote
-        # setup a custom client to use longer timeouts
-        client = Selenium::WebDriver::Remote::Http::Default.new
-        client.timeout = @config[:remote_timeout]
+        load_remote_driver
+      when :firefox
+        load_firefox_driver
+      when :chrome
+        load_chrome_driver
+      else
+        load_other_driver driver_name
+      end
 
-        driver = Selenium::WebDriver.for :remote, {
-          :url => @config[:remote_host],
-          :http_client => client,
-          :desired_capabilities => @config[:capabilities]
-        }
+      # apply browser configuration to new driver
+      driver.manage.window.resize_to(@config[:window_width], @config[:window_height]) rescue nil
 
-        driver.send(:bridge).setWindowSize(@config[:window_width], @config[:window_height])
-      else
-        driver = Selenium::WebDriver.for driver_name.to_sym
+      return driver
+    end
+
+    def load_remote_driver
+      client = Selenium::WebDriver::Remote::Http::Default.new
+      client.timeout = @config[:remote_timeout]
 
-        # apply browser configuration to new driver
-        driver.manage.window.resize_to(@config[:window_width], @config[:window_height]) rescue nil
+      if config_present? :proxy
+        client.proxy = Selenium::WebDriver::Proxy.new({
+          :http => @config[:proxy],
+          :ssl => @config[:proxy]
+        })
       end
 
-      return driver
+      Selenium::WebDriver.for(:remote, {
+        :url => @config[:remote_host],
+        :http_client => client,
+        :desired_capabilities => @config[:capabilities]
+      })
+    end
+
+    def load_firefox_driver
+      profile = Selenium::WebDriver::Firefox::Profile.new
+
+      if config_present? :proxy
+        profile.proxy = Selenium::WebDriver::Proxy.new({
+          :http => @config[:proxy],
+          :ssl => @config[:proxy]
+        })
+      end
+
+      Selenium::WebDriver.for :firefox, :profile => profile
+    end
+
+    def load_chrome_driver
+      switches = []
+
+      if config_present? :proxy
+        switches << "--proxy-server=#{@config[:proxy]}"
+        switches << "--ignore-certificate-errors"
+      end
+
+      Selenium::WebDriver.for :chrome, :switches => switches
+    end
+
+    def load_other_driver(_name)
+      raise ConfigurationError.new 'default driver does not support proxy' if config_present? :proxy
+
+      Selenium::WebDriver.for _name.to_sym
+    end
+
+    def config_present?(_key)
+      not (@config[_key].nil? or @config[_key].empty?)
     end
 
   end

data/lib/crabfarm/driver_bucket_pool.rb

@@ -1,17 +1,15 @@
 module Crabfarm
   class DriverBucketPool
 
-    def initialize
+    def initialize(_factory=nil)
+      @factory = _factory || DefaultDriverFactory.new(Crabfarm.config.driver_config)
       @buckets = Hash.new
-      @phantom = nil
-
-      init_phantom_if_required
     end
 
     def driver(_session_id=nil)
       _session_id ||= :default_driver
       bucket = @buckets[_session_id.to_sym]
-      bucket = @buckets[_session_id.to_sym] = DriverBucket.new(_session_id, build_driver_factory) if bucket.nil?
+      bucket = @buckets[_session_id.to_sym] = DriverBucket.new(_session_id, @factory) if bucket.nil?
       bucket
     end
 
@@ -22,29 +20,6 @@ module Crabfarm
 
     def release
       reset
-      @phantom.stop unless @phantom.nil?
-    end
-
-  private
-
-    def init_phantom_if_required
-      if config.phantom_mode_enabled?
-        @phantom = PhantomRunner.new config.phantom_config
-        @phantom.start
-      end
-    end
-
-    def build_driver_factory
-      if config.phantom_mode_enabled?
-        PhantomDriverFactory.new @phantom, config.driver_config
-      else
-        return config.driver_factory if config.driver_factory
-        DefaultDriverFactory.new config.driver_config
-      end
-    end
-
-    def config
-      Crabfarm.config
     end
 
   end

data/lib/crabfarm/modes/console.rb

@@ -10,20 +10,19 @@ module Crabfarm
 
       class ConsoleDsl
 
-        attr_reader :context
-
-        def initialize
-          reload!
+        def initialize(_context)
+          @context = _context
         end
 
         def reload!
-          unless @context.nil?
-            puts "Reloading crawler source".color(:green)
-            @context.release
-            ActiveSupport::Dependencies.clear
-          end
+          puts "Reloading crawler source".color(:green)
+          ActiveSupport::Dependencies.clear
+          @context.reset
+        end
 
-          @context = Crabfarm::Context.new
+        def reset
+          puts "Resetting crawling context".color(:green)
+          @context.reset
         end
 
         def transition(_name=nil, _params={})
@@ -53,17 +52,12 @@ module Crabfarm
           puts "Ejem..."
         end
 
-        def reset
-          puts "Resetting crawling context".color(:green)
-          @context.reset
-        end
-
         alias :t :transition
         alias :r :reset
       end
 
-      def self.start
-        dsl = ConsoleDsl.new
+      def self.start(_context)
+        dsl = ConsoleDsl.new _context
 
         loop do
           begin
@@ -78,7 +72,7 @@ module Crabfarm
         end
 
         puts "Releasing crawling context".color(:green)
-        dsl.context.release
+        _context.release
       end
 
     end

data/lib/crabfarm/modes/generator.rb

@@ -20,6 +20,7 @@ module Crabfarm
           path(_name, 'Gemfile').render('Gemfile', binding)
           path(_name, 'Crabfile').render('Crabfile', binding)
           path(_name, '.rspec').render('dot_rspec', binding)
+          path(_name, '.crabfarm').render('dot_crabfarm', binding)
           path(_name, 'boot.rb').render('boot.rb', binding)
           path(_name, 'bin', 'crabfarm').render('crabfarm_bin', binding, 0755)
           path(_name, 'app', 'parsers', '.gitkeep').render('dot_gitkeep')
@@ -27,6 +28,8 @@ module Crabfarm
           path(_name, 'app', 'helpers', '.gitkeep').render('dot_gitkeep')
           path(_name, 'spec', 'spec_helper.rb').render('spec_helper.rb', binding)
           path(_name, 'spec', 'snapshots', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'spec', 'mementos', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'spec', 'integration', '.gitkeep').render('dot_gitkeep')
         end
       end
 

data/lib/crabfarm/modes/publisher.rb

@@ -0,0 +1,189 @@
+require 'yaml'
+require 'git'
+require 'zlib'
+require 'rubygems/package'
+require 'net/http/post/multipart'
+require 'rainbow'
+require 'rainbow/ext/string'
+require 'digest/sha1'
+
+module Crabfarm
+  module Modes
+    module Publisher
+      extend self
+
+      DEFAULT_HOST = 'http://www.crabfarm.io'
+
+      def publish(_path, _options={})
+
+        @crawler_path = _path
+        @options = _options
+
+        load_config
+        return unless dry_run or authenticated?
+        detect_git_repo
+
+        if inside_git_repo?
+          if not unsafe and is_tree_dirty?
+            puts "Aborting: Your working copy has uncommited changes! Use the --unsafe option to force.".color(:red)
+            return
+          end
+          load_files_from_git
+        else
+          load_files_from_fs
+        end
+
+        build_package
+        compress_package
+        generate_signature
+
+        send_package unless dry_run
+      end
+
+    private
+
+      def dry_run
+        @options.fetch(:dry, false)
+      end
+
+      def unsafe
+        @options.fetch(:unsafe, false)
+      end
+
+      def config_path
+        File.join(@crawler_path, '.crabfarm')
+      end
+
+      def home_config_path
+        File.join(Dir.home, '.crabfarm')
+      end
+
+      def load_config
+        config = YAML.load_file config_path
+
+        if File.exists? home_config_path
+          home_config = YAML.load_file home_config_path
+          config = home_config.merge config
+        end
+
+        @token = config['token']
+        @name = config['name']
+        @host = config['host'] || DEFAULT_HOST
+        @include = config['files']
+      end
+
+      def authenticated?
+        # TODO: if no token, ask for credentials and fetch token
+        if @token.nil? or @token.empty?
+          puts "No crabfarm API token has been provided".color(:red)
+          return false
+        end
+
+        true
+      end
+
+      def is_tree_dirty?
+        @git.chdir do
+          status = @git.status
+          (status.changed.count + status.added.count + status.deleted.count + status.untracked.count) > 0
+        end
+      end
+
+      def detect_git_repo
+        git_path = @crawler_path
+
+        path_to_git = []
+        while git_path != '/'
+          if File.exists? File.join(git_path, '.git')
+            @git = Git.open git_path
+            @rel_path = if path_to_git.count > 0 then File.join(*path_to_git.reverse!) else nil end
+            return
+          else
+            path_to_git << File.basename(git_path)
+            git_path = File.expand_path('..', git_path)
+          end
+        end
+
+        @git = nil
+      end
+
+      def inside_git_repo?
+        not @git.nil?
+      end
+
+      def load_files_from_git
+        @git.chdir do
+          @ref = @git.log.first.sha
+          puts "Packaging files from current HEAD (#{@ref}):".color(:green)
+          entries = @git.gtree(@ref).full_tree.map(&:split)
+          entries = entries.select { |e| e[1] == 'blob' }
+
+          @file_list = []
+          entries.each do |entry|
+            path = unless @rel_path.nil?
+              next unless entry[3].starts_with? @rel_path
+              entry[3][@rel_path.length+1..-1]
+            else entry[3] end
+
+            if @include.any? { |p| File.fnmatch? p, path }
+              @file_list << [path, entry[0].to_i(8), @git.show(@ref, entry[3])]
+            end
+          end
+        end
+      end
+
+      def load_files_from_fs
+        puts "Packaging files (no version control)".color(:green)
+        @file_list = Dir[*@include].map do |path|
+          full_path = File.join(@crawler_path, path)
+          [path, File.stat(full_path).mode, File.read(full_path)]
+        end
+        @ref = "filesystem"
+      end
+
+      def build_package
+        @package = StringIO.new("")
+        Gem::Package::TarWriter.new(@package) do |tar|
+          @file_list.each do |f|
+            puts "+ #{f[0]} - #{f[1]}"
+            path, mode, contents = f
+            tar.add_file(path, mode) { |tf| tf.write contents }
+          end
+        end
+
+        @package.rewind
+      end
+
+      def compress_package
+        @cpackage = StringIO.new("")
+        writer = Zlib::GzipWriter.new(@cpackage)
+        writer.write @package.string
+        writer.close
+      end
+
+      def generate_signature
+        @signature = Digest::SHA1.hexdigest @package.string
+        puts "Package SHA1: #{@signature}"
+      end
+
+      def send_package
+        url = URI.join(@host, 'api/crawlers/', @name)
+
+        req = Net::HTTP::Put::Multipart.new(url.path, {
+          "repo" => UploadIO.new(StringIO.new(@cpackage.string), "application/x-gzip", "tree.tar.gz"),
+          "sha" => @signature,
+          "ref" => @ref
+        }, {
+          'X-Api-Token' => @token
+        })
+
+        res = Net::HTTP.start(url.host, url.port) do |http|
+          http.request(req)
+        end
+
+        puts res.body
+      end
+
+    end
+  end
+end

data/lib/crabfarm/modes/recorder.rb

@@ -0,0 +1,42 @@
+require 'rainbow'
+require 'rainbow/ext/string'
+require 'crabfarm/crabtrap_runner'
+
+module Crabfarm
+  module Modes
+    class Recorder
+
+      def self.start(_target)
+        return puts "Must provide a recording name" unless _target.is_a? String
+
+        crabtrap_config = Crabfarm.config.crabtrap_config
+        crabtrap_config[:mode] = :capture
+        crabtrap_config[:bucket_path] = File.join(CF_PATH, 'spec/mementos', _target + '.json.gz')
+
+        crabtrap = CrabtrapRunner.new crabtrap_config
+        crabtrap.start
+
+        driver_config = Crabfarm.config.driver_config
+        driver_config[:name] = :firefox
+        driver_config[:proxy] = "127.0.0.1:#{crabtrap.port}"
+
+        driver = DefaultDriverFactory.new(driver_config).build_driver nil
+
+        begin
+          puts "Press Ctrl-C to stop capturing."
+          loop do
+            driver.current_url
+            sleep 1.0
+          end
+        rescue Selenium::WebDriver::Error::WebDriverError, SystemExit, Interrupt
+          # noop
+        end
+
+        puts "Releasing crawling context".color(:green)
+        driver.quit rescue nil
+        crabtrap.stop
+      end
+
+    end
+  end
+end

data/lib/crabfarm/rspec.rb

@@ -1,4 +1,7 @@
-CF_TEST_CONTEXT = Crabfarm::Context::new
+require 'crabfarm/crabtrap_context'
+
+CF_TEST_CONTEXT = Crabfarm::CrabtrapContext::new
+CF_TEST_CONTEXT.load
 CF_TEST_BUCKET = CF_TEST_CONTEXT.driver
 
 module Crabfarm
@@ -15,6 +18,28 @@ module Crabfarm
       CF_TEST_BUCKET.parse(described_class, _options)
     end
 
+    def crawl(_state=nil, _params={})
+      if _state.is_a? Hash
+        _params = _state
+        _state = nil
+      end
+
+      if _state.nil?
+        return nil unless described_class < BaseState # TODO: maybe raise an error here.
+        @state = @last_state = CF_TEST_CONTEXT.run_state(described_class, _params)
+      else
+        @last_state = CF_TEST_CONTEXT.run_state(_state, _params)
+      end
+    end
+
+    def state
+      @state ||= crawl
+    end
+
+    def last_state
+      @last_state
+    end
+
     def parser
       @parser
     end
@@ -26,9 +51,16 @@ RSpec.configure do |config|
   config.include Crabfarm::RSpec
 
   config.before(:example) do |example|
+
     if example.metadata[:parsing]
       @parser = parse example.metadata[:parsing], example.metadata[:using] || {}
     end
+
+    if example.metadata[:crawling]
+      CF_TEST_CONTEXT.replay File.join(CF_PATH, 'spec/mementos', example.metadata[:crawling] + '.json.gz')
+    else
+      CF_TEST_CONTEXT.pass_through
+    end
   end
 
   config.after(:suite) do

data/lib/crabfarm/templates/dot_crabfarm.erb

@@ -0,0 +1,9 @@
+host: 'http://www.crabfarm.io'
+name: '<%= name %>'
+files:
+- Crabfile
+- Gemfile
+- Gemfile.lock
+- boot.rb
+- app/**/*.*
+- bin/**/*.*

data/lib/crabfarm/templates/dot_gitignore.erb

@@ -1,2 +1 @@
-Gemfile.lock
 logs/*.*

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.0.9"
+  VERSION = "0.0.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-12 00:00:00.000000000 Z
+date: 2015-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jbuilder
@@ -142,6 +142,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 2.10.2
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multipart-post
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -287,6 +315,7 @@ email:
 - ignacio@platan.us
 executables:
 - crabfarm
+- crabtrap
 extensions: []
 extra_rdoc_files: []
 files:
@@ -301,6 +330,8 @@ files:
 - lib/crabfarm/cli.rb
 - lib/crabfarm/configuration.rb
 - lib/crabfarm/context.rb
+- lib/crabfarm/crabtrap_context.rb
+- lib/crabfarm/crabtrap_runner.rb
 - lib/crabfarm/default_driver_factory.rb
 - lib/crabfarm/driver_bucket.rb
 - lib/crabfarm/driver_bucket_pool.rb
@@ -314,6 +345,8 @@ files:
 - lib/crabfarm/mocks/noop_driver.rb
 - lib/crabfarm/modes/console.rb
 - lib/crabfarm/modes/generator.rb
+- lib/crabfarm/modes/publisher.rb
+- lib/crabfarm/modes/recorder.rb
 - lib/crabfarm/modes/server.rb
 - lib/crabfarm/phantom_driver_factory.rb
 - lib/crabfarm/phantom_runner.rb
@@ -325,6 +358,7 @@ files:
 - lib/crabfarm/templates/boot.rb.erb
 - lib/crabfarm/templates/crabfarm_bin.erb
 - lib/crabfarm/templates/Crabfile.erb
+- lib/crabfarm/templates/dot_crabfarm.erb
 - lib/crabfarm/templates/dot_gitignore.erb
 - lib/crabfarm/templates/dot_gitkeep.erb
 - lib/crabfarm/templates/dot_rspec.erb
@@ -337,6 +371,7 @@ files:
 - lib/crabfarm/version.rb
 - lib/crabfarm.rb
 - bin/crabfarm
+- bin/crabtrap
 homepage: https://github.com/platanus/crabfarm-gem
 licenses:
 - MIT