crabfarm 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 45541f82668f0910accb44339916af9e37595470
-  data.tar.gz: 99de58252a283fd334a5c1aafc491eec16610c21
+  metadata.gz: 9ffaef8409650267bf6e4272421008ff9c38d05e
+  data.tar.gz: 996ba45929699ec7eebac5d8d4a1a24f231cf4a1
 SHA512:
-  metadata.gz: 826f03dc142d062f35c2e7a3ff0cf7c56c16127704c95f81d39a14a0b702ea57c4da07250471ad0b9f55b3825ef51312317b7fc8c0aced47e1fb3f395e87c5d4
-  data.tar.gz: 63863c7f0ede26385cdc1993a97433cfb16911edd593d30488ff5569366aaa38654084f22391c4ce6dad47105b8b691947c37af4ab7ecd9146ee24c1fbc63066
+  metadata.gz: 4d5cffdf273b3f31c807502f036583ab89e9ad58b66aa2f13a79112350f418a246535702109bcdabf8cbded64410414b8fa39a581af079f99d205b28c1387cc9
+  data.tar.gz: 6dd417dbebd417b32fa738c58af2927651d6106f66909b4748328dcb9965976c3f206f9d792362740793ffd1b8ae0dc18bb07dc6ea5fe29f8581b53c60728fa6

data/lib/crabfarm/configuration.rb

@@ -27,7 +27,7 @@ module Crabfarm
       [:phantom_lock_file, :string, 'Phantomjs lock file path, only for phantomjs driver.'],
 
       # Crabtrap launcher configuration
-      [:crabtrap_bin, :string, 'Crabtrap binary path.'],
+      [:crabtrap_bin_path, :string, 'Crabtrap binary path.'],
       [:crabtrap_port, :integer, 'Crabtrap port, defaults to 4000.'],
       [:crabtrap_mode, ['capture', 'replay'], 'Crabtrap operation mode.']
     ]
@@ -68,7 +68,7 @@ module Crabfarm
         phantom_bin_path: 'phantomjs',
         phantom_lock_file: nil,
 
-        crabtrap_bin: 'crabtrap',
+        crabtrap_bin_path: 'crabtrap',
         crabtrap_port: 4000
       }
     end
@@ -113,7 +113,7 @@ module Crabfarm
 
     def crabtrap_config
       {
-        bin_path: crabtrap_bin,
+        bin_path: crabtrap_bin_path,
         port: crabtrap_port,
         proxy: proxy
       }

data/lib/crabfarm/crabtrap_context.rb

@@ -39,11 +39,11 @@ module Crabfarm
     end
 
     def driver_config
-      super.merge(proxy: proxy_address)
+      if @runner.is_running? then super.merge(proxy: proxy_address) else super end
     end
 
     def phantom_config
-      super.merge(proxy: proxy_address)
+      if @runner.is_running? then super.merge(proxy: proxy_address) else super end
     end
 
     def proxy_address

data/lib/crabfarm/crabtrap_runner.rb

@@ -8,6 +8,10 @@ module Crabfarm
       @pid = nil
     end
 
+    def is_running?
+      not @pid.nil?
+    end
+
     def port
       @config[:port] # TODO: maybe select port dynamically...
     end
@@ -17,8 +21,13 @@ module Crabfarm
     end
 
     def start
-      @pid = Process.spawn({}, crabtrap_cmd)
-      wait_for_server
+      begin
+        @pid = Process.spawn({}, crabtrap_cmd)
+        wait_for_server
+      rescue
+        puts "Could not find crabtrap at #{@config[:bin_path]}, memento replaying is disabled!"
+        @pid = nil
+      end
     end
 
     def stop

data/lib/crabfarm/engines/safe_state_loop.rb

@@ -6,7 +6,6 @@ module Crabfarm
     class SafeStateLoop
 
       def initialize
-        @context = Crabfarm::Context.new
         @running = true
         @working = false
         @lock = Mutex.new
@@ -16,7 +15,6 @@ module Crabfarm
       def release
         @running = false
         @thread.join
-        @context.release
       end
 
       def change_state(_name, _params={}, _wait=nil)
@@ -90,29 +88,38 @@ module Crabfarm
       end
 
       def crawl_loop
-        while @running
-          if @working
-            @elapsed = Benchmark.measure do
-              begin
-                ActiveSupport::Dependencies.clear
-                logger.info "StateLoop: loading state: #{@next_state_name}"
-                @doc = @context.run_state(@next_state_name, @next_state_params).output_as_json
-                logger.info "StateLoop: state loaded successfully: #{@next_state_name}"
-                @error = nil
-              rescue Exception => e
-                logger.error "StateLoop: error while loading state: #{@next_state_name}"
-                logger.error e
-                @doc = nil
-                @error = e
-              end
-            end.real
-
-            @lock.synchronize {
-              @state_name = @next_state_name
-              @state_params = @next_state_params
-              @working = false
-            }
-          else sleep 0.2 end
+        context = Crabfarm::Context.new
+
+        begin
+          while @running
+            if @working
+              @elapsed = Benchmark.measure do
+                begin
+                  ActiveSupport::Dependencies.clear
+                  logger.info "StateLoop: loading state: #{@next_state_name}"
+                  @doc = context.run_state(@next_state_name, @next_state_params).output_as_json
+                  logger.info "StateLoop: state loaded successfully: #{@next_state_name}"
+                  @error = nil
+                rescue Exception => e
+                  logger.error "StateLoop: error while loading state: #{@next_state_name}"
+                  logger.error e
+                  @doc = nil
+                  @error = e
+                end
+              end.real
+
+              @lock.synchronize {
+                @state_name = @next_state_name
+                @state_params = @next_state_params
+                @working = false
+              }
+            else sleep 0.2 end
+          end
+        rescue Exception => e
+          logger.fatal "StateLoop: unhandled exception!"
+          logger.fatal e
+        ensure
+          context.release
         end
       end
 

data/lib/crabfarm/modes/generator.rb

@@ -30,6 +30,7 @@ module Crabfarm
           path(_name, 'spec', 'snapshots', '.gitkeep').render('dot_gitkeep')
           path(_name, 'spec', 'mementos', '.gitkeep').render('dot_gitkeep')
           path(_name, 'spec', 'integration', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'logs').render('dot_gitkeep')
         end
       end
 

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-25 00:00:00.000000000 Z
+date: 2015-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jbuilder
@@ -315,7 +315,6 @@ email:
 - ignacio@platan.us
 executables:
 - crabfarm
-- crabtrap
 extensions: []
 extra_rdoc_files: []
 files:
@@ -371,7 +370,6 @@ files:
 - lib/crabfarm/version.rb
 - lib/crabfarm.rb
 - bin/crabfarm
-- bin/crabtrap
 homepage: https://github.com/platanus/crabfarm-gem
 licenses:
 - MIT

data/bin/crabtrap

@@ -1,347 +0,0 @@
-#!/usr/bin/env node
-
-var net = require('net'),
-	http = require('http'),
-	https = require('https'),
-	url = require('url'),
-	fs = require('fs'),
-	zlib = require('zlib');
-
-// Globals
-
-var HTTPS_OPTIONS = {
-	key: '-----BEGIN RSA PRIVATE KEY-----\nMIIBOQIBAAJBAK/L/lXb/kxUzve1olo71s6mQLvuQCm3z2wqClq71NLerFnaXpN+\nFrNPy7+R3gZ1hdWXqbN5NqpWDMM9fcbd7p0CAwEAAQJAUDImN3Lhgl7Z/+TLSJCt\nwJ3VQCZC/QUOSdCv4o53Wy5aL/n8ootYFC3eoFC2Nal5bnH6onP9YR+X9l3HKLaT\n3QIhANXwb5SvJ+Kewa8F5wNHo9LFjSbL7WSSb1MyvYnOeFlPAiEA0lvaLz6UXRDL\n6T6Z1fkF0exmQqVimeL5qjY5o9Gk5lMCH1A52Z3oEQzqe7cmf3q7YrOnYUcrMdqF\nDzojzO/gfUECIQCe9fImiW+r9CljFH9Dhm6zd6S+8CNWjoKD8X4VITMvKQIgb3sg\nq9gPVzXn/+f8Qcc2KILSh3ffkIpA8yJK9omUIxI=\n-----END RSA PRIVATE KEY-----\n',
-	cert: '-----BEGIN CERTIFICATE-----\nMIIBmDCCAUICCQDGtiGKgI9AXjANBgkqhkiG9w0BAQUFADBTMQswCQYDVQQGEwJD\nTDELMAkGA1UECBMCUk0xETAPBgNVBAcTCFNhbnRpYWdvMREwDwYDVQQKEwhQbGF0\nYW51czERMA8GA1UEAxMIQ3JhYnRyYXAwHhcNMTUwMTE1MjAxNzMzWhcNNDIwNjAx\nMjAxNzMzWjBTMQswCQYDVQQGEwJDTDELMAkGA1UECBMCUk0xETAPBgNVBAcTCFNh\nbnRpYWdvMREwDwYDVQQKEwhQbGF0YW51czERMA8GA1UEAxMIQ3JhYnRyYXAwXDAN\nBgkqhkiG9w0BAQEFAANLADBIAkEAr8v+Vdv+TFTO97WiWjvWzqZAu+5AKbfPbCoK\nWrvU0t6sWdpek34Ws0/Lv5HeBnWF1Zeps3k2qlYMwz19xt3unQIDAQABMA0GCSqG\nSIb3DQEBBQUAA0EAmecqIZqQ8OXSIj0V2VKaIXwz8RBnhLzU7BJwcsWJE/Bex7zB\nWP+vLv9ML5ZRLCsXjL5IOav8qAX/NZXjoN3e3Q==\n-----END CERTIFICATE-----\n'
-};
-
-var LOG = {
-	DEBUG: 0,
-	INFO: 1,
-	WARN: 2,
-	ERROR: 3
-};
-
-var STACK = [],
-	MODE = false,
-	SOURCE = null,
-	PORT = 4000,
-	LOG_LEVEL = LOG.WARN;
-
-(function() {
-	if(process.argv.length < 2) throw 'Must provide a proxy mode';
-	MODE = process.argv[2];
-	var i = 3;
-
-	if(MODE != 'pass') {
-		if(process.argv.length < 3) throw 'Must provide a bucket path';
-		SOURCE = process.argv[3];
-		i = 4;
-	}
-
-	for(; i < process.argv.length; i++) {
-		var parts = process.argv[i].split('=');
-		switch(parts[0]) {
-			case '--port': PORT = parseInt(parts[1], 10); break;
-			case '--quiet': PORT = parseInt(parts[1], 10); break;
-			default: throw 'Invalid option ' + parts[0];
-		}
-	}
-})();
-
-// Utility methods
-
-function log(_level, _message) {
-	if(_level == LOG.DEBUG) _message = '\t' + _message;
-	if(_level >= LOG_LEVEL) console.log(_message);
-}
-
-function forOwn(_obj, _cb) {
-	for(var key in _obj) {
-		if(_obj.hasOwnProperty(key)) {
-			_cb(key, _obj[key]);
-		}
-	}
-}
-
-function keysToLowerCase(_obj) {
-	var result = {};
-	forOwn(_obj, function(k,v) { result[k.toLowerCase()] = v; });
-	return result;
-}
-
-function pickRandomPort() {
-	return 0; // This could fail on Linux...
-}
-
-function matchRequestToResource(_req, _resource) {
-	return _resource.method.toLowerCase() == _req.method.toLowerCase() && _resource.url == _req.url;
-}
-
-function matchRequestToResourceWOQuery(_req, _resource) {
-	if(_resource.method.toLowerCase() == _req.method.toLowerCase()) return false;
-
-	var reqUrl = url.parse(_req.url, true),
-		resUrl = url.parse(_resource.url, true);
-
-	return reqUrl.hostname == resUrl.hostname && reqUrl.pathname == resUrl.pathname;
-}
-
-function findAndMoveLast(_req, _array, _matches) {
-	for(var i = 0, l = _array.length; i < l; i++) {
-		if(_matches(_req, _array[i])) {
-			var resource = _array.splice(i, 1)[0];
-			_array.push(resource);
-			return resource;
-		}
-	}
-
-	return null;
-}
-
-function loadStackFrom(_path, _then) {
-	var data = fs.readFileSync(_path);
-	zlib.gunzip(data, function(err, buffer) {
-		if (!err) STACK = JSON.parse(buffer.toString());
-		_then();
-	});
-}
-
-function saveStackTo(_path, _then) {
-	var data = JSON.stringify(STACK);
-	zlib.gzip(data, function(err, buffer) {
-		if (!err) fs.writeFileSync(_path, buffer);
-		_then();
-	});
-}
-
-function resolveAndServeResource(_req, _resp) {
-	var resource = findInStack(_req);
-	if(resource) {
-		log(LOG.INFO, "Serving: " + resource.method + ' ' + resource.url);
-		log(LOG.DEBUG, "HTTP " + resource.statusCode);
-		log(LOG.DEBUG, JSON.stringify(resource.headers));
-
-		serveResource(resource, _resp);
-	} else {
-		log(LOG.WARN, 'Not found: ' + _req.url);
-		_resp.statusCode = 404;
-		_resp.end();
-	}
-}
-
-function serveLastResource(_resp) {
-	serveResource(STACK[STACK.length-1], _resp);
-}
-
-function serveResource(_resource, _resp) {
-	_resp.statusCode = _resource.statusCode;
-
-	forOwn(_resource.headers, function(k, v) { _resp.setHeader(k, v); });
-
-	if(_resource.content) {
-		var buf = new Buffer(_resource.content, _resource.encoding);
-		_resp.end(buf);
-	} else {
-		_resp.end();
-	}
-}
-
-function findAndMoveLast(_req, _matches) {
-	for(var i = 0, l = STACK.length; i < l; i++) {
-		if(_matches(_req, STACK[i])) {
-			var resource = STACK.splice(i, 1)[0];
-			STACK.push(resource);
-			return resource;
-		}
-	}
-
-	return null;
-}
-
-function findInStack(_req, _partial) {
-	return findAndMoveLast(_req, matchRequestToResource) ||
-		findAndMoveLast(_req, matchRequestToResourceWOQuery);
-}
-
-function cacheResponse(_req, _resp, _cb) {
-
-	log(LOG.INFO, "Caching Response");
-	log(LOG.DEBUG, "HTTP " + _resp.statusCode);
-	log(LOG.DEBUG, JSON.stringify(keysToLowerCase(_resp.headers)));
-
-	var encoding = null,
-		// TODO: consider storing port and protocoll in the resource.
-		resource = {
-			url: _req.url,
-			statusCode: _resp.statusCode,
-			method: _req.method,
-			// inHeaders: req.headers, // store request headers to aid in recognition?
-			headers: keysToLowerCase(_resp.headers),
-			content: '',
-			encoding: 'base64'
-		},
-		contentEncoding = resource.headers['content-encoding'],
-		contentType = resource.headers['content-type'],
-		outStream = _resp;
-
-	// add decompression if supported encoding:
-	if(contentEncoding == 'gzip') {
-		outStream = _resp.pipe(zlib.createGunzip());
-		delete resource.headers['content-encoding'];
-		contentEncoding = null;
-	} else if(contentEncoding == 'deflate') {
-		outStream = _resp.pipe(zlib.createInflate());
-		delete resource.headers['content-encoding'];
-		contentEncoding = null;
-	}
-
-	// use utf8 encoding for uncompresed text:
-	if(!contentEncoding && contentType) {
-		contentType = contentType.match(/([^\/]+)\/([^\s]+)(?:\s+(.+))?/i);
-		if(contentType && (contentType[1] == 'text' || contentType[1] == 'application')) {
-			resource.encoding = 'utf-8';
-		}
-	}
-
-	// remove unwanted headers:
-	delete resource.headers['content-length'];
-
-	// start receiving data:
-	if(resource.encoding) outStream.setEncoding(resource.encoding);
-	outStream.on('data', function(_chunk) {
-		resource.content += _chunk;
-	});
-
-	// when all data is received, store resource (dont know how this will handle more than one request)
-	outStream.on('end', function() {
-		STACK.push(resource);
-		_cb();
-	});
-}
-
-function prepareForwardRequest(_req) {
-	var urlObj = url.parse(_req.url);
-
-	var options = {
-		method: _req.method,
-		host: urlObj.host,
-		path: urlObj.path,
-		rejectUnauthorized: false,
-		headers: keysToLowerCase(_req.headers)
-	};
-
-	// Rewrite headers
-	options.headers['accept-encoding'] = 'gzip,deflate';
-	return options;
-}
-
-function passRequest(_req, _resp) {
-	log(LOG.INFO, 'Passing through ' + _req.method + ' request for ' + _req.url);
-
-	var urlObj = url.parse(_req.url);
-	var forward = (urlObj.protocol == 'https:' ? https : http).request({
-		method: _req.method,
-		host: urlObj.host,
-		path: urlObj.path,
-		headers: _req.headers
-	}, function(_fw_resp) {
-		// pipe response back untouched
-		_resp.writeHead(_fw_resp.statusCode, _fw_resp.headers);
-		_fw_resp.pipe(_resp);
-	});
-
-	_req.pipe(forward);
-}
-
-function captureRequest(_req, _resp, _useSSL) {
-	log(LOG.INFO, 'Forwarding ' + _req.method + ' request for ' + _req.url);
-
-	var urlObj = url.parse(_req.url);
-	var options = {
-		method: _req.method,
-		host: urlObj.host,
-		path: urlObj.path,
-		rejectUnauthorized: false,
-		headers: keysToLowerCase(_req.headers)
-	};
-
-	// Rewrite headers
-	options.headers['accept-encoding'] = 'gzip,deflate';
-	log(LOG.DEBUG, JSON.stringify(options));
-
-	var forward = (urlObj.protocol == 'https:' ? https : http).request(options, function(_fw_resp) {
-		cacheResponse(_req, _fw_resp, function() {
-			serveLastResource(_resp);
-		});
-	});
-
-	_req.pipe(forward); // forward request data
-}
-
-function replayRequest(_req, _resp) {
-	log(LOG.INFO, 'Resolving ' + _req.method + ' request for ' + _req.url);
-	resolveAndServeResource(_req, _resp);
-}
-
-function selectProxy() {
-	switch(MODE) {
-		case 'pass': return passRequest;
-		case 'capture': return captureRequest;
-		case 'replay': return replayRequest;
-		default: throw 'Invalid proxy mode';
-	}
-}
-
-var PROXY_FUN = selectProxy(),
-	SERVER = http.createServer(PROXY_FUN);
-
-// Special handler for HTTPS request, creates a dedicated HTTPS proxy per connection,
-// that way the CONNECT tunnel can be intercepted, requires support for self signed
-// certificates in the client.
-SERVER.on('connect', function (_req, _sock, _head) {
-
-	var urlObj = url.parse('http://' + _req.url);
-	log(LOG.INFO, 'New HTTPS request: starting https intercept on ' + urlObj.hostname);
-
-	var httpsServ = https.createServer(HTTPS_OPTIONS, function(_req, _resp) {
-		_req.url = 'https://' + urlObj.hostname + _req.url;
-		PROXY_FUN(_req, _resp);
-	});
-
-	httpsServ.listen(pickRandomPort());
-
-	var tunnelSock = net.connect(httpsServ.address().port, function() {
-		_sock.write('HTTP/1.1 200 Connection Established\r\n' +
-			'Proxy-agent: Node-Proxy\r\n' +
-			'\r\n');
-		tunnelSock.write(_head);
-		tunnelSock.pipe(_sock);
-		_sock.pipe(tunnelSock);
-	});
-
-	_sock.on('close', function() {
-		httpsServ.close();
-	});
-});
-
-console.log("Starting crabtrap! mode: " + MODE);
-
-if(MODE == 'replay') {
-	loadStackFrom(SOURCE, SERVER.listen.bind(SERVER, PORT));
-} else {
-	SERVER.listen(PORT);
-}
-
-var EXITING = false;
-process.on('SIGINT', function() {
-	if(EXITING) return;
-	EXITING = true;
-
-	console.log("Shutting down crabtrap!");
-	SERVER.close();
-	if(MODE == 'capture') {
-		saveStackTo(SOURCE, process.exit.bind(process));
-	} else {
-		process.exit();
-	}
-});