quandl_sandbox 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 191b24f362d9b5c0144dd44ffcf008ae39d84cad
+  data.tar.gz: 9c33e859a5685e26b2d28e2f0b95d5b5d967b697
+SHA512:
+  metadata.gz: 8b309b71cccc432a123e7025a4a0d5a7a8217d84f1d62ea9cdb41e1f4b23b98e031038722e92e3a245490df825c926dfa293aabb5758a05193539bc772b5d921
+  data.tar.gz: 6d802be7e7936959ef883453ed55a1e868ea020c63197f59d144103a8624d2cd154aa58ffabbe1c94c0608404be835c216ca327b02fa12f27991ad1673eb3078

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2012-2013 Blake Hilscher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# Installation
+
+```ruby
+
+gem 'quandl_sandbox'
+
+```
+
+
+# Configuration
+
+```ruby
+
+require 'quandl/sandbox'
+
+Quandl::Sandbox.config do |c|
+  c.gateway = '10.110.110.10'
+  c.gateway_user = 'ubuntu'
+  c.subnet_id = 'subnet-a1a345b7'
+  c.environment = 'staging'
+  c.webhoook = 'http://example.com/api/v1/execute/callback'
+end
+
+```
+
+
+# Execution
+
+#### build and execute job in sandbox
+
+```ruby
+
+sw = Quandl::Sandbox::Repository.new(
+  job_uid: 'xyz', 
+  repository: 'http://github.com/blakehilscher/goog_scraper/', 
+  reference: 'master' )
+  
+sw.execute
+
+```
+
+j = Job.find_by_uid('xyz').status
+=> 'executing
+
+sleep(10)
+
+Job.find_by_uid('xyz').status
+=> 'retrieved'

data/lib/quandl/sandbox/attributes.rb

@@ -0,0 +1,52 @@
+module Quandl
+module Sandbox
+
+module Attributes
+  
+  extend ActiveSupport::Concern
+
+  included do
+  
+    @attributes = []
+  
+  end
+  
+  module ClassMethods
+  
+    def attributes
+      @attributes
+    end
+    
+    def define_attributes(*names)
+      names.each do |name|
+        name = name.to_sym
+        next if @attributes.include?(name)
+        @attributes << name
+        attr_accessor(name)
+      end
+    end
+    
+  end
+  
+  def initialize(*args)
+    self.attributes = args.extract_options!
+    super if defined?(super)
+  end
+  
+  def attributes
+    self.class.attributes.inject({}){|m,name| m.merge({ name.to_sym => self.send(name) }) }
+  end
+  
+  def attributes=(pairs)
+    # ensure symbolic keys
+    pairs.symbolize_keys!
+    # assign each acceptable pair
+    self.class.attributes.each do |name|
+      self.send("#{name}=", pairs[name] ) if pairs[name]
+    end
+  end
+  
+end
+
+end
+end

data/lib/quandl/sandbox/configuration.rb

@@ -0,0 +1,44 @@
+module Quandl::Sandbox
+  
+  class << self
+    attr_accessor :configuration
+  end
+  
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+  
+  def self.configure
+    yield(configuration)
+    true
+  end
+  
+  class Configuration
+  
+    include Quandl::Sandbox::Attributes
+  
+    define_attributes :image, :instance_type, :availability_zone, :ssh_gateway, 
+      :ssh_user, :subnet_id, :environment, :verbose, :webhoook, :tmp_dir,
+      :aws_access_key_id, :aws_secret_access_key, :key_name, :key_file
+    
+    def initialize
+      @instance_type = 'm1.small'
+      @availability_zone = 'us-east-1c'
+      @ssh_user = 'ubuntu'
+      @environment ||= ( ENV['APP_ENV'] || ENV['RAILS_ENV'] || 'development' )
+      @verbose = ( @environment == 'production' ) ? false : true
+      @key_name = 'quandl_3'
+      @tmp_dir = File.join( Quandl::Sandbox.root, 'tmp/' )
+    end
+    
+    def key_data
+      File.read(key_file)
+    end
+    
+    def to_h
+      self.attributes
+    end
+    
+  end
+  
+end

data/lib/quandl/sandbox/dataset.rb

@@ -0,0 +1,45 @@
+class Quandl::Sandbox::Dataset
+  
+  attr_accessor :source_code, :code, :name, :description, :data, :column_names
+  
+  def self.from_output(outputs)
+    outputs.split("----\n").collect do |output|
+      self.new(output)
+    end
+  end
+  
+  def initialize(output)
+    rows = output.to_s.rstrip.split("\n")
+    self.full_code = rows.shift.strip
+    self.name = rows.shift.strip
+    self.description = rows.shift.strip
+    self.column_names = rows.shift
+    self.data = rows.join("\n").strip
+  end
+  
+  def inspect
+    "<##{self.class.name}" + 
+    [:full_code, :name, :description, :column_names].inject({}){|m,k| m[k] = self.send(k); m }.to_s +
+    " >"
+  end
+  
+  def column_names=(names)
+    names = names.split(",").collect(&:strip) if names.is_a?(String)
+    @column_names = Array(names).flatten
+  end
+  
+  def full_code=(value)
+    value = value.split('/')
+    self.source_code = value[0]
+    self.code = value[1]
+  end
+  
+  def full_code
+    [source_code, code].join('/')
+  end
+  
+  def data=(rows)
+    @data = Quandl::Data.new(rows)
+  end
+  
+end

data/lib/quandl/sandbox/ec2.rb

@@ -0,0 +1,49 @@
+class Quandl::Sandbox::EC2
+  
+  class << self
+    
+    def auto_retry(attempt = 3, &block)
+      block.call
+    rescue => e
+      Quandl::Logger.debug("auto_retry #{e} #{attempt}")
+      Quandl::Sandbox::EC2.gateway_reset!
+      attempt -= 1
+      auto_retry(attempt, &block) if attempt > 0
+    end
+    
+    def find(uid)
+      instances.filter('instance-id', uid ).collect{ |i| i }.first
+    end
+    
+    def instances
+      interface.instances.filter('subnet-id', Sandbox.configuration.subnet_id )
+    end
+    
+    def gateway_reset!
+      @gateway.shutdown!
+      @gateway = nil
+    rescue => err
+      Quandl::Logger.error("gateway_reset! #{err}")
+    end
+    
+    def gateway
+      @gateway ||= Net::SSH::Gateway.new( gateway_host, gateway_user )  
+    end
+    
+    def gateway_user
+      @gateway_user ||= Sandbox.configuration.ssh_gateway.split("@").first
+    end
+    
+    def gateway_host
+      @gateway_host ||= Sandbox.configuration.ssh_gateway.split("@").last
+    end
+    
+    def interface
+      @interface ||= AWS::EC2.new(
+        access_key_id:      Sandbox.configuration.aws_access_key_id,
+        secret_access_key:  Sandbox.configuration.aws_secret_access_key)
+    end
+  
+  end
+  
+end

data/lib/quandl/sandbox/job.rb

@@ -0,0 +1,37 @@
+class Quandl::Sandbox::Job
+  
+  attr_accessor :repository, :server, :server_id, :output, :attempts
+  
+  def initialize(*args)
+    options = args.extract_options!.symbolize_keys!
+    self.repository = Quandl::Sandbox::Repository.new(options[:repository]) if options[:repository].present?
+    self.server = Quandl::Sandbox::Server.find( options[:server_id] ) if options[:server_id].present?
+  end
+  
+  def datasets
+    @datasets ||= Quandl::Sandbox::Dataset.from_output( output[:stdout] )
+  end
+  
+  def execute
+    # ensure the ssh daemon has started
+    server.ensure_connection_established!
+    # run commands that need to happen before importer
+    server.exec( repository.script.before_execute )
+    # execute importer
+    self.output = server.exec( repository.script.execute )
+  rescue => err
+    Quandl::Logger.error( err )
+  ensure
+    server.delete
+  end
+  
+  def output=(value)
+    @datasets = nil
+    @output = value
+  end
+  
+  def server
+    @server ||= Quandl::Sandbox::Server.create
+  end
+  
+end

data/lib/quandl/sandbox/repository.rb

@@ -0,0 +1,67 @@
+class Quandl::Sandbox::Repository
+  
+  include Quandl::Sandbox::Attributes
+  
+  define_attributes :job_uid, :git, :ref
+  
+  def git=(value)
+    @full_path = nil
+    @git = URI.escape( value.to_s )
+  end
+  
+  def ref=(value)
+    @ref = URI.escape( value.to_s )
+  end
+  
+  def full_path
+    @full_path ||= File.join( '/home/ubuntu', dirname )
+  end
+  
+  def dirname
+    @dirname ||= git.to_s.parameterize
+  end
+  
+  def script
+    @script ||= Script.new(self)
+  end
+  
+  class Script
+    
+    attr_accessor :repository
+    
+    def initialize(repo)
+      self.repository = repo
+    end
+    
+    def before_execute
+      [ clone, checkout, pull, bundle ].join("\n")
+    end
+    
+    def execute
+      "cd #{r.full_path}; bundle exec ruby import.rb"
+    end
+    
+    def clone
+      %Q{git clone "#{r.git}" "#{r.full_path}"}
+    end
+  
+    def checkout
+      %Q{ cd #{r.full_path}
+      git checkout -b "#{r.ref}"}
+    end
+  
+    def pull
+      %Q{ cd #{r.full_path}; git pull origin "#{r.ref}" }
+    end
+    
+    def bundle
+      "cd #{r.full_path}; bundle update"
+    end
+    
+    def r
+      repository
+    end
+    
+  end
+  
+end

data/lib/quandl/sandbox/server/attributes.rb

@@ -0,0 +1,57 @@
+module Quandl
+module Sandbox
+
+class Server
+module Attributes
+  
+  extend ActiveSupport::Concern
+  
+  included do
+    include Quandl::Sandbox::Attributes
+    
+    define_attributes :image, :instance_type, :availability_zone,
+      :ssh_gateway, :ssh_user, :subnet_id, :environment, :node_name,
+      :key_name
+    
+    attr_reader :new_record
+    
+    def node_name
+      @node_name ||= "#{environment}-sandbox-worker-#{uid}"
+    end
+    
+  end
+  
+  def initialize(*args)
+    # is a new record
+    self.new_record = true
+    # default attributes
+    self.attributes = Quandl::Sandbox.configuration.to_h
+    # onwards
+    super
+  end
+  
+  def uid
+    @uid ||= SecureRandom.hex(6)
+  end
+  
+  def tags
+    {
+      Name:                   node_name,
+      cluster_type:           'quandl_sandbox',
+      instance_type:          'quandl_ugc_worker',
+      instance_environment:   environment,
+      vpc:                    ssh_gateway.present? ? 'true' : 'false'
+    }
+  end
+  
+  private
+  
+  def new_record=(value)
+    @new_record = (value == true)
+  end
+  
+end
+end
+
+end
+end

data/lib/quandl/sandbox/server/instance.rb

@@ -0,0 +1,116 @@
+module Quandl::Sandbox::Server::Instance
+
+  extend ActiveSupport::Concern
+
+  included do
+  
+    TIMEOUT = 60
+  
+    delegate :status, :delete, :exists?, :subnet,
+    to: :instance, allow_nil: true
+
+    attr_reader :instance
+
+  end
+
+  module ClassMethods
+    
+    def find_or_create(uid)
+      find(uid) || create
+    end
+
+    def find(uid)
+      # lookup instance
+      instance = Sandbox::EC2.find(uid)
+      # build server given instance
+      instance.present? ? self.new( instance: instance ) : nil
+    end
+
+    def create(*args)
+      s = self.new(*args)
+      s.launch!
+      s
+    end
+    
+    def delete(id)
+      i = Sandbox::EC2.find(id)
+      i.present? ? i.delete : false
+    end
+
+  end
+  
+  def initialize(*args)
+    # grab instance if present
+    instance = args.try(:last).try(:[], :instance)
+    # upwards
+    super
+    # assign instance if given
+    self.instance = instance if instance.present?
+  end
+  
+  def private_ip_address
+    @private_ip_address ||= instance.try(:private_ip_address)
+  end
+
+  def instance_id
+    @instance_id ||= instance.try(:id)
+  end
+
+  def running?
+    @running ||= (status == :running) ? true : nil
+  end
+
+  def launch!
+    # only new servers can be launched
+    return false unless new_record
+    # create, tag, wait
+    self.instance = create_instance!
+    add_tags_to_instance
+  end
+
+  def await_instance_uninterruptedly
+    t1 = Time.now
+    Quandl::Logger.debug "#{instance_id}: Waiting for instance to start up ... "
+    Timeout::timeout(TIMEOUT) {
+      sleep 0.5 while status == :pending
+    }
+    Quandl::Logger.debug "#{instance_id}: Instance started. (#{t1.elapsed_ms})"
+    true
+  end
+
+
+  private
+
+
+  def instance=(instance)
+    raise( ArgumentError, "instance must be of class AWS::EC2::Instance") unless instance.kind_of?(AWS::EC2::Instance)
+    return false unless new_record
+    # assign instance
+    @instance = instance
+    assign_attributes_using_instance
+    self.new_record = false
+  end
+
+  def create_instance!
+    Sandbox::EC2.instances.create({
+      image_id:             image,
+      subnet_id:            subnet_id,
+      availability_zone:    availability_zone,
+      key_name:             key_name,
+      instance_type:        instance_type 
+    })
+  end
+
+  def assign_attributes_using_instance
+    self.class.attributes.each do |name|
+      self.send("#{name}=", instance.send(name)) if instance.respond_to?(name)
+    end
+    @node_name = instance.tags['Name']
+    @uid = @node_name.to_s.split('-').last
+  end
+
+  def add_tags_to_instance
+    tags.each{|key, value| instance.add_tag( key.to_s, value: value.to_s ) }
+  end
+
+end

data/lib/quandl/sandbox/server/ssh.rb

@@ -0,0 +1,64 @@
+module Quandl::Sandbox::Server::SSH
+
+  extend ActiveSupport::Concern
+  
+  def ensure_connection_established!
+    Sandbox::EC2.auto_retry(3){ exec("hostname") }
+  end
+  
+  def upload!(local_path, remote_path="/home/ubuntu/")
+    ssh do |tunnel|
+      tunnel.scp.upload!( local_path, remote_path, recursive: true ) do |ch, name, sent, total|
+        Quandl::Logger.debug("#{name}: #{sent}/#{total}")
+      end
+    end
+  end
+
+  def exec(command)
+    exec_through_channel!(command)
+  end
+  
+  def exec_through_channel!(command)
+    t1 = Time.now
+    result = { stdout: '', stderr: '' }
+    self.ssh do |ssh|
+      # run event loop until channel closes
+      channel = ssh.open_channel do |ch|
+        # command to execute
+        ch.exec command do |ch, success|
+          # fail fast
+          raise "could not execute command" unless success
+          # on_data is stdout
+          ch.on_data do |c, data|
+            $stdout.print data
+            result[:stdout] += data
+          end
+          # on_extended_data is stderr
+          ch.on_extended_data do |c, type, data|
+            $stderr.print data
+            result[:stderr] += data
+          end
+          # when command completes
+          ch.on_close { Quandl::Logger.debug("#{instance_id}: Hasta La Vista! (#{t1.elapsed_ms})") }
+        end
+      end
+      channel.wait
+    end
+    result
+  end
+  
+  def ssh(&block)
+    # wait for server to be ready
+    await_instance_uninterruptedly unless running?
+    # establish connection
+    Quandl::Logger.debug "#{instance_id}: Waiting for sshd to start ... "
+    # execute block through tunnel
+    output = nil
+    Sandbox::EC2.gateway.ssh( private_ip_address, ssh_user,
+      key_data: Sandbox.configuration.key_data, keys_only: true ) do |ssh|
+      output = block.call(ssh)
+    end
+    output
+  end
+
+end

data/lib/quandl/sandbox/server.rb

@@ -0,0 +1,9 @@
+require 'quandl/sandbox/server/attributes'
+require 'quandl/sandbox/server/instance'
+require 'quandl/sandbox/server/ssh'
+
+class Quandl::Sandbox::Server
+  include Quandl::Sandbox::Server::Attributes
+  include Quandl::Sandbox::Server::Instance
+  include Quandl::Sandbox::Server::SSH
+end

data/lib/quandl/sandbox/version.rb

@@ -0,0 +1,5 @@
+module Quandl
+module Sandbox
+  VERSION = '0.1.0'
+end
+end

data/lib/quandl/sandbox.rb

@@ -0,0 +1,34 @@
+require 'quandl/sandbox/version'
+require 'quandl/logger'
+require 'quandl/operation'
+
+require 'aws-sdk'
+require 'uri'
+
+
+require 'net/ssh/gateway'
+require 'net/scp'
+
+require "active_support"
+require "active_support/inflector"
+require "active_support/core_ext/hash"
+require "active_support/core_ext/object"
+
+require 'timeout'
+require 'securerandom'
+
+require 'quandl/sandbox/attributes'
+require 'quandl/sandbox/configuration'
+require 'quandl/sandbox/ec2'
+require 'quandl/sandbox/job'
+require 'quandl/sandbox/repository'
+require 'quandl/sandbox/server'
+require 'quandl/sandbox/dataset'
+
+module Quandl
+module Sandbox
+  def self.root
+    @root ||= File.expand_path(File.join(__FILE__, '../../../'))
+  end
+end
+end

data/spec/lib/quandl/sandbox/configuration_spec.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Sandbox::Configuration do
+  subject{ Quandl::Sandbox::Configuration.new }
+  
+  [ :image, :instance_type, :availability_zone, :ssh_gateway, 
+    :ssh_user, :subnet_id, :environment, :verbose, :webhoook ].each do |name|
+    
+    it { should respond_to name }
+  end
+  
+  describe "#to_h" do
+    subject{ Quandl::Sandbox::Configuration.new.to_h }
+    its([:instance_type]){ should eq 'm1.small' }
+    its([:availability_zone]){ should eq 'us-east-1c' }
+    its([:ssh_user]){ should eq 'ubuntu' }
+  end
+  
+  describe "#configuration" do
+    subject{ Quandl::Sandbox.configuration }
+    its(:aws_access_key_id){ should eq ENV['AWS_ACCESS_KEY_ID'] }
+    its(:aws_secret_access_key){ should eq ENV['AWS_SECRET_ACCESS_KEY'] }
+  end
+  
+end

data/spec/lib/quandl/sandbox/ec2_spec.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Sandbox::EC2 do
+  subject{ Quandl::Sandbox::EC2 }
+  its(:interface){ should be_a AWS::EC2 }
+  
+end

data/spec/lib/quandl/sandbox/job_spec.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Sandbox::Job do
+  let(:job){ Quandl::Sandbox::Job.new }
+  subject{ job }
+  describe "#datasets" do
+    before(:each){ job.stub(:output).and_return(stubbed_job_output) }
+    
+    subject{ job.datasets }
+    
+    its(:count){ should eq 4 }
+    
+    describe "#first" do
+      subject{ job.datasets.first }
+
+      its(:source_code){ should eq "QUANDL_SANDBOX_TEST" }
+      its(:code){ should eq "DATASET_0"}
+      its(:name){ should eq "Test Dataset 0" }
+      its(:description){ should eq "Description of the dataset 0"}
+      its(:column_names){ should eq ['Date', 'High', 'Low', 'Average'] }
+      its(:full_code){ should eq "QUANDL_SANDBOX_TEST/DATASET_0"}
+      its('data.count'){ should eq 10 }
+      its('data.first.count'){ should eq 4 }
+    end
+    
+    
+  end
+end

data/spec/lib/quandl/sandbox/repository_spec.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Sandbox::Repository do
+  
+  let(:config){ Quandl::Sandbox.configuration }
+  let(:worker){ Quandl::Sandbox::Repository.new }
+  
+  context "without arguments" do
+    subject{ worker }
+    its(:git){ should be_blank }
+    its(:ref){ should be_blank }
+  end
+    
+  context "with arguments" do
+    subject{ Quandl::Sandbox::Repository.new( git: 'github.com/test42/test84', ref: 'master') }
+    its(:git){ should eq 'github.com/test42/test84'}
+    its(:ref){ should eq 'master'}
+  end
+  
+end

data/spec/lib/quandl/sandbox/server_spec.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Sandbox::Server do
+  
+  let(:config){ Quandl::Sandbox.configuration }
+  let(:server){ Quandl::Sandbox::Server.new }
+  
+  subject{ server }
+  
+  Quandl::Sandbox::Server.attributes.each do |name|
+    its(name){ should eq config.send(name) } if Quandl::Sandbox.configuration.respond_to?(name)
+  end
+  
+  describe "#uid" do
+    subject{ server.uid }
+    its(:length){ should eq 12 }
+  end
+  
+  describe "#node_name" do
+    subject{ server.node_name }
+    it{ should match /#{server.environment}/ }
+    it{ should match /sandbox-worker/ }
+    it{ should match /#{server.uid}/ }
+  end
+  
+  context "with created server" do
+    
+    before(:all) do
+      @server = Quandl::Sandbox::Server.create
+    end
+    
+    subject{ @server }
+    
+    its(:exists?){ should be_true }
+    its(:launch!){ should be_false }
+    its(:instance_id){ should be_present }
+    its(:subnet_id){ should eq Quandl::Sandbox.configuration.subnet_id }
+    
+    describe ".find" do
+      it "should find the server" do
+        s = Quandl::Sandbox::Server.find(subject.instance_id)
+        s.exists?.should be_true
+      end
+      it "should not find servers outside of the subnet" do
+        s = Quandl::Sandbox::Server.find("i-a4002dc5")
+        s.should be_nil
+      end
+    end
+    
+    describe "#exec" do
+      it "should get the hostname" do
+        # get internal_ip through exec('hostname')
+        internal_ip = subject.exec('hostname').rstrip.split('-')[1..-1].join('.')
+        # compare against private ip address
+        subject.private_ip_address.should eq internal_ip
+      end
+    end
+
+    after(:all) do
+      @server.delete rescue nil
+    end
+    
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,38 @@
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require "rspec"
+require 'factory_girl'
+require 'quandl/data'
+require 'quandl/fabricate'
+require 'pry'
+
+factory_dir = File.join( File.dirname(__FILE__), 'factories/**/*.rb' )
+Dir.glob( factory_dir ).each{|f| require(f); puts f }
+
+require 'quandl/sandbox'
+
+Quandl::Logger.use File.join( File.dirname(__FILE__), '../log/test.log' )
+
+Quandl::Sandbox.configure do |c|
+  c.ssh_gateway = ENV['QUANDL_SANDBOX_GATEWAY']
+  c.subnet_id = ENV['QUANDL_SANDBOX_SUBNET']
+  c.image = ENV['QUANDL_SANDBOX_IMAGE']
+  c.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
+  c.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  c.key_file = "/Users/blake/.ssh/quandl_3.pem"
+  c.environment = 'test'
+end
+
+RSpec.configure do |config|
+  config.include FactoryGirl::Syntax::Methods
+end
+
+def stubbed_job_output
+  4.times.collect do |i|
+    "QUANDL_SANDBOX_TEST/DATASET_#{i}\n" +
+    "Test Dataset #{i}\n" +
+    "Description of the dataset #{i}\n" +
+    "Date, High, Low, Average\n" +
+    "#{Quandl::Fabricate::Data.rand(rows: 10, columns: 3).to_csv}}"
+  end.join("----\n")
+end

metadata

@@ -0,0 +1,225 @@
+--- !ruby/object:Gem::Specification
+name: quandl_sandbox
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Blake Hilscher
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: factory_girl_rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fivemat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: net-ssh-gateway
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: quandl_data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: Launch and execute code inside an ec2 instance. Supports vpc private
+  subnet via ssh gateway. Use webhooks to notify success or failure.
+email:
+- blake@hilscher.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/quandl/sandbox/attributes.rb
+- lib/quandl/sandbox/configuration.rb
+- lib/quandl/sandbox/dataset.rb
+- lib/quandl/sandbox/ec2.rb
+- lib/quandl/sandbox/job.rb
+- lib/quandl/sandbox/repository.rb
+- lib/quandl/sandbox/server/attributes.rb
+- lib/quandl/sandbox/server/instance.rb
+- lib/quandl/sandbox/server/ssh.rb
+- lib/quandl/sandbox/server.rb
+- lib/quandl/sandbox/version.rb
+- lib/quandl/sandbox.rb
+- LICENSE
+- README.md
+- spec/lib/quandl/sandbox/configuration_spec.rb
+- spec/lib/quandl/sandbox/ec2_spec.rb
+- spec/lib/quandl/sandbox/job_spec.rb
+- spec/lib/quandl/sandbox/repository_spec.rb
+- spec/lib/quandl/sandbox/server_spec.rb
+- spec/spec_helper.rb
+homepage: http://blake.hilscher.ca/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.10
+signing_key: 
+specification_version: 4
+summary: Launch and execute code inside an ec2 instance.
+test_files:
+- spec/lib/quandl/sandbox/configuration_spec.rb
+- spec/lib/quandl/sandbox/ec2_spec.rb
+- spec/lib/quandl/sandbox/job_spec.rb
+- spec/lib/quandl/sandbox/repository_spec.rb
+- spec/lib/quandl/sandbox/server_spec.rb
+- spec/spec_helper.rb