pipely 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffc556ad3308b03b691185eb21f4ee10f26592f9
-  data.tar.gz: caf8a2a7b8c63c281979bde1ad75f75bdc0f963f
+  metadata.gz: 84180497ce48add0459eec37acfdb95bf66ebe05
+  data.tar.gz: d39f9fef3430493934320d8a88b38a2ced74b42f
 SHA512:
-  metadata.gz: 3b6a6b168162ec377f29e5f416a956fa90c3a1d7a21c462f5104d0ebf8db11d340e95b90ddbaf6a5de74f21673cb15084e2a99eb156d200b69acc54d6243bd06
-  data.tar.gz: 7b659bdb5a01ebe0b4306a3a8c8ec7c426df709e3e56df95a50c1f7134fa3c7d5235efd887b187d1ebbf1c545c2d57d50c33bcb6fa4599a167e6b5930271eda6
+  metadata.gz: dae88dbeb0ed78e2197b90cba2601ed218e7bfc8787e5ca4a28618e5b167b2e9d010bfe378670b8b4a8d688f7b624f44abc123f75173046fbf25e3afa1d69555
+  data.tar.gz: acbb97a002b4f611be91bd2768e25ed75637e9eba962a632d04a4203bfb2db35ca4141d03fe14dfed196313c53e4f49e4945187e3b4b9d8fbc2a9f64d84a0866

data/lib/pipely/bundler.rb

@@ -0,0 +1,29 @@
+require 'pipely/bundler/bundle'
+require 'pipely/bundler/gem_packager'
+require 'pipely/bundler/project_gem'
+
+module Pipely
+
+  #
+  # Module for packaging up a gem project and its dependencies, as they exist
+  # on your machine, for deployment.
+  #
+  # None of this code is specific to AWS Data Pipelines, and it could be used
+  # anywhere else you want to an in-development gem with frozen dependencies.
+  #
+  module Bundler
+
+    # List all the gems used in this project in the format:
+    #
+    #   { name => path_to_cache_file }
+    #
+    # For gems that are git- or path-sourced, it will first build a fresh cache
+    # file for the gem.
+    #
+    def self.gem_files(vendor_dir='vendor/pipeline')
+      ProjectGem.load(vendor_dir).gem_files
+    end
+
+  end
+
+end

data/lib/pipely/bundler/bundle.rb

@@ -0,0 +1,73 @@
+require 'fileutils'
+
+module Pipely
+  module Bundler
+
+    #
+    # Provides access to a bundle's list of gems
+    #
+    class Bundle
+
+      attr_reader :spec_set
+
+      SOURCE_TYPES = %w[Bundler::Source::Git Bundler::Source::Path]
+
+      def self.build(vendor_dir,
+                     groups=[:default],
+                     definition=::Bundler.definition)
+        new(
+          vendor_dir,
+          definition.specs_for(groups),
+          definition.instance_variable_get(:@locked_sources)
+        )
+      end
+
+      def initialize(vendor_dir, spec_set, locked_sources)
+        @spec_set = spec_set
+        @locked_sources = locked_sources
+        @vendor_dir = vendor_dir
+        unless Dir.exists? @vendor_dir
+          FileUtils.mkdir_p(@vendor_dir)
+        end
+
+      end
+
+      def gem_files(gem_packager=GemPackager.new(@vendor_dir))
+        gem_files = {}
+
+        @spec_set.to_a.each do |spec|
+          gem_files.merge!(gem_file(spec, gem_packager))
+        end
+
+        gem_files
+      end
+
+    private
+
+      def gem_file(spec, gem_packager)
+        if source = locked_sources_by_name[spec.name]
+          gem_packager.build_from_source(source.name, source.path)
+        else
+          gem_packager.package(spec)
+        end
+      end
+
+      def locked_sources_by_name
+        return @locked_sources_by_name if @locked_sources_by_name
+
+        @locked_sources_by_name = {}
+
+        @locked_sources.each do |source|
+          # Only include git or path sources.
+          if SOURCE_TYPES.include?(source.class.name)
+            @locked_sources_by_name[source.name] = source
+          end
+        end
+
+        locked_sources_by_name
+      end
+
+    end
+
+  end
+end

data/lib/pipely/bundler/gem_packager.rb

@@ -0,0 +1,119 @@
+require 'fileutils'
+require 'excon'
+
+module Pipely
+  module Bundler
+
+    #
+    # Builds cache files for git- or path-sourced gems.
+    #
+    class GemPackager
+
+      # Alert upon gem-building failures
+      class GemBuildError < RuntimeError ; end
+
+      # Alert upon gem-fetching failures
+      class GemFetchError < RuntimeError ; end
+
+      def initialize(vendor_dir)
+        @vendor_dir = vendor_dir
+        unless Dir.exists? @vendor_dir
+          FileUtils.mkdir_p(@vendor_dir)
+        end
+      end
+
+      def package(spec)
+        if vendored_gem = vendor_local_gem(spec)
+          vendored_gem
+
+        # Finally, some gems do not exist in the cache or as source.  For
+        # instance, json is shipped with the ruby dist. Try to fetch directly
+        # from rubygems.
+        else
+          gem_file_name = "#{spec.name}-#{spec.version}.gem"
+          { spec.name => download_from_rubygems(gem_file_name)}
+        end
+      end
+
+      def vendor_local_gem(spec)
+        gem_file = spec.cache_file
+        vendored_gem = File.join( @vendor_dir, File.basename(gem_file) )
+
+        if File.exists?(vendored_gem)
+          { spec.name => vendored_gem }
+
+        # Gem exists in the local ruby gems cache
+        elsif File.exists? gem_file
+
+          # Copy to vendor dir
+          FileUtils.cp(gem_file, vendored_gem)
+
+          { spec.name => vendored_gem }
+
+        # If source exists, build a gem from it
+        elsif File.directory?(spec.gem_dir)
+          build_from_source(spec.name, spec.gem_dir)
+        else
+          nil
+        end
+      end
+
+      def build_from_source(spec_name, source_path)
+        gem_spec_path = "#{spec_name}.gemspec"
+
+        # Build the gemspec
+        gem_spec = Gem::Specification::load(
+          File.join(source_path,gem_spec_path))
+
+        gem_file = build_gem(spec_name, source_path)
+
+        # Move to vendor dir
+        FileUtils.mv(
+          File.join(source_path,gem_file),
+          File.join(@vendor_dir,gem_file))
+
+        { gem_spec.name => File.join(@vendor_dir, gem_file) }
+      end
+
+      def build_gem(spec_name, source_path)
+        gem_spec_path = "#{spec_name}.gemspec"
+
+        Dir.chdir(source_path) do
+          result = `gem build #{gem_spec_path} 2>&1`
+
+          if result =~ /ERROR/i
+            raise GemBuildError.new(
+              "Failed to build #{gem_spec_path} \n" << result)
+          else
+            result.scan(
+                /File:(.+.gem)$/).flatten.first.strip
+          end
+        end
+      end
+
+      def download_from_rubygems(gem_file_name)
+        vendored_gem = File.join( @vendor_dir, gem_file_name )
+
+        # XXX: add link on wiki details what is going on here
+        puts "Fetching gem #{gem_file_name} directly from rubygems, most " +
+             "likely this gem was packaged along with your ruby " +
+             "distrubtion, for more details see LINK"
+
+        ruby_gem_url = "https://rubygems.org/downloads/#{gem_file_name}"
+        response = Excon.get( ruby_gem_url, {
+          middlewares: Excon.defaults[:middlewares] +
+                       [Excon::Middleware::RedirectFollower]
+        })
+
+        if response.status == 200
+          File.open(vendored_gem, 'w') { |file| file.write( response.body ) }
+          return vendored_gem
+        else
+          raise GemFetchError.new(
+            "Failed to find #{gem_file_name} at rubygems, recieved
+            #{response.status} with #{response.body}" )
+        end
+      end
+    end
+  end
+end

data/lib/pipely/bundler/project_gem.rb

@@ -0,0 +1,47 @@
+module Pipely
+  module Bundler
+
+    #
+    # Builds the project's gem from gemspec and pulls in its dependencies via
+    # the gem's bundle.
+    #
+    class ProjectGem
+
+      attr_reader :project_spec
+
+      def self.load(vendor_dir)
+        if gem_spec = Dir.glob("*.gemspec").first
+          # project gem spec
+          new(Gem::Specification::load(gem_spec), vendor_dir)
+        else
+          raise "Failed to find gemspec"
+        end
+      end
+
+      def initialize(project_spec, vendor_dir)
+        @project_spec = project_spec
+        @vendor_dir = vendor_dir
+        unless Dir.exists? @vendor_dir
+          FileUtils.mkdir_p(@vendor_dir)
+        end
+      end
+
+      def gem_files
+        # Project gem should be at the bottom of the dep list
+        @gem_files ||= dependency_gem_files.merge(project_gem_file)
+      end
+
+      def dependency_gem_files(bundle=Bundle.build(@vendor_dir))
+        # Always exclude bundler and the project gem
+        gems_to_exclude = [ @project_spec.name, 'bundler' ]
+
+        bundle.gem_files.reject { |name, path| gems_to_exclude.include?(name) }
+      end
+
+      def project_gem_file(gem_packager=GemPackager.new(@vendor_dir))
+        gem_packager.build_from_source(@project_spec.name, Dir.pwd)
+      end
+
+    end
+  end
+end

data/lib/pipely/deploy/bootstrap.rb

@@ -1,115 +1,35 @@
+require 'pipely/bundler'
+require 'pipely/deploy/bootstrap_context'
+require 'pipely/deploy/s3_uploader'
+
 module Pipely
   module Deploy
 
     # Helps bootstrap a pipeline
     class Bootstrap
 
-      attr_reader :bucket_name
-      attr_reader :s3_gems_path
       attr_reader :project_spec
       attr_reader :gem_files
 
-      def initialize(s3_bucket, s3_gems_path)
-        @s3_bucket = s3_bucket
-        @bucket_name = s3_bucket.name
-        @s3_gems_path = s3_gems_path
+      def initialize(s3_uploader)
+        @s3_uploader = s3_uploader
       end
 
       # Builds the project's gem from gemspec, uploads the gem to s3, and
       # uploads all the gem dependences to S3
       def build_and_upload_gems
-
-        # placeholder, name will be nil if a gemspec is not defined
-        project_gem_name = nil
-
-        gem_spec = Dir.glob("*.gemspec").first
-        if gem_spec
-
-          # Build pipeline gem
-          @project_spec = Gem::Specification::load(gem_spec)
-          # build the gem
-          project_gem_file =
-            `gem build ./#{gem_spec}`.scan(
-              /File:(.+.gem)$/).flatten.first.strip
-          project_gem_name = @project_spec.name
-          upload_gem(project_gem_file)
-        end
-
-        @gem_files = upload_gems_from_bundler(project_gem_name)
-
-        # project gem has to be loaded last
-        @gem_files << project_gem_file if @project_spec
+        @gem_files = Pipely::Bundler.gem_files
+        @s3_uploader.upload(@gem_files.values)
       end
 
-      def context
-        BootstrapContext.new(
-          @gem_files.map{ |file|
-            File.join("s3://", @s3_bucket.name, gem_s3_path(file) )
-          } )
-      end
-
-      def gem_s3_path(gem_file)
-        filename = File.basename(gem_file)
-        File.join(@s3_gems_path, filename)
-      end
-
-    private
-
-      def s3_gem_exists?( gem_file )
-        !@s3_bucket.objects[gem_s3_path(gem_file)].nil?
-      end
-
-      def upload_gem( gem_file )
-        puts "uploading #{gem_file} to #{gem_s3_path(gem_file)}"
-        @s3_bucket.objects[gem_s3_path(gem_file)].write(File.open(gem_file))
-      end
-
-      def upload_gems_from_bundler(project_gem_name)
-        gem_files = []
-        Bundler.definition.specs_for([:default]).each do |spec|
-          # Exclude project from gem deps
-          unless spec.name == project_gem_name
-            gem_file = spec.cache_file
-
-            # XXX: Some gems do not exist in the cache, e.g. json. Looks the
-            #      gem is already packaged with the ruby dist
-            if File.exists? gem_file
-              gem_files << gem_file
-
-              # only upload gem if it doesnt exist already
-              unless s3_gem_exists?( gem_file )
-                upload_gem(gem_file)
-              end
-            end
-          end
+      def context(s3_steps_path)
+        BootstrapContext.new.tap do |context|
+          context.gem_files = @s3_uploader.s3_urls(gem_files.values)
+          context.s3_steps_path = s3_steps_path
         end
-
-        gem_files
-      end
-    end
-
-    # Context passed to the erb templates
-    class BootstrapContext
-      attr_reader :gem_files
-
-      def initialize(gem_files)
-        @gem_files = gem_files
       end
 
-      def install_gems_script
-        script = ""
-
-        @gem_files.each do |gem_file|
-          filename = File.basename(gem_file)
-          script << %Q[
-# #{filename}
-hadoop fs -copyToLocal #{gem_file} /home/hadoop/#{filename}
-gem install --local /home/hadoop/#{filename} --no-ri --no-rdoc
-          ]
-        end
-
-        script
-      end
     end
+
   end
 end

data/lib/pipely/deploy/bootstrap_context.rb

@@ -0,0 +1,37 @@
+
+module Pipely
+  module Deploy
+
+    # Context passed to the erb templates
+    class BootstrapContext
+      attr_accessor :gem_files
+      attr_accessor :s3_steps_path
+
+      def install_gems_script(transport = :hadoop_fs, &blk)
+        script = ""
+
+        case transport.to_sym
+        when :hadoop_fs
+          transport_cmd = 'hadoop fs -copyToLocal'
+        when :awscli
+          transport_cmd = 'aws s3 cp'
+        else
+          raise "Unsupported transport: #{transport}" unless blk
+        end
+
+        @gem_files.each do |gem_file|
+          filename = File.basename(gem_file)
+          command = "#{transport_cmd} #{gem_file} #{filename}" if transport_cmd
+          command = yield(gem_file, filename, command) if blk
+          script << %Q[
+# #{filename}
+#{command}
+gem install --force --local #{filename} --no-ri --no-rdoc
+]
+        end
+
+        script
+      end
+    end
+  end
+end

data/lib/pipely/deploy/s3_uploader.rb

@@ -0,0 +1,58 @@
+require 'digest/md5'
+
+module Pipely
+  module Deploy
+
+    #
+    # Manage syncing of local files to a particular S3 path
+    #
+    class S3Uploader
+
+      attr_reader :bucket_name
+      attr_reader :s3_path
+
+      def initialize(s3_bucket, s3_path)
+        @s3_bucket = s3_bucket
+        @bucket_name = s3_bucket.name
+        @s3_path = s3_path
+      end
+
+      def s3_file_path(file)
+        filename = File.basename(file)
+        File.join(@s3_path, filename)
+      end
+
+      def s3_urls(files)
+        files.map do |file|
+          File.join("s3://", @s3_bucket.name, s3_file_path(file) )
+        end
+      end
+
+      def upload(files)
+        files.each do |file|
+          upload_file(file)
+        end
+      end
+
+      #
+      # Upload file to S3 unless ETAGs already match.
+      #
+      def upload_file(file)
+        target_path = s3_file_path(file)
+        s3_object = @s3_bucket.objects[target_path]
+
+        content = File.read(file)
+        digest = Digest::MD5.hexdigest(content)
+
+        if s3_object.exists? && (digest == s3_object.etag.gsub('"', ''))
+          puts "skipping #{file} to #{target_path} (ETAG matches)"
+        else
+          puts "uploading #{file} to #{target_path}"
+          s3_object.write(content)
+        end
+      end
+
+    end
+
+  end
+end

data/lib/pipely/tasks/upload_pipeline_as_gem.rb

@@ -1,5 +1,6 @@
 require 'rake'
 require 'aws'
+require 'erubis'
 require 'pipely/deploy/bootstrap'
 
 module Pipely
@@ -59,19 +60,18 @@ module Pipely
       end
 
       def build_bootstrap_context
-        bootstrap_helper =
-          Pipely::Deploy::Bootstrap.new(s3_bucket, @s3_gems_path)
+        s3_uploader = Pipely::Deploy::S3Uploader.new(s3_bucket, @s3_gems_path)
+        bootstrap_helper = Pipely::Deploy::Bootstrap.new(s3_uploader)
         bootstrap_helper.build_and_upload_gems
 
         # erb context
         {
-          bootstrap: bootstrap_helper.context,
+          bootstrap: bootstrap_helper.context(@s3_steps_path),
           config: config
         }
       end
 
       def upload_to_s3( upload_filename, body )
-
         s3_dest = File.join(@s3_steps_path, upload_filename)
         puts "uploading #{s3_dest}" if verbose
         s3_bucket.objects[s3_dest].write(body)

data/lib/pipely/tasks/upload_steps.rb

@@ -57,11 +57,8 @@ module Pipely
 
       def run_task(verbose)
         with_bucket do |bucket|
-          step_files.each do |file_name|
-            dest = file_name.sub(/^#{local_path}/, s3_path)
-            puts "uploading #{dest}" if verbose
-            bucket.objects[dest].write(File.read(file_name))
-          end
+          s3_uploader = Pipely::Deploy::S3Uploader.new(bucket, s3_path)
+          s3_uploader.upload(step_files)
         end
       end
 

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.7.0" unless defined?(::Pipely::VERSION)
+  VERSION = "0.8.0" unless defined?(::Pipely::VERSION)
 end

data/spec/lib/pipely/bundler/bundle_spec.rb

@@ -0,0 +1,72 @@
+require 'pipely/bundler/bundle'
+
+describe Pipely::Bundler::Bundle do
+
+  describe ".build" do
+    let(:groups) { [ :group1 ] }
+    let(:definition) { double "Bundler::Definition" }
+    let(:spec_set) { double }
+
+    before do
+      definition.stub(:specs_for).with(groups) { spec_set }
+    end
+
+    it 'builds a Bundle instance with the spec_set' do
+      bundle = described_class.build('vendor/test', groups, definition)
+      expect(bundle.spec_set).to eq(spec_set)
+    end
+  end
+
+  let(:pipely_spec) { double("Gem::Specification", name: "pipely") }
+  let(:gem1_spec)   { double("Gem::Specification", name: "gem1")   }
+  let(:gem2_spec)   { double("Gem::Specification", name: "gem2")   }
+
+  let(:pipely_source) do
+    Bundler::Source::Path.new('name' => "pipely", 'path' => '.')
+  end
+
+  let(:spec_set) { [ pipely_spec, gem1_spec, gem2_spec ] }
+  let(:locked_sources) { [ pipely_source ] }
+
+  subject { described_class.new('vendor/test', spec_set, locked_sources) }
+
+  describe "#gem_files" do
+    let(:gem_packager) { double }
+
+    before do
+      gem_packager.stub(:package).and_return do |spec|
+        { spec.name => '/path/to/cache/file.gem' }
+      end
+
+      gem_packager.stub(:build_from_source).and_return do |name, path|
+        { name => "#{path}/#{name}-X.Y.Z.gem" }
+      end
+    end
+
+    it "returns a cache file for each gem" do
+      gem_files = subject.gem_files(gem_packager)
+      expect(gem_files.keys).to match_array(%w[ gem1 gem2 pipely ])
+    end
+
+    context "given a packaged/non-locked gem" do
+      it "returns the gems and their existing cache files" do
+        expect(gem_packager).to receive(:package).with(gem1_spec)
+        expect(gem_packager).to receive(:package).with(gem2_spec)
+
+        subject.gem_files(gem_packager)
+      end
+    end
+
+    context "given a locked-source gem" do
+      it "should build new cache files from source" do
+        expect(gem_packager).to receive(:build_from_source).with(
+          pipely_source.name,
+          pipely_source.path
+        )
+
+        subject.gem_files(gem_packager)
+      end
+    end
+  end
+
+end

data/spec/lib/pipely/bundler/gem_packager_spec.rb

@@ -0,0 +1,101 @@
+require 'fileutils'
+require 'pipely/bundler/gem_packager'
+
+describe Pipely::Bundler::GemPackager do
+
+  subject { described_class.new(vendor_path) }
+  let(:vendor_path) { 'vendor/test' }
+
+  before(:each) {
+    unless Dir.exists? vendor_path
+      FileUtils.mkdir_p 'vendor/test'
+    end
+  }
+
+  describe "#package" do
+    let(:gem_spec) do
+      double("spec",
+        name: 'test',
+        cache_file: 'a/cache/file',
+        gem_dir:'a/gem/dir',
+        version:'0.0.1'
+      )
+    end
+
+    let(:vendored_gem) { "vendor/test/file" }
+
+    context "with a cache file" do
+      before do
+        allow(File).to receive(:exists?).with(vendored_gem) { false }
+        allow(File).to receive(:exists?).with(gem_spec.cache_file) { true }
+        allow(FileUtils).to receive(:cp).with(
+          gem_spec.cache_file, vendored_gem)
+      end
+
+      it "returns the cache file" do
+        expect(subject.package(gem_spec)).to eq(
+          {gem_spec.name => vendored_gem}
+        )
+      end
+    end
+
+    context "without a cache file" do
+      before do
+        allow(File).to receive(:exists?).with(gem_spec.cache_file) { false }
+        allow(File).to receive(:exists?).with(vendored_gem) { false }
+      end
+
+      context "if source is available" do
+        before do
+          allow(File).to receive(:directory?).with(gem_spec.gem_dir) { true }
+        end
+
+        it "builds the gem from source" do
+          expect(subject).to receive(:build_from_source).and_return(
+            {"test"=>"a/packaged/file"})
+
+          expect(subject.package(gem_spec)).to eq({"test"=>"a/packaged/file"})
+        end
+      end
+
+      context "if source not available, e.g. json-1.8.1 built into Ruby 2.1" do
+        before do
+          allow(File).to receive(:directory?).with(gem_spec.gem_dir) { false }
+        end
+
+        it "downloads from rubygems" do
+          response = double(:response, status: 200, body: 'im a gem')
+          expect(Excon).to receive(:get).with(
+            "https://rubygems.org/downloads/test-0.0.1.gem", anything())
+            .and_return(response)
+          expect(subject.package(gem_spec)).to eq(
+            {"test"=>"vendor/test/test-0.0.1.gem"})
+        end
+      end
+
+      context "if source not available and not on rubygems" do
+        before do
+          allow(File).to receive(:directory?).with(gem_spec.gem_dir) { false }
+        end
+
+        it "raises" do
+          response = double(:response, status: 400, body: 'test')
+          expect(Excon).to receive(:get).with(
+            "https://rubygems.org/downloads/test-0.0.1.gem", anything())
+            .and_return(response)
+
+          expect { subject.package(gem_spec) }.to raise_error
+        end
+      end
+    end
+  end
+
+  describe "#build_from_source" do
+    context "with bad spec" do
+      it "raises" do
+        expect { subject.build_from_source("bad-name", ".") }.to raise_error
+      end
+    end
+  end
+
+end

data/spec/lib/pipely/bundler/project_gem_spec.rb

@@ -0,0 +1,95 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+require 'pipely/deploy/bootstrap'
+
+describe Pipely::Bundler::ProjectGem do
+
+  let(:project_spec) do
+    double "Gem::Specification",
+      name: "my-project",
+      file_name: "/path/to/cache/my-project.gem"
+  end
+
+  subject { described_class.new(project_spec, 'vendor/test') }
+
+  describe ".load" do
+    let(:filename) { 'foo.gemspec' }
+    let(:gemspec) { double }
+
+    before do
+      allow(Dir).to receive(:glob).with("*.gemspec") { [ filename ] }
+      allow(Gem::Specification).to receive(:load).with(filename) { gemspec }
+    end
+
+    it "loads the gemspec" do
+      loaded = described_class.load('vendor/test')
+      expect(loaded.project_spec).to eq(gemspec)
+    end
+  end
+
+  describe "#gem_files" do
+    let(:dependency_gem_files) do
+      {
+        'packaged-gem1' => '/path/to/cache/packaged-gem1.gem',
+        'built-from-source-gem1' => '/path/to/cache/built-from-source-gem1.gem',
+      }
+    end
+
+    let(:project_gem_file) do
+      {
+        project_spec.name => project_spec.file_name
+      }
+    end
+
+    before do
+      allow(subject).to receive(:dependency_gem_files) { dependency_gem_files }
+      allow(subject).to receive(:project_gem_file) { project_gem_file }
+    end
+
+    it "combines the dependency_gem_files and the project_gem_file" do
+      expect(subject.gem_files.keys).to match_array(
+        dependency_gem_files.keys + project_gem_file.keys
+      )
+    end
+
+    it "lists the project_gem_file last" do
+      expect(subject.gem_files.keys.last).to eq(project_spec.name)
+    end
+  end
+
+  describe "#dependency_gem_files" do
+    let(:bundle_gem_files) do
+      {
+        'packaged-gem1' => '/path/to/cache/packaged-gem1.gem',
+        'built-from-source-gem1' => '/path/to/cache/built-from-source-gem1.gem',
+        'bundler' => '/path/to/cache/bundler.gem',
+        project_spec.name => project_spec.file_name,
+      }
+    end
+
+    let(:bundle) do
+      double "Pipely::Bundler::Bundle", gem_files: bundle_gem_files
+    end
+
+    it "should filter out the bundler gem and the project gem" do
+      result = subject.dependency_gem_files(bundle)
+      expect(result.keys).to eq(%w[ packaged-gem1 built-from-source-gem1 ])
+    end
+  end
+
+  describe "#project_gem_file" do
+    let(:gem_packager) { double "Pipely::Bundler::GemPackager" }
+    let(:project_gem_file) { double }
+
+    before do
+      allow(gem_packager).to receive(:build_from_source) { project_gem_file }
+    end
+
+    it "should return the project's own gem file" do
+      result = subject.project_gem_file(gem_packager)
+      expect(result).to eq(project_gem_file)
+    end
+  end
+
+end

data/spec/lib/pipely/deploy/bootstrap_context_spec.rb

@@ -0,0 +1,57 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+require 'pipely/deploy/bootstrap_context'
+require 'fileutils'
+
+describe Pipely::Deploy::BootstrapContext do
+  subject do
+    Pipely::Deploy::BootstrapContext.new.tap do |context|
+      context.gem_files = ['one.gem', 'two.gem']
+    end
+  end
+
+  describe "#install_gems_script" do
+    it "defaults to hadoop fs" do
+      expect(subject.install_gems_script).to eql "
+# one.gem
+hadoop fs -copyToLocal one.gem one.gem
+gem install --force --local one.gem --no-ri --no-rdoc
+
+# two.gem
+hadoop fs -copyToLocal two.gem two.gem
+gem install --force --local two.gem --no-ri --no-rdoc
+"
+    end
+
+    context "with aws cli" do
+      it "should build script for aws cli" do
+        expect(subject.install_gems_script(:awscli) ).to eql "
+# one.gem
+aws s3 cp one.gem one.gem
+gem install --force --local one.gem --no-ri --no-rdoc
+
+# two.gem
+aws s3 cp two.gem two.gem
+gem install --force --local two.gem --no-ri --no-rdoc
+"
+      end
+    end
+
+    context "with yield" do
+      it "should build script for aws cli" do
+        expect(subject.install_gems_script(:awscli) do |file,filename,command|
+          "custom command - #{file} #{filename} #{command}"
+        end).to eql "
+# one.gem
+custom command - one.gem one.gem aws s3 cp one.gem one.gem
+gem install --force --local one.gem --no-ri --no-rdoc
+
+# two.gem
+custom command - two.gem two.gem aws s3 cp two.gem two.gem
+gem install --force --local two.gem --no-ri --no-rdoc
+"
+      end
+    end
+  end
+end

data/spec/lib/pipely/deploy/bootstrap_spec.rb

@@ -2,85 +2,52 @@
 
 require 'spec_helper'
 require 'pipely/deploy/bootstrap'
-require 'fog'
 require 'fileutils'
 
-module Pipely
-  describe Deploy::Bootstrap do
-    subject { described_class.new(s3_bucket, 'test_path/gems') }
-    let(:s3_bucket) do
-       s3 = AWS::S3.new
-       s3.buckets['a-test-bucket']
-    end
-
-    it "should have bucket name" do
-      expect(subject.bucket_name).to eql 'a-test-bucket'
-    end
-
-    it "should have a s3 gems path" do
-      expect(subject.s3_gems_path).to eql 'test_path/gems'
-    end
-
-    describe "#build_and_upload_gems" do
-      let(:build_and_upload_gems) do
-        VCR.use_cassette('build_and_upload_gems') do
-          subject.build_and_upload_gems
-        end
-      end
-
-      it "should create project gem" do
-        build_and_upload_gems
-
-        expect(File).to exist(subject.project_spec.file_name)
-      end
+describe Pipely::Deploy::Bootstrap do
 
-      it "should upload gems" do
-        specs = Bundler.definition.specs_for([:default])
+  subject { described_class.new(s3_uploader) }
 
-        gems = specs.map(&:file_name).reject do |g|
-          # Ignore bundler, since it could be a system installed gem (travis)
-          # without a cache file
-          g.match(/^json-\d+\.\d+\.\d+\.gem$/) or
-            g.match(/^bundler-\d+\.\d+\.\d+\.gem$/)
-        end
+  let(:s3_uploader) { double }
 
-        objects = double(:objects)
-        s3_object = double('s3_object', write: nil)
-
-        allow(objects).to receive(:[]) { s3_object }
+  let(:gem_files) do
+    {
+      'packaged-gem1' => '/path/to/cache/packaged-gem1.gem',
+      'built-from-source-gem1' => '/path/to/cache/built-from-source-gem1.gem',
+    }
+  end
 
-        gems.each do |gem|
-          expect(objects).to receive(:[]).with(subject.gem_s3_path(gem))
-        end
+  describe "#build_and_upload_gems" do
+    before do
+      allow(Pipely::Bundler).to receive(:gem_files) { gem_files }
+    end
 
-        expect(s3_bucket).to(receive(:objects)).
-          at_least(gems.size).times.
-          at_most(gems.size + 1).times. # allow for Bundler
-          and_return(objects)
+    it 'uploads each gem' do
+      expect(s3_uploader).to receive(:upload).with(gem_files.values)
 
-        build_and_upload_gems
-      end
+      subject.build_and_upload_gems
     end
+  end
 
-    describe "#context" do
-      let(:context) { subject.context}
+  describe "#context" do
+    let(:context) { subject.context(s3_steps_path) }
+    let(:s3_steps_path) { 'a/test/path' }
+    let(:s3_gem_paths) { double }
 
-      before do
-        VCR.use_cassette('build_and_upload_gems') do
-          subject.build_and_upload_gems
-        end
-      end
+    before do
+      allow(subject).to receive(:gem_files) { gem_files }
 
-      it "should have gem_files" do
-        expect(context.gem_files).to_not be_nil
+      allow(s3_uploader).to receive(:s3_urls).with(gem_files.values) do
+        s3_gem_paths
       end
+    end
 
-      it "should be an s3 url" do
-        expect(context.gem_files.first).to(
-          match( /^s3:\/\/#{subject.bucket_name}\/#{subject.s3_gems_path}/ )
-        )
-      end
+    it "should have s3 steps path" do
+      expect(context.s3_steps_path).to eq(s3_steps_path)
     end
 
+    it "builds S3 urls to the uploaded gem files" do
+      expect(context.gem_files).to eq(s3_gem_paths)
+    end
   end
 end

data/spec/lib/pipely/deploy/s3_uploader_spec.rb

@@ -0,0 +1,54 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+require 'pipely/deploy/s3_uploader'
+require 'tempfile'
+
+describe Pipely::Deploy::S3Uploader do
+
+  subject { described_class.new(s3_bucket, 'test_path/gems') }
+
+  let(:s3_bucket) do
+     s3 = AWS::S3.new
+     s3.buckets['a-test-bucket']
+  end
+
+  it "should have bucket name" do
+    expect(subject.bucket_name).to eq('a-test-bucket')
+  end
+
+  it "should have a s3 path" do
+    expect(subject.s3_path).to eq('test_path/gems')
+  end
+
+  describe "#upload(files)" do
+    let(:objects) { double(:objects) }
+
+    let(:s3_object) do
+      double('s3_object', write: nil, exists?: true, etag: 'mismatch')
+    end
+
+    let(:files) do
+      [
+        Tempfile.new('packaged-gem1.gem').path,
+        Tempfile.new('built-from-source-gem1.gem').path,
+      ]
+    end
+
+    before do
+      allow(objects).to receive(:[]) { s3_object }
+      allow(s3_bucket).to receive(:objects) { objects }
+    end
+
+    it 'uploads each file' do
+      files.each do |file|
+        expect(objects).to receive(:[]).with(subject.s3_file_path(file))
+      end
+
+      expect(s3_bucket).to receive(:objects).exactly(files.size).times
+
+      subject.upload(files)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -3,6 +3,7 @@ require 'aws-sdk'
 require 'fog'
 require 'rspec'
 require 'vcr'
+require 'pry'
 
 $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-11 00:00:00.000000000 Z
+date: 2014-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
@@ -206,6 +206,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - matt@swipely.com
@@ -230,12 +244,18 @@ files:
 - lib/pipely/build/s3_path_builder.rb
 - lib/pipely/build/template.rb
 - lib/pipely/build/template_helpers.rb
+- lib/pipely/bundler.rb
+- lib/pipely/bundler/bundle.rb
+- lib/pipely/bundler/gem_packager.rb
+- lib/pipely/bundler/project_gem.rb
 - lib/pipely/component.rb
 - lib/pipely/definition.rb
 - lib/pipely/dependency.rb
 - lib/pipely/deploy.rb
 - lib/pipely/deploy/bootstrap.rb
+- lib/pipely/deploy/bootstrap_context.rb
 - lib/pipely/deploy/client.rb
+- lib/pipely/deploy/s3_uploader.rb
 - lib/pipely/fog_client.rb
 - lib/pipely/graph_builder.rb
 - lib/pipely/live_pipeline.rb
@@ -257,11 +277,16 @@ files:
 - spec/lib/pipely/build/s3_path_builder_spec.rb
 - spec/lib/pipely/build/template_spec.rb
 - spec/lib/pipely/build_spec.rb
+- spec/lib/pipely/bundler/bundle_spec.rb
+- spec/lib/pipely/bundler/gem_packager_spec.rb
+- spec/lib/pipely/bundler/project_gem_spec.rb
 - spec/lib/pipely/component_spec.rb
 - spec/lib/pipely/definition_spec.rb
 - spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/deploy/bootstrap_context_spec.rb
 - spec/lib/pipely/deploy/bootstrap_spec.rb
 - spec/lib/pipely/deploy/client_spec.rb
+- spec/lib/pipely/deploy/s3_uploader_spec.rb
 - spec/lib/pipely/graph_builder_spec.rb
 - spec/lib/pipely/reference_list_spec.rb
 - spec/lib/pipely_spec.rb
@@ -297,11 +322,16 @@ test_files:
 - spec/lib/pipely/build/s3_path_builder_spec.rb
 - spec/lib/pipely/build/template_spec.rb
 - spec/lib/pipely/build_spec.rb
+- spec/lib/pipely/bundler/bundle_spec.rb
+- spec/lib/pipely/bundler/gem_packager_spec.rb
+- spec/lib/pipely/bundler/project_gem_spec.rb
 - spec/lib/pipely/component_spec.rb
 - spec/lib/pipely/definition_spec.rb
 - spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/deploy/bootstrap_context_spec.rb
 - spec/lib/pipely/deploy/bootstrap_spec.rb
 - spec/lib/pipely/deploy/client_spec.rb
+- spec/lib/pipely/deploy/s3_uploader_spec.rb
 - spec/lib/pipely/graph_builder_spec.rb
 - spec/lib/pipely/reference_list_spec.rb
 - spec/lib/pipely_spec.rb