pipely 0.8.3 → 0.10.0

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.8.3" unless defined?(::Pipely::VERSION)
+  VERSION = '0.10.0' unless defined?(::Pipely::VERSION)
 end

data/spec/fixtures/bootstrap_contexts/green.rb

@@ -0,0 +1,9 @@
+module Fixtures
+  module BootstrapContexts
+    module Green
+      def green
+        "green"
+      end
+    end
+  end
+end

data/spec/fixtures/bootstrap_contexts/simple.rb

@@ -0,0 +1,9 @@
+module Fixtures
+  module BootstrapContexts
+    module Simple
+      def simple
+        "simple"
+      end
+    end
+  end
+end

data/spec/fixtures/templates/bootstrap.sh.erb

@@ -0,0 +1,4 @@
+one
+two
+three
+<%= bootstrap.simple %>

data/spec/lib/pipely/build/environment_config_spec.rb

@@ -0,0 +1,58 @@
+require 'pipely/build/environment_config'
+
+describe Pipely::Build::EnvironmentConfig do
+
+  describe '.load(filename, environment)' do
+    let(:filename) { 'path/to/config/yaml.yml' }
+
+    let(:config) do
+      YAML.load(<<-EOS)
+        my_env:
+          key: 'my_val'
+        production:
+          key: 'prod_val'
+        staging:
+          key: 'staging_val'
+      EOS
+    end
+
+    before do
+      allow(YAML).to receive(:load_file).with(filename) { config }
+    end
+
+    context 'given a custom environment' do
+      subject { described_class.load(filename, 'my_env') }
+
+      it 'loads config from a YAML file' do
+        expect(subject[:key]).to eq('my_val')
+      end
+    end
+
+    context 'given the "production" environment' do
+      subject { described_class.load(filename, 'production') }
+
+      it 'loads config from a YAML file' do
+        expect(subject[:key]).to eq('prod_val')
+      end
+
+      it 'supports legacy defaults' do
+        expect(subject[:s3_prefix]).to eq('production/:namespace')
+        expect(subject[:scheduler]).to eq('daily')
+      end
+    end
+
+    context 'given the "staging" environment' do
+      subject { described_class.load(filename, 'staging') }
+
+      it 'loads config from a YAML file' do
+        expect(subject[:key]).to eq('staging_val')
+      end
+
+      it 'supports legacy defaults' do
+        expect(subject[:s3_prefix]).to eq('staging/:whoami/:namespace')
+        expect(subject[:scheduler]).to eq('now')
+      end
+    end
+  end
+
+end

data/spec/lib/pipely/build/s3_path_builder_spec.rb

@@ -35,9 +35,13 @@ describe Pipely::Build::S3PathBuilder do
     should eq("s3://asset-bucket/run-prefix/shared/\#{format(@scheduledStartTime,'YYYY-MM-dd')}")
   }
 
+  its(:bucket_relative_s3_asset_prefix) {
+    should eq("run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}")
+  }
+
   describe "#to_hash" do
     it 'includes the necessary keys for supplying config to a Template' do
-      expect(subject.to_hash.keys).to match_array([
+      expect(subject.to_hash.keys).to include(
         :s3_log_prefix,
         :s3_step_prefix,
         :s3n_step_prefix,
@@ -45,7 +49,35 @@ describe Pipely::Build::S3PathBuilder do
         :s3n_asset_prefix,
         :s3_shared_asset_prefix,
         :bucket_relative_s3_asset_prefix,
-      ])
+      )
+    end
+  end
+
+  context "when a custom template is specified via config" do
+    subject {
+      described_class.new(
+        foo: 'my-value',
+        templates: {
+          bar: ':protocol://my-bucket/:foo/okay'
+        }
+      )
+    }
+
+    its(:s3_bar_prefix) {
+      should eq('s3://my-bucket/my-value/okay')
+    }
+
+    its(:s3n_bar_prefix) {
+      should eq('s3n://my-bucket/my-value/okay')
+    }
+
+    describe "#to_hash" do
+      it 'includes the keys for the custom template' do
+        expect(subject.to_hash.keys).to include(
+          :s3_bar_prefix,
+          :s3n_bar_prefix,
+        )
+      end
     end
   end
 

data/spec/lib/pipely/build/template_spec.rb

@@ -17,17 +17,17 @@ describe Pipely::Build::Template do
   end
 
   describe "#streaming_hadoop_step(options)" do
-    let(:s3_path_builder) {
-      Pipely::Build::S3PathBuilder.new(
-        logs: 'log-bucket',
-        steps: 'step-bucket',
-        assets: 'asset-bucket',
-        prefix: 'run-prefix'
-      )
-    }
-
     before do
-      subject.apply_config(s3_path_builder.to_hash)
+      # emulate applying config from S3PathBuilder, as done in Definition#to_json
+      subject.apply_config({
+        s3_log_prefix: "s3://log-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}",
+        s3_step_prefix: "s3://step-bucket/run-prefix",
+        s3n_step_prefix: "s3n://step-bucket/run-prefix",
+        s3_asset_prefix: "s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}",
+        s3n_asset_prefix: "s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}",
+        s3_shared_asset_prefix: "s3://asset-bucket/run-prefix/shared/\#{format(@scheduledStartTime,'YYYY-MM-dd')}",
+        bucket_relative_s3_asset_prefix: "run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}",
+      })
     end
 
     it "builds a streaming hadoop step" do

data/spec/lib/pipely/build_spec.rb

@@ -1,3 +1,32 @@
+require 'pipely/build'
+
 describe Pipely::Build do
 
+  describe '.build_definition(template, environment, config_path)' do
+
+    let(:template) { double }
+    let(:environment) { 'production' }
+    let(:config_path) { 'path/to/config' }
+
+    let(:config) { double }
+
+    before do
+      allow(Pipely::Build::EnvironmentConfig).to receive(:load).
+        with(config_path, environment.to_sym).
+        and_return(config)
+    end
+
+    it 'builds a Definition' do
+      expect(
+        described_class.build_definition(template, environment, config_path)
+      ).to eq(
+        Pipely::Build::Definition.new(
+          template,
+          environment.to_sym,
+          config
+        )
+      )
+    end
+  end
+
 end

data/spec/lib/pipely/deploy/bootstrap_context_spec.rb

@@ -2,7 +2,6 @@
 
 require 'spec_helper'
 require 'pipely/deploy/bootstrap_context'
-require 'fileutils'
 
 describe Pipely::Deploy::BootstrapContext do
   subject do
@@ -11,9 +10,20 @@ describe Pipely::Deploy::BootstrapContext do
     end
   end
 
-  describe "#install_gems_script" do
-    it "defaults to hadoop fs" do
-      expect(subject.install_gems_script).to eql "
+  let(:aws_install_gems_script) do
+"
+# one.gem
+aws s3 cp one.gem one.gem
+gem install --force --local one.gem --no-ri --no-rdoc
+
+# two.gem
+aws s3 cp two.gem two.gem
+gem install --force --local two.gem --no-ri --no-rdoc
+"
+  end
+
+  let(:hadoop_install_gems_script) do
+"
 # one.gem
 hadoop fs -copyToLocal one.gem one.gem
 gem install --force --local one.gem --no-ri --no-rdoc
@@ -22,35 +32,113 @@ gem install --force --local one.gem --no-ri --no-rdoc
 hadoop fs -copyToLocal two.gem two.gem
 gem install --force --local two.gem --no-ri --no-rdoc
 "
+  end
+
+  describe "#install_gems_script" do
+    it "with hadoop fs" do
+      expect(subject.install_gems_script(:hadoop_fs)).to eql(
+        hadoop_install_gems_script)
     end
 
     context "with aws cli" do
       it "should build script for aws cli" do
-        expect(subject.install_gems_script(:awscli) ).to eql "
+        expect(subject.install_gems_script(:awscli) ).to eql(
+          aws_install_gems_script)
+      end
+    end
+
+    context "with yield" do
+      it "should build script for aws cli" do
+        expect(subject.install_gems_script(:awscli) do |command,file,filename|
+          "custom command - #{file} #{filename} #{command}"
+        end).to eql "
 # one.gem
-aws s3 cp one.gem one.gem
+custom command - one.gem one.gem aws s3 cp
 gem install --force --local one.gem --no-ri --no-rdoc
 
 # two.gem
-aws s3 cp two.gem two.gem
+custom command - two.gem two.gem aws s3 cp
 gem install --force --local two.gem --no-ri --no-rdoc
 "
       end
     end
 
-    context "with yield" do
-      it "should build script for aws cli" do
-        expect(subject.install_gems_script(:awscli) do |file,filename,command|
-          "custom command - #{file} #{filename} #{command}"
-        end).to eql "
+    context "using the emr context" do
+      describe "#install_gems_script" do
+        it "build script using hadoop fs" do
+          expect(subject.install_gems_script(:hadoop_fs)).to eql "
 # one.gem
-custom command - one.gem one.gem aws s3 cp one.gem one.gem
+hadoop fs -copyToLocal one.gem one.gem
 gem install --force --local one.gem --no-ri --no-rdoc
 
 # two.gem
-custom command - two.gem two.gem aws s3 cp two.gem two.gem
+hadoop fs -copyToLocal two.gem two.gem
 gem install --force --local two.gem --no-ri --no-rdoc
 "
+        end
+      end
+    end
+
+    context "using the emr context" do
+      let(:emr) { subject.emr }
+
+      describe '#install_gems_script' do
+        it 'should be same as parent hadoop install script' do
+          expect(emr.install_gems_script).to eq(hadoop_install_gems_script)
+        end
+      end
+    end
+
+    context "using the ec2 context" do
+      let(:ec2) { subject.ec2 }
+
+      describe '#install_gems_script' do
+        it 'should be same as parent aws install script' do
+          expect(ec2.install_gems_script).to eq(aws_install_gems_script)
+        end
+      end
+
+      describe "#as_root" do
+
+        context "on first run" do
+          it "should build script with ssh init" do
+            expect(ec2.as_root { "Custom Script here" }).to eql "
+# Set up ssh access
+if [ ! -f ~/.ssh/id_rsa ]; then
+  mkdir -p ~/.ssh
+  ssh-keygen -P '' -f ~/.ssh/id_rsa
+  cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+  chmod 600 ~/.ssh/authorized_keys
+fi
+
+# Use ssh to bypass the sudo \"require tty\" setting
+ssh -o \"StrictHostKeyChecking no\" -t -t ec2-user@localhost <<- EOF
+  sudo su -;
+Custom Script here
+  # exit twice, once for su and once for ssh
+  exit;
+  exit;
+EOF
+"
+          end
+        end
+
+        context "on consective runs" do
+          it "should build script" do
+            ec2.as_root { "First run" }
+
+            expect(ec2.as_root { "Second run" }).to eql "
+# Use ssh to bypass the sudo \"require tty\" setting
+ssh -o \"StrictHostKeyChecking no\" -t -t ec2-user@localhost <<- EOF
+  sudo su -;
+Second run
+  # exit twice, once for su and once for ssh
+  exit;
+  exit;
+EOF
+"
+          end
+        end
       end
     end
   end

data/spec/lib/pipely/deploy/bootstrap_registry_spec.rb

@@ -0,0 +1,32 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+require 'pipely/deploy/bootstrap_registry'
+
+describe Pipely::Deploy::BootstrapRegistry do
+
+  subject { described_class }
+
+  describe "#mixins" do
+    it "should default to empty" do
+      expect(subject.mixins).to be_empty
+    end
+  end
+
+  describe "#register_mixins" do
+    context "with a mixin" do
+      let(:mixin) { "Fixtures::BootstrapContexts::Green" }
+      let(:result) { [mixin] }
+      it "should registry mixin" do
+        expect(subject.register_mixins(mixin)).to eql(result)
+        expect(subject.mixins).to eql(result)
+      end
+    end
+
+    context "when a mixin cannot be required" do
+      it "should raise" do
+        expect { subject.register_mixins('bad::mixin') }.to raise_error
+      end
+    end
+  end
+end

data/spec/lib/pipely/deploy/bootstrap_spec.rb

@@ -2,14 +2,15 @@
 
 require 'spec_helper'
 require 'pipely/deploy/bootstrap'
+require 'pipely/deploy/bootstrap_registry'
 require 'fileutils'
+require 'fixtures/bootstrap_contexts/simple'
+require 'fixtures/bootstrap_contexts/green'
 
 describe Pipely::Deploy::Bootstrap do
 
-  subject { described_class.new(s3_uploader) }
-
-  let(:s3_uploader) { double }
-
+  subject { described_class.new(gem_files, s3_steps_path) }
+  let(:s3_steps_path) { 'a/test/path' }
   let(:gem_files) do
     {
       'packaged-gem1' => '/path/to/cache/packaged-gem1.gem',
@@ -17,37 +18,53 @@ describe Pipely::Deploy::Bootstrap do
     }
   end
 
-  describe "#build_and_upload_gems" do
-    before do
-      allow(Pipely::Bundler).to receive(:gem_files) { gem_files }
+  describe "#context" do
+    context "without any mixins" do
+      let(:context) { subject.context }
+
+      it "should have s3 steps path" do
+        expect(context.s3_steps_path).to eq(s3_steps_path)
+      end
+
+      it "builds S3 urls to the uploaded gem files" do
+        expect(context.gem_files).to eq(gem_files)
+      end
     end
 
-    it 'uploads each gem' do
-      expect(s3_uploader).to receive(:upload).with(gem_files.values)
+    context "with one mixin" do
+      let(:context) { subject.context( mixin.name ) }
+      let(:mixin) { Fixtures::BootstrapContexts::Simple }
 
-      subject.build_and_upload_gems
+      it "should have Simple mixin method" do
+        expect(context.simple).to eq("simple")
+      end
     end
-  end
 
-  describe "#context" do
-    let(:context) { subject.context(s3_steps_path) }
-    let(:s3_steps_path) { 'a/test/path' }
-    let(:s3_gem_paths) { double }
+    context "with multiple mixins" do
+      let(:context) { subject.context( mixins.map(&:name) ) }
+      let(:mixins) do
+        [Fixtures::BootstrapContexts::Simple,Fixtures::BootstrapContexts::Green]
+      end
 
-    before do
-      allow(subject).to receive(:gem_files) { gem_files }
+      it "should have simple mixin method" do
+        expect(context.simple).to eq("simple")
+      end
 
-      allow(s3_uploader).to receive(:s3_urls).with(gem_files.values) do
-        s3_gem_paths
+      it "should have green mixin method" do
+        expect(context.green).to eq("green")
       end
     end
 
-    it "should have s3 steps path" do
-      expect(context.s3_steps_path).to eq(s3_steps_path)
-    end
+    context "with mixin from BootstrapRegistry" do
+      let(:context) { subject.context }
+      before do
+        Pipely::Deploy::BootstrapRegistry.instance.register_mixins(
+          "Fixtures::BootstrapContexts::Simple")
+      end
 
-    it "builds S3 urls to the uploaded gem files" do
-      expect(context.gem_files).to eq(s3_gem_paths)
+      it "should have green mixin method" do
+        expect(context.green).to eq("green")
+      end
     end
   end
 end