pipely 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 74943132df56cd3648e12d5f45fb4761fb4bc3d2
-  data.tar.gz: bee0d005acd89ba86f703090877350f667120e3f
+  metadata.gz: ffc556ad3308b03b691185eb21f4ee10f26592f9
+  data.tar.gz: caf8a2a7b8c63c281979bde1ad75f75bdc0f963f
 SHA512:
-  metadata.gz: 1e98b2c4ea1c5042b70b8a56f87b11a45211444f9b7d7fbae84bf27ddea676f11d1d51a06315599581743e62674618d818ae2c896ca8be3fda01999cfc02185c
-  data.tar.gz: 69f416836edc06e0a24b8fea78ad1568b7a1ecea873e63ad58a7d598c0e50126067bc33ff0ea7eed7879a89ffe4b69d43fd6ddfcb890a0f8eda3e95ee4f90e3f
+  metadata.gz: 3b6a6b168162ec377f29e5f416a956fa90c3a1d7a21c462f5104d0ebf8db11d340e95b90ddbaf6a5de74f21673cb15084e2a99eb156d200b69acc54d6243bd06
+  data.tar.gz: 7b659bdb5a01ebe0b4306a3a8c8ec7c426df709e3e56df95a50c1f7134fa3c7d5235efd887b187d1ebbf1c545c2d57d50c33bcb6fa4599a167e6b5930271eda6

data/lib/pipely/deploy/bootstrap.rb

@@ -0,0 +1,115 @@
+module Pipely
+  module Deploy
+
+    # Helps bootstrap a pipeline
+    class Bootstrap
+
+      attr_reader :bucket_name
+      attr_reader :s3_gems_path
+      attr_reader :project_spec
+      attr_reader :gem_files
+
+      def initialize(s3_bucket, s3_gems_path)
+        @s3_bucket = s3_bucket
+        @bucket_name = s3_bucket.name
+        @s3_gems_path = s3_gems_path
+      end
+
+      # Builds the project's gem from gemspec, uploads the gem to s3, and
+      # uploads all the gem dependences to S3
+      def build_and_upload_gems
+
+        # placeholder, name will be nil if a gemspec is not defined
+        project_gem_name = nil
+
+        gem_spec = Dir.glob("*.gemspec").first
+        if gem_spec
+
+          # Build pipeline gem
+          @project_spec = Gem::Specification::load(gem_spec)
+          # build the gem
+          project_gem_file =
+            `gem build ./#{gem_spec}`.scan(
+              /File:(.+.gem)$/).flatten.first.strip
+          project_gem_name = @project_spec.name
+          upload_gem(project_gem_file)
+        end
+
+        @gem_files = upload_gems_from_bundler(project_gem_name)
+
+        # project gem has to be loaded last
+        @gem_files << project_gem_file if @project_spec
+      end
+
+      def context
+        BootstrapContext.new(
+          @gem_files.map{ |file|
+            File.join("s3://", @s3_bucket.name, gem_s3_path(file) )
+          } )
+      end
+
+      def gem_s3_path(gem_file)
+        filename = File.basename(gem_file)
+        File.join(@s3_gems_path, filename)
+      end
+
+    private
+
+      def s3_gem_exists?( gem_file )
+        !@s3_bucket.objects[gem_s3_path(gem_file)].nil?
+      end
+
+      def upload_gem( gem_file )
+        puts "uploading #{gem_file} to #{gem_s3_path(gem_file)}"
+        @s3_bucket.objects[gem_s3_path(gem_file)].write(File.open(gem_file))
+      end
+
+      def upload_gems_from_bundler(project_gem_name)
+        gem_files = []
+        Bundler.definition.specs_for([:default]).each do |spec|
+          # Exclude project from gem deps
+          unless spec.name == project_gem_name
+            gem_file = spec.cache_file
+
+            # XXX: Some gems do not exist in the cache, e.g. json. Looks the
+            #      gem is already packaged with the ruby dist
+            if File.exists? gem_file
+              gem_files << gem_file
+
+              # only upload gem if it doesnt exist already
+              unless s3_gem_exists?( gem_file )
+                upload_gem(gem_file)
+              end
+            end
+          end
+        end
+
+        gem_files
+      end
+    end
+
+    # Context passed to the erb templates
+    class BootstrapContext
+      attr_reader :gem_files
+
+      def initialize(gem_files)
+        @gem_files = gem_files
+      end
+
+      def install_gems_script
+        script = ""
+
+        @gem_files.each do |gem_file|
+          filename = File.basename(gem_file)
+          script << %Q[
+# #{filename}
+hadoop fs -copyToLocal #{gem_file} /home/hadoop/#{filename}
+gem install --local /home/hadoop/#{filename} --no-ri --no-rdoc
+          ]
+        end
+
+        script
+      end
+    end
+  end
+end

data/lib/pipely/deploy/client.rb

@@ -61,6 +61,8 @@ module Pipely
             @log.warn(error)
           end
         end
+
+        created_pipeline_id
       end
 
       def existing_pipelines(pipeline_name)

data/lib/pipely/tasks.rb

@@ -1,5 +1,6 @@
 require 'pipely/tasks/upload_steps'
 require 'pipely/tasks/deploy'
+require 'pipely/tasks/upload_pipeline_as_gem'
 require 'pipely/tasks/graph'
 require 'pipely/tasks/definition'
 

data/lib/pipely/tasks/upload_pipeline_as_gem.rb

@@ -0,0 +1,81 @@
+require 'rake'
+require 'aws'
+require 'pipely/deploy/bootstrap'
+
+module Pipely
+  module Tasks
+    class UploadPipelineAsGem < ::Rake::TaskLib
+      include ::Rake::DSL if defined?(::Rake::DSL)
+
+      # Name of this rake task
+      attr_accessor :name
+
+      attr_accessor :bucket_name
+      attr_accessor :s3_steps_path
+      attr_accessor :s3_gems_path
+      attr_accessor :config
+
+      def initialize(*args, &task_block)
+        setup_ivars(args)
+
+        task name, *args do |_, task_args|
+          RakeFileUtils.send(:verbose, verbose) do
+            if task_block
+              task_block.call(*[self, task_args].slice(0, task_block.arity))
+            end
+
+            run_task verbose
+          end
+        end
+
+        Rake::Task["upload_steps"].enhance [name]
+      end
+
+      def setup_ivars(args)
+        @name = args.shift || 'deploy:upload_pipeline_as_gem'
+        @verbose = true
+      end
+
+      def run_task(verbose)
+        context = build_bootstrap_context
+
+        Dir.glob("templates/*.erb").each do |erb_file|
+          upload_filename = File.basename(erb_file).sub( /\.erb$/, '' )
+
+          # Exclude the pipeline.json
+          if upload_filename == 'pipeline.json'
+            next
+          end
+
+          template_erb = Erubis::Eruby.new( File.read(erb_file) )
+          upload_to_s3( upload_filename, template_erb.result(context) )
+        end
+      end
+
+      private
+      def s3_bucket
+          s3 = AWS::S3.new
+          s3.buckets[@bucket_name]
+      end
+
+      def build_bootstrap_context
+        bootstrap_helper =
+          Pipely::Deploy::Bootstrap.new(s3_bucket, @s3_gems_path)
+        bootstrap_helper.build_and_upload_gems
+
+        # erb context
+        {
+          bootstrap: bootstrap_helper.context,
+          config: config
+        }
+      end
+
+      def upload_to_s3( upload_filename, body )
+
+        s3_dest = File.join(@s3_steps_path, upload_filename)
+        puts "uploading #{s3_dest}" if verbose
+        s3_bucket.objects[s3_dest].write(body)
+      end
+    end
+  end
+end

data/lib/pipely/tasks/upload_steps.rb

@@ -1,6 +1,5 @@
 require 'rake'
 require 'rake/tasklib'
-require 'fog'
 
 module Pipely
   module Tasks
@@ -57,11 +56,11 @@ module Pipely
       end
 
       def run_task(verbose)
-        with_bucket do |directory|
+        with_bucket do |bucket|
           step_files.each do |file_name|
             dest = file_name.sub(/^#{local_path}/, s3_path)
             puts "uploading #{dest}" if verbose
-            directory.files.create(key: dest, body: File.read(file_name))
+            bucket.objects[dest].write(File.read(file_name))
           end
         end
       end
@@ -69,9 +68,11 @@ module Pipely
     private
 
       def with_bucket
-        storage = Fog::Storage.new({ provider: 'AWS' })
-        if directory = storage.directories.detect{ |d| d.key == s3_bucket_name }
-          yield(directory)
+        s3 = AWS::S3.new
+        bucket = s3.buckets[s3_bucket_name]
+
+        if bucket.exists?
+          yield(bucket)
         else
           raise "Couldn't find S3 bucket '#{s3_bucket_name}'"
         end

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.6.2" unless defined?(::Pipely::VERSION)
+  VERSION = "0.7.0" unless defined?(::Pipely::VERSION)
 end

data/spec/fixtures/vcr_cassettes/build_and_upload_gems.yml

@@ -0,0 +1,47 @@
+---
+http_interactions:
+- request:
+    method: put
+    uri: https://a-test-bucket.s3.amazonaws.com/test_path/gems/pipely-0.7.0.gem
+    body:
+      encoding: UTF-8
+      string: !binary |-
+        bWV0YWRhdGEuZ3oAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+    headers:
+      Content-Type:
+      - ''
+      Accept-Encoding:
+      - ''
+      Content-Length:
+      - '21504'
+      User-Agent:
+      - aws-sdk-ruby/1.50.0 ruby/2.0.0 x86_64-darwin13.3.0
+      Date:
+      - Wed, 10 Sep 2014 00:58:35 GMT
+      Authorization:
+      - AWS 123
+      Accept:
+      - '*/*'
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Amz-Id-2:
+      - 123
+      X-Amz-Request-Id:
+      - 123
+      Date:
+      - Wed, 10 Sep 2014 00:58:36 GMT
+      Etag:
+      - '"123"'
+      Content-Length:
+      - '0'
+      Server:
+      - AmazonS3
+    body:
+      encoding: UTF-8
+      string: ''
+    http_version:
+  recorded_at: Wed, 10 Sep 2014 00:58:35 GMT
+recorded_with: VCR 2.9.2

data/spec/lib/pipely/deploy/bootstrap_spec.rb

@@ -0,0 +1,86 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'spec_helper'
+require 'pipely/deploy/bootstrap'
+require 'fog'
+require 'fileutils'
+
+module Pipely
+  describe Deploy::Bootstrap do
+    subject { described_class.new(s3_bucket, 'test_path/gems') }
+    let(:s3_bucket) do
+       s3 = AWS::S3.new
+       s3.buckets['a-test-bucket']
+    end
+
+    it "should have bucket name" do
+      expect(subject.bucket_name).to eql 'a-test-bucket'
+    end
+
+    it "should have a s3 gems path" do
+      expect(subject.s3_gems_path).to eql 'test_path/gems'
+    end
+
+    describe "#build_and_upload_gems" do
+      let(:build_and_upload_gems) do
+        VCR.use_cassette('build_and_upload_gems') do
+          subject.build_and_upload_gems
+        end
+      end
+
+      it "should create project gem" do
+        build_and_upload_gems
+
+        expect(File).to exist(subject.project_spec.file_name)
+      end
+
+      it "should upload gems" do
+        specs = Bundler.definition.specs_for([:default])
+
+        gems = specs.map(&:file_name).reject do |g|
+          # Ignore bundler, since it could be a system installed gem (travis)
+          # without a cache file
+          g.match(/^json-\d+\.\d+\.\d+\.gem$/) or
+            g.match(/^bundler-\d+\.\d+\.\d+\.gem$/)
+        end
+
+        objects = double(:objects)
+        s3_object = double('s3_object', write: nil)
+
+        allow(objects).to receive(:[]) { s3_object }
+
+        gems.each do |gem|
+          expect(objects).to receive(:[]).with(subject.gem_s3_path(gem))
+        end
+
+        expect(s3_bucket).to(receive(:objects)).
+          at_least(gems.size).times.
+          at_most(gems.size + 1).times. # allow for Bundler
+          and_return(objects)
+
+        build_and_upload_gems
+      end
+    end
+
+    describe "#context" do
+      let(:context) { subject.context}
+
+      before do
+        VCR.use_cassette('build_and_upload_gems') do
+          subject.build_and_upload_gems
+        end
+      end
+
+      it "should have gem_files" do
+        expect(context.gem_files).to_not be_nil
+      end
+
+      it "should be an s3 url" do
+        expect(context.gem_files.first).to(
+          match( /^s3:\/\/#{subject.bucket_name}\/#{subject.s3_gems_path}/ )
+        )
+      end
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -1,9 +1,31 @@
 require 'timecop'
+require 'aws-sdk'
 require 'fog'
+require 'rspec'
+require 'vcr'
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+
+AWS.config(
+  access_key_id: "xxx",
+  secret_access_key: "xxx"
+)
 
 Fog.credentials = {
   aws_access_key_id: "xxx",
   aws_secret_access_key: "xxx"
 }
 
-$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+VCR.configure do |c|
+  c.allow_http_connections_when_no_cassette = true
+  c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
+  c.hook_into :webmock
+end
+
+class WebMock::StubSocket
+  attr_accessor :continue_timeout, :read_timeout
+
+  def closed?
+    true
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-01 00:00:00.000000000 Z
+date: 2014-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: erubis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - matt@swipely.com
@@ -192,6 +234,7 @@ files:
 - lib/pipely/definition.rb
 - lib/pipely/dependency.rb
 - lib/pipely/deploy.rb
+- lib/pipely/deploy/bootstrap.rb
 - lib/pipely/deploy/client.rb
 - lib/pipely/fog_client.rb
 - lib/pipely/graph_builder.rb
@@ -205,8 +248,10 @@ files:
 - lib/pipely/tasks/definition.rb
 - lib/pipely/tasks/deploy.rb
 - lib/pipely/tasks/graph.rb
+- lib/pipely/tasks/upload_pipeline_as_gem.rb
 - lib/pipely/tasks/upload_steps.rb
 - lib/pipely/version.rb
+- spec/fixtures/vcr_cassettes/build_and_upload_gems.yml
 - spec/lib/pipely/build/daily_scheduler_spec.rb
 - spec/lib/pipely/build/right_now_scheduler_spec.rb
 - spec/lib/pipely/build/s3_path_builder_spec.rb
@@ -215,6 +260,7 @@ files:
 - spec/lib/pipely/component_spec.rb
 - spec/lib/pipely/definition_spec.rb
 - spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/deploy/bootstrap_spec.rb
 - spec/lib/pipely/deploy/client_spec.rb
 - spec/lib/pipely/graph_builder_spec.rb
 - spec/lib/pipely/reference_list_spec.rb
@@ -245,6 +291,7 @@ signing_key:
 specification_version: 4
 summary: Generate dependency graphs from pipeline definitions.
 test_files:
+- spec/fixtures/vcr_cassettes/build_and_upload_gems.yml
 - spec/lib/pipely/build/daily_scheduler_spec.rb
 - spec/lib/pipely/build/right_now_scheduler_spec.rb
 - spec/lib/pipely/build/s3_path_builder_spec.rb
@@ -253,6 +300,7 @@ test_files:
 - spec/lib/pipely/component_spec.rb
 - spec/lib/pipely/definition_spec.rb
 - spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/deploy/bootstrap_spec.rb
 - spec/lib/pipely/deploy/client_spec.rb
 - spec/lib/pipely/graph_builder_spec.rb
 - spec/lib/pipely/reference_list_spec.rb