shipitron 1.1.0 → 1.3.2

data/lib/shipitron/client/run_ecs_tasks.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shellwords'
 require 'base64'
@@ -16,6 +17,7 @@ module Shipitron
       required :shipitron_task
       required :repository_url
       required :s3_cache_bucket
+      required :build_cache_location
       required :image_name
       required :named_tag
       required :ecs_task_defs
@@ -23,9 +25,11 @@ module Shipitron
       optional :ecs_services
       optional :ecs_service_templates
       optional :build_script
+      optional :skip_push
       optional :post_builds
       optional :simulate
       optional :repository_branch
+      optional :registry
 
       before do
         context.post_builds ||= []
@@ -72,7 +76,7 @@ module Shipitron
               ]
             },
             count: 1,
-            started_by: 'shipitron'
+            started_by: Shipitron::Client::STARTED_BY
           )
 
           if !response.failures.empty?
@@ -108,6 +112,7 @@ module Shipitron
           '--name', context.application,
           '--repository', context.repository_url,
           '--bucket', context.s3_cache_bucket,
+          '--build-cache-location', context.build_cache_location,
           '--image-name', context.image_name,
           '--named-tag', context.named_tag,
           '--region', cluster.region,
@@ -123,10 +128,18 @@ module Shipitron
             ary.concat(context.ecs_services)
           end
 
+          if context.registry != nil
+            ary.concat ['--registry', context.registry]
+          end
+
           if context.build_script != nil
             ary.concat ['--build-script', context.build_script]
           end
 
+          if context.skip_push != nil
+            ary.concat ['--skip-push', context.skip_push.to_s]
+          end
+
           if !context.post_builds.empty?
             ary << '--post-builds'
             ary.concat(context.post_builds.map(&:to_s))

data/lib/shipitron/docker_image.rb

@@ -2,6 +2,7 @@ require 'shipitron'
 
 module Shipitron
   class DockerImage < Hashie::Dash
+    property :registry
     property :name
     property :tag
 
@@ -13,7 +14,9 @@ module Shipitron
         tag_str = tag_str.dup.prepend(':')
       end
 
-      "#{name}#{tag_str}"
+      name_with_registry = [registry, name].compact.join('/')
+
+      "#{name_with_registry}#{tag_str}"
     end
 
     def to_s

data/lib/shipitron/find_docker_volume_name.rb

@@ -0,0 +1,68 @@
+require 'excon'
+require 'json'
+
+module Shipitron
+  class FindDockerVolumeName
+    include Metaractor
+
+    required :container_name
+    required :volume_search
+
+    def call
+      volumes = container_volumes(container_name: container_name)
+
+      volume_metadata = volumes.find do |volume|
+        volume['DockerName'] =~ volume_search
+      end
+
+      if volume_metadata.nil?
+        raise 'Unable to find shipitron-home volume!'
+      end
+
+      context.volume_name = volume_metadata['DockerName']
+    end
+
+    private
+    def container_name
+      context.container_name
+    end
+
+    def volume_search
+      context.volume_search
+    end
+
+    def container_volumes(container_name:)
+      container_metadata = self.task_metadata['Containers'].find do |container|
+        container['Name'] == container_name
+      end
+
+      return {} if container_metadata.nil?
+
+      container_metadata['Volumes']
+    end
+
+    def task_metadata
+      return @task_metadata if defined?(@task_metadata)
+
+      begin
+        response = Excon.get(
+          "#{ENV['ECS_CONTAINER_METADATA_URI_V4']}/task",
+          expects: [200],
+          connect_timeout: 5,
+          read_timeout: 5,
+          write_timeout: 5,
+          tcp_nodelay: true
+        )
+
+        Logger.debug "Metadata result:"
+        Logger.debug(response.body)
+        Logger.debug "\n"
+
+        @task_metadata = JSON.parse(response.body)
+      rescue
+        Logger.info "Metadata uri failed"
+        {}
+      end
+    end
+  end
+end

data/lib/shipitron/git_info.rb

@@ -0,0 +1,57 @@
+require 'shipitron'
+require 'rugged'
+
+module Shipitron
+  class GitInfo < Hashie::Dash
+    property :sha
+    property :short_sha
+    property :email
+    property :name
+    property :summary
+    property :timestamp
+    property :branch
+    property :tag
+
+    def one_liner
+      "#{name} (#{short_sha}): #{summary}"
+    end
+
+    def self.from_path(path:)
+      repo = Rugged::Repository.new(path)
+      commit = repo.last_commit
+      self.new(
+        sha: commit.oid,
+        short_sha: commit.oid[0, 12],
+        email: commit.author.dig(:email),
+        name: commit.author.dig(:name),
+        summary: commit.summary,
+        timestamp: commit.epoch_time.to_s,
+        branch: branch_name(repo: repo),
+        tag: tag_name(repo: repo)
+      )
+    end
+
+    def self.branch_name(repo:)
+      ref = repo.head
+      ref.branch? ? ref.name.sub('refs/heads/', '') : nil
+    end
+
+    def self.tag_name(repo:)
+      ref = repo.head
+      tags = repo.tags
+
+      tags.each do |tag|
+        target_id =
+          if tag.annotated?
+            tag.annotation.target_id
+          else
+            tag.target.oid
+          end
+
+        return tag.name if ref.target_id == target_id
+      end
+
+      nil
+    end
+  end
+end

data/lib/shipitron/mustache_yaml_parser.rb

@@ -3,20 +3,24 @@ require 'yaml'
 
 module Shipitron
   class MustacheYamlParser
-    def initialize(context:nil, view:nil)
-      if (context.nil? && view.nil?) || (!context.nil? && !view.nil?)
+    def initialize(file_path, options = {})
+      @context = options[:context]
+      @view = options[:view]
+
+      if (@context.nil? && @view.nil?) || (!@context.nil? && !@view.nil?)
         raise ArgumentError, 'Either context or view required'
       end
 
-      @context = context
-      @view = view
-
+      @file_path = file_path.is_a?(Pathname) ? file_path.to_s : file_path
       @view ||= Mustache
     end
 
-    def perform(file_path)
-      file_path = file_path.is_a?(Pathname) ? file_path.to_s : file_path
-      YAML.load(@view.render(File.read(file_path), @context))
+    def perform
+      YAML.load(@view.render(File.read(@file_path), @context))
+    end
+
+    def self.perform(file_path, options = {})
+      new(file_path, options).perform
     end
   end
 end

data/lib/shipitron/s3_copy.rb

@@ -0,0 +1,46 @@
+require 'shipitron'
+require 'shipitron/find_docker_volume_name'
+
+module Shipitron
+  class S3Copy
+    include Metaractor
+
+    required :source
+    required :destination
+    required :region
+
+    def call
+      if ENV['FOG_LOCAL']
+        Logger.info `cp #{source.gsub('s3://', '/fog/')} #{destination.gsub('s3://', '/fog/')}`
+        if $? != 0
+          fail_with_error!(message: 'Failed to transfer to/from s3 (mocked).')
+        end
+      else
+        Logger.info "S3 Copy from #{source} to #{destination}"
+
+        shipitron_home_volume = FindDockerVolumeName.call!(
+          container_name: 'shipitron',
+          volume_search: /shipitron-home/
+        ).volume_name
+
+        Logger.info `docker run --rm -t -v #{shipitron_home_volume}:/home/shipitron -e AWS_CONTAINER_CREDENTIALS_RELATIVE_URI amazon/aws-cli:latest --region #{region} s3 cp #{source} #{destination} --quiet --only-show-errors`
+        if $? != 0
+          fail_with_error!(message: 'Failed to transfer to/from s3.')
+        end
+      end
+    end
+
+    private
+    def source
+      context.source
+    end
+
+    def destination
+      context.destination
+    end
+
+    def region
+      context.region
+    end
+  end
+end

data/lib/shipitron/server/deploy_application.rb

@@ -19,6 +19,7 @@ module Shipitron
       required :application
       required :repository_url
       required :s3_cache_bucket
+      required :build_cache_location
       required :docker_image
       required :named_tag
       required :region
@@ -30,6 +31,8 @@ module Shipitron
       optional :build_script
       optional :post_builds
       optional :repository_branch
+      optional :skip_push, default: false
+      optional :registry
 
       around do |interactor|
         if ENV['CONSUL_HOST'].nil?

data/lib/shipitron/server/docker/build_image.rb

@@ -12,7 +12,10 @@ module Shipitron
 
         required :application
         required :docker_image
-        required :git_sha
+        required :git_info
+        required :named_tag
+        required :region
+        optional :registry
 
         organize [
           DownloadBuildCache,

data/lib/shipitron/server/docker/configure.rb

@@ -1,5 +1,6 @@
 require 'shipitron'
 require 'shipitron/consul_keys'
+require 'json'
 
 module Shipitron
   module Server
@@ -9,22 +10,48 @@ module Shipitron
         include ConsulKeys
 
         required :application
+        optional :registry
 
         before do
           configure_consul_client!
         end
 
         def call
-          docker_auth = begin
-                          key = fetch_key(key: "shipitron/#{application}/docker_auth")
-                          key = fetch_key!(key: 'shipitron/docker_auth') if key.nil?
-                          key
-                        end
-          auth_file = Pathname.new('/home/shipitron/.docker/config.json')
-          auth_file.parent.mkpath
-          auth_file.open('wb') do |file|
-            file.puts(docker_auth.to_s)
-            file.chmod(0600)
+          username = fetch_scoped_key('docker_user')
+          password = fetch_scoped_key('docker_password')
+
+          if username && password
+            Logger.info `docker login --username #{username} --password #{password}`
+            if $? != 0
+              fail_with_error!(message: 'Docker login failed.')
+            end
+          end
+
+          if registry
+            case registry
+            when /docker\.io/
+              # do nothing
+            when /\d+\.dkr\.ecr\.us-east-1\.amazonaws\.com/
+              # ECR
+              config_file = Pathname.new('/home/shipitron/.docker/config.json')
+              config_file.parent.mkpath
+
+              config_hash = {}
+              if config_file.file?
+                config_file.open('rb') do |file|
+                  json = file.read
+                  config_hash = JSON.parse(json) rescue {}
+                end
+              end
+
+              config_hash['credHelpers'] ||= {}
+              config_hash['credHelpers'][registry] = 'ecr-login'
+
+              config_file.open('wb') do |file|
+                file.puts(JSON.generate(config_hash))
+                file.chmod(0600)
+              end
+            end
           end
         end
 
@@ -33,6 +60,15 @@ module Shipitron
           context.application
         end
 
+        def registry
+          context.registry
+        end
+
+        def fetch_scoped_key(key)
+          value = fetch_key(key: "shipitron/#{application}/#{key}")
+          value = fetch_key(key: "shipitron/#{key}") if value.nil?
+          value
+        end
       end
     end
   end

data/lib/shipitron/server/docker/push_image.rb

@@ -8,8 +8,11 @@ module Shipitron
 
         required :docker_image
         required :named_tag
+        optional :skip_push, default: false
 
         def call
+          return if context.skip_push
+
           Logger.info "Pushing docker image #{docker_image} and #{docker_image.name_with_tag(named_tag)}"
 
           Logger.info `docker tag #{docker_image} #{docker_image.name_with_tag(named_tag)}`

data/lib/shipitron/server/docker/run_build_script.rb

@@ -9,17 +9,16 @@ module Shipitron
 
         required :application
         required :docker_image
-        required :git_sha
-        optional :build_script
-
-        before do
-          context.build_script ||= 'shipitron/build.sh'
-        end
+        required :git_info
+        required :named_tag
+        optional :build_script, default: 'shipitron/build.sh'
+        optional :registry
 
         def call
           Logger.info 'Building docker image'
 
-          docker_image.tag = git_sha
+          docker_image.registry = registry if registry != nil
+          docker_image.tag = git_info.short_sha
 
           FileUtils.cd("/home/shipitron/#{application}") do
             unless Pathname.new(build_script).exist?
@@ -27,7 +26,7 @@ module Shipitron
             end
 
             cmd = TTY::Command.new
-            result = cmd.run!("#{build_script} #{docker_image}")
+            result = cmd.run!("#{build_script} #{docker_image} #{named_tag}")
 
             if result.failure?
               fail_with_error!(message: "build script exited with non-zero code: #{result.exit_status}")
@@ -44,13 +43,21 @@ module Shipitron
           context.docker_image
         end
 
-        def git_sha
-          context.git_sha
+        def git_info
+          context.git_info
+        end
+
+        def named_tag
+          context.named_tag
         end
 
         def build_script
           context.build_script
         end
+
+        def registry
+          context.registry
+        end
       end
     end
   end