dockly 3.3.0 → 4.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODI3Y2E3MmU4NzIzNGM3Y2Q5MzA5MzQ0MTU0MmM5OTA5N2VjZTc3Mg==
-  data.tar.gz: !binary |-
-    M2RkM2IzMjJlNzhiMGMwZTdjNGI2NDIzYTliMDEyOGRhNThkOTQ3ZQ==
+SHA1:
+  metadata.gz: 817b3a75ce89c73e9ad40e97913c7e694f6d5d39
+  data.tar.gz: 7a0a50ba3a0c833f77deefda46eace5917b0b1d2
 SHA512:
-  metadata.gz: !binary |-
-    OTBmZjQ3OWUwOTU2N2RkYjk4ODYyY2M2ZmQ2ZDM5YmUzNjhkNGUwNWUzNmQ1
-    MWNlYmE4ZjlmZmYyMTczZDQ1MTRmODhhNjBhNjc3YzA2Nzg4NDdkODMwOWYw
-    ZjgyOTMxZWRjZDc4YmU2YzdiZmIxZTc3Nzg1Zjg1YWI4NDVmYjc=
-  data.tar.gz: !binary |-
-    YTZjYjU2OTYzZDc2ZmViM2IyNmUwZTE4NzQ3ZmYyNmM4OGQwMWM1NDc4NGE3
-    N2FjZjIzZjNiYWI4ZGNiNTJlODg3MmRjMjRjZTcxMDcyMTQwODg1OGQ4MjYz
-    ZWI4NWQ4NDNhMmJlYTFjNDA3ODdjNGIyOTFmMjI5ZGM1ZWUwNzg=
+  metadata.gz: d3e29e2b40e5dede34e76d19a7635b2e8d051fc799b41961871a5fee4e2011c1616954518aef46cae0f591561452180816e786aa0a919a9b0713c9c94b6c61a3
+  data.tar.gz: d4f4dc7bd97cbca3071ef9099ae8150b77b826daa4ba62c999d88ddb15dd62f175523f0c802d0e949d361c933179c987cb887fe1066917a8ad10f9387ed5bbef

data/.ruby-version

@@ -1 +1 @@
-1.9.3-p547
+2.3.1

data/.travis.yml

@@ -1,14 +1,13 @@
 language: ruby
 rvm:
-    - 1.9
-    - 2.0
-    - 2.1
-    - 2.2
+  - 2.0
+  - 2.1
+  - 2.2
+  - 2.3
 script:
   - git fetch --tags
   - CI=true bundle exec rake
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -qq rpm
-  - gem install bundler
-
+  - gem install bundler -v 1.17.3

data/README.md

@@ -174,6 +174,11 @@ In addition to the above attributes, `docker` has the following references:
     - allows one
     - class: `Dockly::Docker::Registry`
     - description: a registry to push to in lieu of exporting as a tar -- the registry will be automatically pulled upon installing the package
+- `ecr`
+    - required: `false`
+    - allows one
+    - class: `Dockly::Docker::ECR`
+    - description: an AWS ECR Docker registry to push to in lieu of exporting as a tar -- the registry will be automatically pulled upon installing the package
 
 Need finer control of Docker packages? We also wrote [docker-api](https://github.com/swipely/docker-api).
 
@@ -200,6 +205,23 @@ The `registry` DSL is used to define Docker Registries. It has the following att
     - default: `https://index.docker.io/v1`
     - description: the server where the registry lives
 
+`ecr`
+--------
+
+The `ecr` DSL is used to define AWS ECR Docker registries. It has the following attributes:
+
+- `server_address`
+    - required: `true`
+    - description: the server where the registry lives without the repo name, eg `accountid.dkr.ecr.region.amazonaws.com`
+- `username`
+    - required: `false`
+    - default: pulled from auth data obtained using assumed role's credentials
+    - description: the username to authenticate
+- `password`
+    - required: `false`
+    - default: pulled from auth data obtained using assumed role's credentials
+    - description: the password to authenticate
+
 `foreman`
 ---------
 

data/dockly.gemspec

@@ -18,14 +18,17 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'docker-api', '>= 1.14.0'
   gem.add_dependency 'dockly-util', '>= 0.0.9', '< 1.0'
   gem.add_dependency 'excon'
-  gem.add_dependency 'aws-sdk', '~> 2.0'
+  gem.add_dependency 'aws-sdk-core', '~> 3'
+  gem.add_dependency 'aws-sdk-s3', '~> 1'
+  gem.add_dependency 'aws-sdk-ecr', '~> 1'
   gem.add_dependency 'foreman'
   gem.add_dependency 'fpm', '~> 1.2.0'
   gem.add_dependency 'minigit', '~> 0.0.4'
   gem.add_development_dependency 'cane'
   gem.add_development_dependency 'pry'
-  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rake', '< 11.0'
   gem.add_development_dependency 'rspec', '~> 2.14.1'
   gem.add_development_dependency 'vcr'
-  gem.add_development_dependency 'webmock'
+  gem.add_development_dependency 'webmock', '<= 1.18.0'
+  gem.add_development_dependency 'addressable', '<= 2.3.6'
 end

data/lib/dockly.rb

@@ -4,7 +4,9 @@ require 'dockly/util/git'
 require 'foreman/cli_fix'
 require 'foreman/export/base_fix'
 require 'minigit'
-require 'aws-sdk'
+require 'aws-sdk-core'
+require 'aws-sdk-s3'
+require 'aws-sdk-ecr'
 require 'open3'
 
 module Dockly

data/lib/dockly/bash_builder.rb

@@ -24,7 +24,8 @@ module Dockly
     generate_snippet_for :s3_docker_import, [:s3_url, :repo, :tag]
     generate_snippet_for :s3_diff_docker_import, [:base_image, :diff_image, :repo, :tag]
     generate_snippet_for :registry_import, [:repo, :tag], { :tag => "latest" }
+    generate_snippet_for :auth_ecr, [:server_address]
 
-    generate_snippet_for :docker_tag_latest, [:repo, :tag]
+    generate_snippet_for :docker_tag_latest, [:repo, :tag, :new_name]
   end
 end

data/lib/dockly/build_cache/docker.rb

@@ -77,7 +77,7 @@ class Dockly::BuildCache::Docker < Dockly::BuildCache::Base
     debug 'Restarting the container to copy the cache\'s output'
     # Restart the container so we can copy its output
     container = container.commit.run('sleep 3600')
-    container.copy(output_directory) { |chunk| file.write(chunk.to_s) }
+    container.archive_out(output_directory) { |chunk| file.write(chunk.to_s) }
     container.kill
     file.tap(&:rewind)
   end

data/lib/dockly/deb.rb

@@ -77,7 +77,6 @@ class Dockly::Deb
   end
 
   def upload_to_s3
-    info "uploading to s3"
     return if s3_bucket.nil?
     raise "Package wasn't created!" unless File.exist?(build_path)
     info "uploading package to s3"
@@ -126,7 +125,14 @@ private
     add_files(@dir_package)
     add_docker_auth_config(@dir_package)
     add_docker(@dir_package)
-    add_startup_script(@dir_package) if package_startup_script
+
+    if package_startup_script.is_a?(String)
+      raise ArgumentError,
+            'package_startup_script filename must not be empty if specified' if package_startup_script.empty?
+      add_startup_script(@dir_package, package_startup_script)
+    elsif package_startup_script
+      add_startup_script(@dir_package)
+    end
 
     convert_package
 
@@ -174,7 +180,7 @@ private
   end
 
   def add_docker_auth_config(package)
-    return if (registry = get_registry).nil? || !registry.authentication_required?
+    return if (registry = get_registry).nil? || !should_auth?(registry)
     info "adding docker config file"
     registry.generate_config_file!
 
@@ -185,11 +191,18 @@ private
     package.attributes[:prefix] = nil
   end
 
+  def should_auth?(registry)
+    registry.is_a?(Dockly::Docker::Registry) &&
+      registry.authentication_required?
+  end
+
   def add_docker(package)
     return if docker.nil? || docker.s3_bucket
     info "adding docker image"
     docker.generate!
-    return unless docker.registry.nil?
+
+    return if get_registry
+
     package.attributes[:prefix] = docker.package_dir
     Dir.chdir(File.dirname(docker.tar_path)) do
       package.input(File.basename(docker.tar_path))
@@ -198,7 +211,7 @@ private
   end
 
   def get_registry
-    if docker && registry = docker.registry
+    if docker && (registry = docker.registry)
       registry
     end
   end
@@ -207,38 +220,53 @@ private
     scripts = ["#!/bin/bash"]
     bb = Dockly::BashBuilder.new
     scripts << bb.normalize_for_dockly
-    if get_registry
-      scripts << bb.registry_import(docker.repo, docker.tag)
-    elsif docker
-      if docker.s3_bucket.nil?
-        docker_output = File.join(docker.package_dir, docker.export_filename)
-        if docker.tar_diff
-          scripts << bb.file_diff_docker_import(docker.import, docker_output, docker.name, docker.tag)
-        else
-          scripts << bb.file_docker_import(docker_output, docker.name, docker.tag)
+
+    if docker
+      if (registry = docker.registry)
+        if registry.is_a?(Dockly::Docker::ECR)
+          scripts << bb.auth_ecr(registry.server_address)
         end
+
+        scripts << bb.registry_import(docker.repo, docker.tag)
+        scripts << bb.docker_tag_latest(docker.repo, docker.tag, docker.name)
       else
-        if docker.tar_diff
-          scripts << bb.s3_diff_docker_import(docker.import, docker.s3_url, docker.name, docker.tag)
-        else
-          scripts << bb.s3_docker_import(docker.s3_url, docker.name, docker.tag)
-        end
+        scripts += collect_non_registry_scripts(bb)
       end
-      scripts << bb.docker_tag_latest(docker.repo, docker.tag)
     end
     scripts.join("\n")
   end
 
-  def add_startup_script(package)
+  def collect_non_registry_scripts(bb)
+    scripts = []
+
+    if docker.s3_bucket.nil?
+      docker_output = File.join(docker.package_dir, docker.export_filename)
+      if docker.tar_diff
+        scripts << bb.file_diff_docker_import(docker.import, docker_output, docker.name, docker.tag)
+      else
+        scripts << bb.file_docker_import(docker_output, docker.name, docker.tag)
+      end
+    else
+      if docker.tar_diff
+        scripts << bb.s3_diff_docker_import(docker.import, docker.s3_url, docker.name, docker.tag)
+      else
+        scripts << bb.s3_docker_import(docker.s3_url, docker.name, docker.tag)
+      end
+    end
+
+    scripts << bb.docker_tag_latest(docker.repo, docker.tag, docker.repo)
+  end
+
+  def add_startup_script(package, startup_script = "dockly-startup.sh")
     ensure_present! :build_dir
-    startup_script_path = File.join(build_dir, "dockly-startup.sh")
+    startup_script_path = File.join(build_dir, startup_script)
     File.open(startup_script_path, 'w+') do |f|
       f.write(post_startup_script)
       f.chmod(0755)
     end
     package.attributes[:prefix] = "/opt/dockly"
     Dir.chdir(build_dir) do
-      package.input("dockly-startup.sh")
+      package.input(startup_script)
     end
     package.attributes[:prefix] = nil
   end

data/lib/dockly/docker.rb

@@ -10,11 +10,13 @@ class Dockly::Docker
   include Dockly::Util::Logger::Mixin
 
   autoload :Registry, 'dockly/docker/registry'
+  autoload :ECR, 'dockly/docker/ecr'
 
   logger_prefix '[dockly docker]'
 
   dsl_class_attribute :build_cache, Dockly::BuildCache.model, type: Array
-  dsl_class_attribute :registry, Dockly::Docker::Registry
+  dsl_class_attribute :docker_registry, Dockly::Docker::Registry
+  dsl_class_attribute :ecr, Dockly::Docker::ECR
 
   dsl_attribute :name, :import, :git_archive, :build, :tag, :build_dir, :package_dir,
     :timeout, :cleanup_images, :tar_diff, :s3_bucket, :s3_object_prefix
@@ -32,6 +34,10 @@ class Dockly::Docker
     (@build_env ||= {}).tap { |env| env.merge!(hash) if hash.is_a?(Hash) }
   end
 
+  def registry
+    ecr || docker_registry
+  end
+
   def copy_from_s3(sha)
     return if s3_bucket.nil?
     object = s3_object_for(sha)
@@ -177,7 +183,9 @@ class Dockly::Docker
     repo = "#{name}-base"
     tag = "dockly-#{Dockly::VERSION}-#{File.basename(import).split('.').first}"
     info "looking for imported base image with tag: #{tag}"
-    image = Docker::Image.all.find { |img| img.info['RepoTags'].include?("#{repo}:#{tag}") }
+    image = Docker::Image.all.find do |img|
+      img.info['RepoTags'] && img.info['RepoTags'].include?("#{repo}:#{tag}")
+    end
     if image
       info "found imported base image: #{image.id}"
       image
@@ -316,14 +324,26 @@ class Dockly::Docker
   end
 
   def push_to_registry(image)
-    ensure_present! :registry
+    raise "No registry to push to" if registry.nil?
+
     info "Exporting #{image.id} to Docker registry at #{registry.server_address}"
+
     registry.authenticate!
-    image = Docker::Image.all(:all => true).find { |img|
-      img.id.start_with?(image.id) || image.id.start_with?(img.id)
-    }
+
+    image =
+      Docker::Image
+        .all(:all => true)
+        .find do |img|
+          img.id.include?(image.id) || image.id.include?(img.id)
+        end
+
     raise "Could not find image after authentication" if image.nil?
-    image.push(registry.to_h, :registry => registry.server_address)
+
+    image.push(registry.to_h, :registry => registry.server_address) do |resp|
+      if resp.include?('errorDetail')
+        raise "Error pushing to registry: #{resp}"
+      end
+    end
   end
 
   def fetch_import

data/lib/dockly/docker/ecr.rb

@@ -0,0 +1,60 @@
+class Dockly::Docker::ECR
+  include Dockly::Util::DSL
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[dockly docker ecr]'
+
+  dsl_attribute :name, :server_address, :password, :username
+
+  def authenticate!
+    @username ||= login_from_aws[0]
+    @password ||= login_from_aws[1]
+
+    ensure_present! :password, :server_address, :username
+
+    debug "Attempting to authenticate at #{server_address}"
+
+    ::Docker.authenticate!(self.to_h)
+
+    info "Successfully authenticated at #{server_address}"
+  rescue ::Docker::Error::AuthenticationError
+    raise "Could not authenticate at #{server_address}"
+  end
+
+  def authentication_required?
+    true
+  end
+
+  def default_server_address?
+    false
+  end
+
+  def login_from_aws
+    @login_from_aws ||=
+      Base64
+        .decode64(auth_data.authorization_token)
+        .split(':')
+  end
+
+  def auth_data
+    @auth_data ||=
+      client
+        .get_authorization_token
+        .authorization_data
+        .first
+  end
+
+  def client
+    @client ||= Aws::ECR::Client.new(region: 'us-east-1')
+  end
+
+  def to_h
+    ensure_present! :username, :password, :server_address
+
+    {
+      'serveraddress' => "https://#{server_address}",
+      'username' => username,
+      'password' => password
+    }
+  end
+end

data/lib/dockly/version.rb

@@ -1,6 +1,6 @@
 module Dockly
-  MAJOR = 3
-  MINOR = 3
+  MAJOR = 4
+  MINOR = 2
   PATCH = 0
   RELEASE = nil
 

data/snippets/auth_ecr.erb

@@ -0,0 +1,11 @@
+auth_ecr() {
+  $(aws ecr get-login --region us-east-1 --no-include-email)
+}
+
+worked=1
+for attempt in {1..10}; do
+  [[ $worked != 0 ]] || break
+  auth_ecr && worked=0 || (log "ecr auth: attempt $attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "failed to auth to ecr"
+log "ecr auth: successfully authenticated"

data/snippets/docker_tag_latest.erb

@@ -1,3 +1,3 @@
 <% if data[:tag] %>
-docker tag <%= data[:repo] %>:<%= data[:tag] %> <%= data[:repo] %>:latest
+docker tag <%= data[:repo] %>:<%= data[:tag] %> <%= data[:new_name] %>:latest
 <% end %>

data/spec/dockly/bash_builder_spec.rb

@@ -32,8 +32,8 @@ describe Dockly::BashBuilder do
 
     context "when there is a tag" do
       it "tags the repo:tag as repo:latest" do
-        output = subject.docker_tag_latest("test_repo", "a_tag")
-        expect(output).to include("docker tag test_repo:a_tag test_repo:latest")
+        output = subject.docker_tag_latest("registry/test_repo", "a_tag", "test_repo")
+        expect(output).to include("docker tag registry/test_repo:a_tag test_repo:latest")
       end
     end
   end

data/spec/dockly/build_cache/base_spec.rb

@@ -14,8 +14,11 @@ describe Dockly::BuildCache::Base do
   describe '#up_to_date?' do
     context 'when the object exists in s3' do
       before do
-        allow(subject.connection)
-          .to receive(:head_object)
+        allow(Dockly)
+          .to receive(:s3)
+          .and_return(
+            Aws::S3::Client.new(stub_responses: true)
+          )
       end
 
       its(:up_to_date?) { should be_true }
@@ -23,9 +26,13 @@ describe Dockly::BuildCache::Base do
 
     context 'when the object does not exist in s3' do
       before do
-        allow(subject.connection)
-          .to receive(:head_object)
-          .and_raise(Aws::S3::Errors::NoSuchKey.new('Some Error', 500))
+        allow(Dockly)
+          .to receive(:s3)
+          .and_return(
+            Aws::S3::Client.new(
+              stub_responses: { :head_object => 'NoSuchKey' }
+            )
+          )
       end
 
       its(:up_to_date?) { should be_false }
@@ -37,9 +44,12 @@ describe Dockly::BuildCache::Base do
     let(:object) { double(:object) }
 
     before do
-      allow(subject.connection)
-        .to receive(:get_object)
-        .and_return(object)
+      allow(Dockly)
+        .to receive(:s3)
+        .and_return(Aws::S3::Client.new(
+          stub_responses: { :get_object => object }
+        ))
+
       allow(object)
         .to receive(:body)
         .and_return(StringIO.new('hey dad').tap(&:rewind))