dockly 2.7.2 → 3.0.0

data/Gemfile

@@ -4,6 +4,3 @@ source 'https://rubygems.org'
 gem 'foreman', :git => 'https://github.com/adamjt/foreman'
 
 gemspec
-
-gem 'fog-core'
-gem 'fog-json'

data/dockly.gemspec

@@ -18,10 +18,9 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'docker-api', '>= 1.14.0'
   gem.add_dependency 'dockly-util', '~> 0.0.9'
   gem.add_dependency 'excon'
-  gem.add_dependency 'fog', '~> 1.21'
+  gem.add_dependency 'aws-sdk', '~> 2.0'
   gem.add_dependency 'foreman'
   gem.add_dependency 'fpm', '~> 1.2.0'
-  gem.add_dependency 'grit'
   gem.add_dependency 'rugged'
   gem.add_development_dependency 'cane'
   gem.add_development_dependency 'pry'

data/lib/dockly.rb

@@ -1,16 +1,17 @@
 require 'dockly/util'
 require 'dockly/util/tar'
 require 'dockly/util/git'
-require 'fog'
 require 'foreman/cli_fix'
 require 'foreman/export/base_fix'
 require 'rugged'
+require 'aws-sdk'
 
 module Dockly
+  LOAD_FILE = 'dockly.rb'
+
   attr_reader :instance, :git_sha
   attr_writer :load_file
 
-  autoload :AWS, 'dockly/aws'
   autoload :Foreman, 'dockly/foreman'
   autoload :BashBuilder, 'dockly/bash_builder'
   autoload :BuildCache, 'dockly/build_cache'
@@ -18,10 +19,11 @@ module Dockly
   autoload :Deb, 'dockly/deb'
   autoload :History, 'dockly/history'
   autoload :Rpm, 'dockly/rpm'
+  autoload :S3Writer, 'dockly/s3_writer'
   autoload :TarDiff, 'dockly/tar_diff'
   autoload :VERSION, 'dockly/version'
 
-  LOAD_FILE = 'dockly.rb'
+  module_function
 
   def load_file
     @load_file || LOAD_FILE
@@ -48,6 +50,41 @@ module Dockly
     }
   end
 
+  def git_sha
+    @git_sha ||= Dockly::Util::Git.sha
+  end
+
+  def assume_role(role_name = nil)
+    @assume_role = role_name if role_name
+    @assume_role
+  end
+
+  def perform_role_assumption
+    return if assume_role.nil?
+    Aws.config.update(
+      credentials: Aws::AssumeRoleCredentials.new(
+        role_arn: assume_role, role_session_name: 'dockly'
+      )
+    )
+  end
+
+  def aws_region(region = nil)
+    @aws_region = region unless region.nil?
+    @aws_region || 'us-east-1'
+  end
+
+  def s3
+    @s3 ||= Aws::S3::Client.new(region: aws_region)
+  end
+
+  [:debs, :rpms, :dockers, :foremans].each do |method|
+    define_method(method) do
+      inst[method]
+    end
+
+    module_function method
+  end
+
   {
     :deb => Dockly::Deb,
     :rpm => Dockly::Rpm,
@@ -61,21 +98,9 @@ module Dockly
         klass.new!(:name => sym, &block)
       end
     end
-  end
 
-  [:debs, :rpms, :dockers, :foremans].each do |method|
-    define_method(method) do
-      inst[method]
-    end
+    module_function method
   end
-
-  def git_sha
-    @git_sha ||= Dockly::Util::Git.git_sha
-  end
-
-  module_function :inst, :load_inst, :setup, :load_file, :load_file=,
-                  :deb,  :rpm,  :docker,  :foreman, :git_sha,
-                  :debs, :rpms, :dockers, :foremans
 end
 
 require 'dockly/rake_task'

data/lib/dockly/build_cache/base.rb

@@ -48,9 +48,9 @@ class Dockly::BuildCache::Base
 
   def up_to_date?
     ensure_present! :s3_bucket, :s3_object_prefix
-    connection.head_object(s3_bucket, s3_object(hash_output))
+    connection.head_object(bucket: s3_bucket, key: s3_object(hash_output))
     true
-  rescue Excon::Errors::NotFound
+  rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchKey
     false
   end
 
@@ -63,17 +63,17 @@ class Dockly::BuildCache::Base
     FileUtils.mkdir_p(File.dirname(file_path))
     unless File.exist?(file_path)
       debug 'Pulling build cache from s3'
-      object = connection.get_object(s3_bucket, file_name)
+      object = connection.get_object(bucket: s3_bucket, key: file_name)
       debug 'Pulled build cache from s3'
 
       file = File.open(file_path, 'w+b')
-      file.write(object.body)
+      file.write(object.body.read)
       file.tap(&:rewind)
     else
       info 'Build cache already exists locally'
       File.open(file_path, 'rb')
     end
-  rescue Excon::Errors::NotFound
+  rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchKey
     nil
   end
 
@@ -89,8 +89,12 @@ class Dockly::BuildCache::Base
 
   def push_to_s3(file)
     ensure_present! :s3_bucket, :s3_object_prefix
-    connection.put_object(s3_bucket, s3_object(hash_output), file.read)
-    connection.copy_object(s3_bucket, s3_object(hash_output), s3_bucket, s3_object("latest"))
+    connection.put_object(bucket: s3_bucket, key: s3_object(hash_output), body: file)
+    connection.copy_object(
+      copy_source: [s3_bucket, s3_object(hash_output)].join('/'),
+      bucket: s3_bucket,
+      key: s3_object('latest')
+    )
   end
 
   def file_output(file)
@@ -118,6 +122,6 @@ class Dockly::BuildCache::Base
   end
 
   def connection
-    Dockly::AWS.s3
+    Dockly.s3
   end
 end

data/lib/dockly/deb.rb

@@ -52,7 +52,11 @@ class Dockly::Deb
     ensure_present! :s3_bucket
     object = s3_object_name_for(sha)
     info "Copying s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object_name}"
-    Dockly::AWS.s3.copy_object(s3_bucket, object, s3_bucket, s3_object_name)
+    Dockly.s3.copy_object(
+      copy_source: File.join(s3_bucket, object),
+      bucket: s3_bucket,
+      key: s3_object_name
+    )
     info "Successfully copied s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object_name}"
   end
 
@@ -63,7 +67,7 @@ class Dockly::Deb
 
   def exists?
     debug "#{name}: checking for package: #{s3_url}"
-    Dockly::AWS.s3.head_object(s3_bucket, s3_object_name)
+    Dockly.s3.head_object(bucket: s3_bucket, key: s3_object_name)
     info "#{name}: found package: #{s3_url}"
     true
   rescue
@@ -76,8 +80,9 @@ class Dockly::Deb
     return if s3_bucket.nil?
     raise "Package wasn't created!" unless File.exist?(build_path)
     info "uploading package to s3"
-    Dockly::AWS.s3.put_bucket(s3_bucket) rescue nil
-    Dockly::AWS.s3.put_object(s3_bucket, s3_object_name, File.new(build_path))
+    File.open(build_path, 'rb') do |file|
+      Dockly.s3.put_object(bucket: s3_bucket, key: s3_object_name, body: file)
+    end
   end
 
   def s3_url
@@ -85,7 +90,7 @@ class Dockly::Deb
   end
 
   def s3_object_name
-    s3_object_name_for(Dockly::Util::Git.git_sha)
+    s3_object_name_for(Dockly::Util::Git.sha)
   end
 
   def s3_object_name_for(sha)

data/lib/dockly/docker.rb

@@ -36,7 +36,11 @@ class Dockly::Docker
     return if s3_bucket.nil?
     object = s3_object_for(sha)
     info "Copying s3://#{s3_bucket}/#{object} to #{s3_bucket}/#{s3_object}"
-    Dockly::AWS.s3.copy_object(s3_bucket, object, s3_bucket, s3_object)
+    Dockly.s3.copy_object(
+      copy_source: File.join(s3_bucket, object),
+      bucket: s3_bucket,
+      key: s3_object
+    )
     info "Successfully copied s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object}"
   end
 
@@ -147,11 +151,11 @@ class Dockly::Docker
 
     FileUtils.rm_rf(git_archive_dir)
     FileUtils.mkdir_p(git_archive_dir)
-    info "archiving #{Dockly::Util::Git.git_sha}"
-    Grit::Git.with_timeout(120) do
-      Dockly::Util::Git.git_repo.archive_to_file(Dockly::Util::Git.git_sha, prefix, git_archive_path, 'tar', 'cat')
+    info "archiving #{Dockly::Util::Git.sha}"
+    File.open(git_archive_path, 'wb') do |file|
+      Dockly::Util::Git.archive(Dockly::Util::Git.sha, prefix, file)
     end
-    info "made the git archive for sha #{Dockly::Util::Git.git_sha}"
+    info "made the git archive for sha #{Dockly::Util::Git.sha}"
     git_archive_path
   end
 
@@ -240,7 +244,7 @@ class Dockly::Docker
       if s3_bucket.nil?
         output = File.open(tar_path, 'wb')
       else
-        output = Dockly::AWS::S3Writer.new(connection, s3_bucket, s3_object)
+        output = Dockly::S3Writer.new(Dockly.s3, s3_bucket, s3_object)
       end
 
       gzip_output = Zlib::GzipWriter.new(output)
@@ -302,7 +306,7 @@ class Dockly::Docker
   end
 
   def s3_object
-    s3_object_for(Dockly::Util::Git.git_sha)
+    s3_object_for(Dockly::Util::Git.sha)
   end
 
   def s3_object_for(sha)
@@ -330,9 +334,9 @@ class Dockly::Docker
       debug "fetching #{import}"
       File.open("#{path}.tmp", 'wb') do |file|
         case import
-          when /^s3:\/\/(?<bucket_name>.+?)\/(?<object_path>.+)$/
-            connection.get_object(Regexp.last_match[:bucket_name],
-                                  Regexp.last_match[:object_path]) do |chunk, remaining, total|
+          when /^s3:\/\/(?<bucket>.+?)\/(?<key>.+)$/
+            bucket, key = Regexp.last_match[:bucket], Regexp.last_match[:key]
+            Dockly.s3.get_object(bucket: bucket, key: key) do |chunk|
               file.write(chunk)
             end
           when /^https?:\/\//
@@ -351,21 +355,15 @@ class Dockly::Docker
   def exists?
     return false unless s3_bucket
     debug "#{name}: checking for package: #{s3_url}"
-    Dockly::AWS.s3.head_object(s3_bucket, s3_object)
+    Dockly.s3.head_object(bucket: s3_bucket, key: s3_object)
     info "#{name}: found package: #{s3_url}"
     true
-  rescue
-    info "#{name}: could not find package: " +
-         "#{s3_url}"
+  rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchKey
+    info "#{name}: could not find package: #{s3_url}"
     false
   end
 
   def repository(value = nil)
     name(value)
   end
-
-private
-  def connection
-    Dockly::AWS.s3
-  end
 end

data/lib/dockly/rake_task.rb

@@ -1,172 +1,167 @@
 require 'rake'
 require 'dockly'
 
-class Rake::DebTask < Rake::Task
-  def needed?
-    raise "Package does not exist" if package.nil?
-    !!ENV['FORCE'] || !package.exists?
-  end
+module Dockly::RakeHelper
+  module_function
 
-  def package
-    Dockly::Deb[name.split(':').last.to_sym]
+  def find_deb!(name)
+    deb = Dockly.debs[name]
+    raise "No deb named #{name}" if deb.nil?
+    deb
   end
-end
 
-class Rake::RpmTask < Rake::Task
-  def needed?
-    raise "Package does not exist" if package.nil?
-    !!ENV['FORCE'] || !package.exists?
+  def find_docker!(name)
+    docker = Dockly.dockers[name]
+    raise "No docker named #{name}" if docker.nil?
+    docker
   end
 
-  def package
-    Dockly::Rpm[name.split(':').last.to_sym]
+  def find_rpm!(name)
+    rpm = Dockly.rpms[name]
+    raise "No rpm named #{name}" if rpm.nil?
+    rpm
   end
 end
 
-class Rake::DockerTask < Rake::Task
-  def needed?
-    raise "Docker does not exist" if docker.nil?
-    !docker.exists?
+namespace :dockly do
+  # Dockly needs to be able to perform HTTP requests, and unfortunately for us
+  # WebMock is enabled by default.
+  task :enable_http do
+    if defined?(WebMock::HttpLibAdapters::NetHttpAdapter)
+      WebMock::HttpLibAdapters::NetHttpAdapter.disable!
+    end
   end
 
-  def docker
-    Dockly::Docker[name.split(':').last.to_sym]
+  task :load do
+    raise "No #{Dockly.load_file} found!" unless File.exist?(Dockly.load_file)
+    Dockly.inst
   end
-end
 
-module Rake::DSL
-  def deb(*args, &block)
-    Rake::DebTask.define_task(*args, &block)
+  task :assume_role => 'dockly:load' do
+    Dockly.perform_role_assumption
   end
 
-  def rpm(*args, &block)
-    Rake::RpmTask.define_task(*args, &block)
+  task :init => ['dockly:enable_http', 'dockly:load', 'dockly:assume_role']
+
+  namespace :deb do
+    task :prepare, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper.find_deb!(args[:name]).create_package!
+    end
+
+    task :upload, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper.find_deb!(args[:name]).upload_to_s3
+    end
+
+    task :copy, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper
+        .find_deb!(args[:name])
+        .copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
+    end
+
+    task :build, [:name] => 'dockly:init' do |t, args|
+      deb = Dockly::RakeHelper.find_deb!(args[:name])
+      deb.build unless deb.exists?
+    end
   end
 
-  def docker(*args, &block)
-    Rake::DockerTask.define_task(*args, &block)
+  namespace :rpm do
+    task :prepare, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper.find_rpm!(args[:name]).create_package!
+    end
+
+    task :upload, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper.find_rpm!(args[:name]).upload_to_s3
+    end
+
+    task :copy, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper
+        .find_rpm!(args[:name])
+        .copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
+    end
+
+    task :build, [:name] => 'dockly:init' do |t, args|
+      rpm = Dockly::RakeHelper.find_rpm!(args[:name])
+      rpm.build unless rpm.exists?
+    end
   end
-end
 
-namespace :dockly do
-  task :load do
-    raise "No dockly.rb found!" unless File.exist?('dockly.rb')
+  namespace :docker do
+    task :prepare, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper.find_docker!(args[:name]).generate_build
+    end
+
+    task :upload, [:name] => 'dockly:init' do |t, args|
+      docker = Dockly::RakeHelper.find_docker!(args[:name])
+      docker.export_only unless docker.exists?
+    end
+
+    task :copy, [:name] => 'dockly:init' do |t, args|
+      Dockly::RakeHelper
+        .find_docker!(args[:name])
+        .copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
+    end
+
+    task :build, [:name] => 'dockly:init' do |t, args|
+      docker = Dockly::RakeHelper.find_docker!(args[:name])
+      docker.generate! unless docker.exists?
+    end
   end
 
-  prepare_targets = []
-  upload_targets = []
-  build_targets = []
-  copy_targets = []
+  task :prepare_all => 'dockly:init' do
+    Dockly.debs.keys.each do |deb|
+      Rake::Task['dockly:deb:prepare'].tap(&:reenable).invoke(deb)
+    end
 
-  namespace :deb do
-    Dockly.debs.values.each do |inst|
-      namespace :prepare do
-        task inst.name => 'dockly:load' do |name|
-          inst.create_package!
-        end
-      end
-
-      namespace :upload do
-        deb inst.name => 'dockly:load' do |name|
-          inst.upload_to_s3
-        end
-      end
-
-      namespace :copy do
-        task inst.name => 'dockly:load' do |name|
-          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
-        end
-      end
-
-      deb inst.name => 'dockly:load' do |name|
-        inst.build
-      end
-
-      prepare_targets << "dockly:deb:prepare:#{inst.name}"
-      upload_targets << "dockly:deb:upload:#{inst.name}"
-      copy_targets << "dockly:deb:copy:#{inst.name}"
-      build_targets << "dockly:deb:#{inst.name}"
+    Dockly.rpms.values.each do |rpm|
+      Rake::Task['dockly:rpm:prepare'].tap(&:reenable).invoke(rpm)
+    end
+
+    Dockly.dockers.values.each do |docker|
+      Rake::Task['dockly:docker:prepare'].tap(&:reenable).invoke(docker)
     end
   end
 
-  namespace :rpm do
-    Dockly.rpms.values.each do |inst|
-      namespace :prepare do
-        task inst.name => 'dockly:load' do |name|
-          inst.create_package!
-        end
-      end
-
-      namespace :upload do
-        rpm inst.name => 'dockly:load' do |name|
-          inst.upload_to_s3
-        end
-      end
-
-      namespace :copy do
-        task inst.name => 'dockly:load' do |name|
-          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
-        end
-      end
-
-      rpm inst.name => 'dockly:load' do |name|
-        inst.build
-      end
-
-      prepare_targets << "dockly:rpm:prepare:#{inst.name}"
-      upload_targets << "dockly:rpm:upload:#{inst.name}"
-      copy_targets << "dockly:rpm:copy:#{inst.name}"
-      build_targets << "dockly:rpm:#{inst.name}"
+  task :upload_all => 'dockly:init' do
+    Dockly.debs.keys.each do |deb|
+      Rake::Task['dockly:deb:upload'].tap(&:reenable).invoke(deb)
+    end
+
+    Dockly.rpms.keys.each do |rpm|
+      Rake::Task['dockly:rpm:upload'].tap(&:reenable).invoke(rpm)
+    end
+
+    Dockly.dockers.keys.each do |docker|
+      Rake::Task['dockly:docker:upload'].tap(&:reenable).invoke(docker)
     end
   end
 
-  namespace :docker do
-    Dockly.dockers.values.each do |inst|
-      # For backwards compatibility
-      namespace :noexport do
-        task inst.name => "dockly:docker:prepare:#{inst.name}"
-      end
-
-      namespace :prepare do
-        task inst.name => 'dockly:load' do
-          Thread.current[:rake_task] = inst.name
-          inst.generate_build
-        end
-      end
-
-      namespace :upload do
-        docker inst.name => 'dockly:load' do
-          Thread.current[:rake_task] = inst.name
-          inst.export_only
-        end
-      end
-
-      namespace :copy do
-        task inst.name => 'dockly:load' do
-          Thread.current[:rake_task] = inst.name
-          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
-        end
-      end
-
-      docker inst.name => 'dockly:load' do
-        Thread.current[:rake_task] = inst.name
-        inst.generate!
-      end
-
-      # Docker image will be generated by 'dockly:deb:package'
-      unless inst.s3_bucket.nil?
-        prepare_targets << "dockly:docker:prepare:#{inst.name}"
-        upload_targets << "dockly:docker:upload:#{inst.name}"
-        copy_targets << "dockly:docker:copy:#{inst.name}"
-        build_targets << "dockly:docker:#{inst.name}"
-      end
+  task :build_all => 'dockly:init' do
+    Dockly.debs.keys.each do |deb|
+      Rake::Task['dockly:deb:build'].tap(&:reenable).invoke(deb)
+    end
+
+    Dockly.rpms.keys.each do |rpm|
+      Rake::Task['dockly:rpm:build'].tap(&:reenable).invoke(rpm)
+    end
+
+    Dockly.dockers.keys.each do |docker|
+      Rake::Task['dockly:docker:build'].tap(&:reenable).invoke(docker)
     end
   end
 
-  multitask :prepare_all => prepare_targets
-  multitask :upload_all => upload_targets
-  multitask :build_all => build_targets
-  multitask :copy_all => copy_targets
+  task :copy_all => 'dockly:init' do
+    Dockly.debs.keys.each do |deb|
+      Rake::Task['dockly:deb:copy'].tap(&:reenable).invoke(deb)
+    end
+
+    Dockly.rpms.keys.each do |rpm|
+      Rake::Task['dockly:rpm:copy'].tap(&:reenable).invoke(rpm)
+    end
+
+    Dockly.dockers.keys.each do |docker|
+      Rake::Task['dockly:docker:copy'].tap(&:reenable).invoke(docker)
+    end
+  end
 
   task :build_or_copy_all do
     if Dockly::History.duplicate_build?