dockly 1.13.0 → 2.0.0

data/.travis.yml

@@ -1,12 +1,8 @@
 language: ruby
 rvm:
-    - 1.9
-    - 2.0
-    - 2.1
-    - 2.2
-script:
-  - git fetch --tags
-  - CI=true bundle exec rake
+    - 1.9.3
+    - 2.0.0
+script: CI=true bundle exec rake
 before_install:
  - sudo apt-get update -qq
  - sudo apt-get install -qq rpm

data/README.md

@@ -9,6 +9,11 @@
 
 Although only a specific type of repository may be used, these assumptions allow us to define a simple DSL to describe your repository.
 
+Tool Requirements
+-----------------
+
+To use the generated startup scripts, you'll need to use AWS CLI v1.5.0+
+
 Usage
 -----
 
@@ -106,9 +111,6 @@ The `docker` DSL is used to define Docker containers. It has the following attri
     - required: `false` -- only required when `import` is not supplied
     - description: the location  of the base image to start building from
     - examples: `paintedfox/ruby`, `registry.example.com/my-custom-image`
-- `build_env`
-    - required: `false`
-    - description: Hash whose values are environment variables and keys are their values. These variables are only used during build commands, exported images will not contain them.
 - `import`
     - required: `false` -- only required when `registry_import` is not supplied
     - description: the location (url or S3 path) of the base image to start building from
@@ -281,7 +283,7 @@ In addition to the above attributes, `deb` has the following references:
     - required: `false`
     - default: `nil`
     - class: `Dockly::Foreman`
-    - description: any Foreman scripts used in the deb.
+    - description: any Foreman scripts used in the deb
 
 `rpm`
 -----

data/Rakefile

@@ -6,11 +6,6 @@ require 'rake'
 require 'dockly'
 require 'rspec/core/rake_task'
 require 'cane/rake_task'
-require 'pry'
-
-task :shell do
-  Pry.start(Dockly)
-end
 
 task :default => [:spec, :quality]
 

data/dockly.gemspec

@@ -22,7 +22,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'foreman'
   gem.add_dependency 'fpm', '~> 1.2.0'
   gem.add_dependency 'grit'
-  gem.add_dependency 'rugged'
   gem.add_development_dependency 'cane'
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rake'

data/lib/dockly/build_cache/base.rb

@@ -9,7 +9,7 @@ class Dockly::BuildCache::Base
   dsl_attribute :s3_bucket, :s3_object_prefix, :use_latest,
                 :hash_command, :build_command, :parameter_commands,
                 :base_dir, :command_dir, :output_dir, :tmp_dir,
-                :keep_old_files, :safe_push_cache
+                :keep_old_files
 
   default_value :use_latest, false
   default_value :parameter_commands, {}
@@ -17,7 +17,6 @@ class Dockly::BuildCache::Base
   default_value :output_dir, '.'
   default_value :tmp_dir, Dir.tmpdir
   default_value :keep_old_files, false
-  default_value :safe_push_cache, false
 
   def execute!
     debug "Looking for cache for hash: #{hash_output}"

data/lib/dockly/build_cache/docker.rb

@@ -1,6 +1,10 @@
 class Dockly::BuildCache::Docker < Dockly::BuildCache::Base
   attr_accessor :image
 
+  def wait_time
+    300 # max 5 minutes
+  end
+
   def execute!
     ensure_present! :image
     super
@@ -25,11 +29,25 @@ class Dockly::BuildCache::Docker < Dockly::BuildCache::Base
     ensure_present! :output_dir
     if cache = pull_from_s3(version)
       debug "inserting to #{output_directory}"
-      if safe_push_cache
-        push_cache_safe(cache)
-      else
-        push_cache_with_volumes(cache)
-      end
+      path = File.expand_path(cache.path)
+      path_parent = File.dirname(path)
+      tar_flags = keep_old_files ? '-xkf' : 'xf'
+      container = ::Docker::Container.create(
+        'Image' => image.id,
+        'Cmd' => ['/bin/bash', '-c', [
+            "mkdir -p #{File.dirname(output_directory)}",
+            '&&',
+            "tar #{tar_flags} #{File.join('/', 'host', path)} -C #{File.dirname(output_directory)}"
+          ].join(' ')
+        ],
+        'Volumes' => {
+          File.join('/', 'host', path_parent) => { path_parent => 'rw' }
+        }
+      )
+      container.start('Binds' => ["#{path_parent}:#{File.join('/', 'host', path_parent)}"])
+      result = container.wait['StatusCode']
+      raise "Got bad status code when copying build cache: #{result}" unless result.zero?
+      self.image = container.commit
       debug "inserted cache into #{output_directory}"
       cache.close
     else
@@ -37,37 +55,6 @@ class Dockly::BuildCache::Docker < Dockly::BuildCache::Base
     end
   end
 
-  def push_cache_safe(cache)
-    container = image.run("mkdir -p #{File.dirname(output_directory)}")
-    image_with_dir = container.tap(&:wait).commit
-    self.image = image_with_dir.insert_local(
-      'localPath' => cache.path,
-      'outputPath' => File.dirname(output_directory)
-    )
-  end
-
-  def push_cache_with_volumes(cache)
-    path = File.expand_path(cache.path)
-    path_parent = File.dirname(path)
-    tar_flags = keep_old_files ? '-xkf' : 'xf'
-    container = ::Docker::Container.create(
-      'Image' => image.id,
-      'Cmd' => ['/bin/bash', '-c', [
-          "mkdir -p #{File.dirname(output_directory)}",
-          '&&',
-          "tar #{tar_flags} #{File.join('/', 'host', path)} -C #{File.dirname(output_directory)}"
-        ].join(' ')
-      ],
-      'Volumes' => {
-        File.join('/', 'host', path_parent) => { path_parent => 'rw' }
-      }
-    )
-    container.start('Binds' => ["#{path_parent}:#{File.join('/', 'host', path_parent)}"])
-    result = container.wait['StatusCode']
-    raise "Got bad status code when copying build cache: #{result}" unless result.zero?
-    self.image = container.commit
-  end
-
   def copy_output_dir(container)
     ensure_present! :output_dir
     file_path = File.join(tmp_dir,s3_object(hash_output))
@@ -105,7 +92,7 @@ class Dockly::BuildCache::Docker < Dockly::BuildCache::Base
     debug "running command `#{command}` on image #{image.id}"
     container = image.run(["/bin/bash", "-c", "cd #{command_directory} && #{command}"])
     debug "command running in container #{container.id}"
-    status = container.wait(docker.timeout)['StatusCode']
+    status = container.wait(wait_time)['StatusCode']
     resp = container.streaming_logs(stdout: true, stderr: true)
     debug "`#{command}` returned the following output:"
     debug resp.strip

data/lib/dockly/deb.rb

@@ -7,11 +7,10 @@ class Dockly::Deb
   logger_prefix '[dockly deb]'
   dsl_attribute :package_name, :version, :release, :arch, :build_dir,
                 :deb_build_dir, :pre_install, :post_install, :pre_uninstall,
-                :post_uninstall, :s3_bucket, :files, :app_user, :vendor,
-                :package_startup_script
+                :post_uninstall, :s3_bucket, :files, :app_user, :vendor
 
   dsl_class_attribute :docker, Dockly::Docker
-  dsl_class_attribute :foreman, Dockly::Foreman, type: Array
+  dsl_class_attribute :foreman, Dockly::Foreman
 
   default_value :version, '0.0'
   default_value :release, '0'
@@ -21,14 +20,12 @@ class Dockly::Deb
   default_value :files, []
   default_value :app_user, 'nobody'
   default_value :vendor, 'Dockly'
-  default_value :package_startup_script, true
 
   def file(source, destination)
     @files << { :source => source, :destination => destination }
   end
 
   def create_package!
-    info "creating package"
     ensure_present! :build_dir, :deb_build_dir
     FileUtils.mkdir_p(File.join(build_dir, deb_build_dir))
     FileUtils.rm(build_path) if File.exist?(build_path)
@@ -44,18 +41,12 @@ class Dockly::Deb
   end
 
   def build
+    info "creating package"
     create_package!
+    info "uploading to s3"
     upload_to_s3
   end
 
-  def copy_from_s3(sha)
-    ensure_present! :s3_bucket
-    object = s3_object_name_for(sha)
-    info "Copying s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object_name}"
-    Dockly::AWS.s3.copy_object(s3_bucket, object, s3_bucket, s3_object_name)
-    info "Successfully copied s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object_name}"
-  end
-
   def build_path
     ensure_present! :build_dir, :deb_build_dir
     File.join(build_dir, deb_build_dir, output_filename)
@@ -67,12 +58,12 @@ class Dockly::Deb
     info "#{name}: found package: #{s3_url}"
     true
   rescue
-    info "#{name}: could not find package: #{s3_url}"
+    info "#{name}: could not find package: " +
+         "#{s3_url}"
     false
   end
 
   def upload_to_s3
-    info "uploading to s3"
     return if s3_bucket.nil?
     raise "Package wasn't created!" unless File.exist?(build_path)
     info "uploading package to s3"
@@ -85,11 +76,7 @@ class Dockly::Deb
   end
 
   def s3_object_name
-    s3_object_name_for(Dockly::Util::Git.git_sha)
-  end
-
-  def s3_object_name_for(sha)
-    "#{package_name}/#{sha}/#{output_filename}"
+    "#{package_name}/#{Dockly::Util::Git.git_sha}/#{output_filename}"
   end
 
   def output_filename
@@ -115,7 +102,7 @@ private
     add_files(@dir_package)
     add_docker_auth_config(@dir_package)
     add_docker(@dir_package)
-    add_startup_script(@dir_package) if package_startup_script
+    add_startup_script(@dir_package)
 
     convert_package
 
@@ -140,14 +127,14 @@ private
   end
 
   def add_foreman(package)
-    return if (foreman || []).empty?
-    foreman.each do |fore|
-      info "adding foreman export '#{fore.name}'"
-      fore.create!
-      package.attributes[:prefix] = fore.init_dir
-      Dir.chdir(fore.build_dir) { package.input('.') }
-      package.attributes[:prefix] = nil
+    return if foreman.nil?
+    info "adding foreman export"
+    foreman.create!
+    package.attributes[:prefix] = foreman.init_dir
+    Dir.chdir(foreman.build_dir) do
+      package.input('.')
     end
+    package.attributes[:prefix] = nil
   end
 
   def add_files(package)

data/lib/dockly/docker.rb

@@ -28,18 +28,6 @@ class Dockly::Docker
   default_value :s3_bucket, nil
   default_value :s3_object_prefix, ""
 
-  def build_env(hash = nil)
-    (@build_env ||= {}).tap { |env| env.merge!(hash) if hash.is_a?(Hash) }
-  end
-
-  def copy_from_s3(sha)
-    return if s3_bucket.nil?
-    object = s3_object_for(sha)
-    info "Copying s3://#{s3_bucket}/#{object} to #{s3_bucket}/#{s3_object}"
-    Dockly::AWS.s3.copy_object(s3_bucket, object, s3_bucket, s3_object)
-    info "Successfully copied s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object}"
-  end
-
   def generate!
     image = generate_build
     export_image(image)
@@ -47,21 +35,6 @@ class Dockly::Docker
     cleanup([image]) if cleanup_images
   end
 
-  def export_only
-    if image = find_image_by_repotag
-      info "Found image by repo:tag: #{repo}:#{tag} - #{image.inspect}"
-      export_image(image)
-    else
-      raise "Could not find image"
-    end
-  end
-
-  def find_image_by_repotag
-    Docker::Image.all.find do |image|
-      image.info["RepoTags"].include?("#{repo}:#{tag}")
-    end
-  end
-
   def generate_build
     Docker.options = { :read_timeout => timeout, :write_timeout => timeout }
     images = {}
@@ -77,10 +50,9 @@ class Dockly::Docker
       info "Successfully pulled #{full_name}"
     end
 
-    images[:two] = add_build_env(images[:one])
-    images[:three] = add_git_archive(images[:two])
-    images[:four] = run_build_caches(images[:three])
-    build_image(images[:four])
+    images[:two] = add_git_archive(images[:one])
+    images[:three] = run_build_caches(images[:two])
+    build_image(images[:three])
   ensure
     cleanup(images.values.compact) if cleanup_images
   end
@@ -97,8 +69,6 @@ class Dockly::Docker
 
   def cleanup(images)
     info 'Cleaning up intermediate images'
-    images ||= []
-    images = images.compact
     ::Docker::Container.all(:all => true).each do |container|
       image_id = container.json['Image']
       if images.any? { |image| image.id.start_with?(image_id) || image_id.start_with?(image.id) }
@@ -174,18 +144,6 @@ class Dockly::Docker
     image
   end
 
-  def add_build_env(image)
-    return image if build_env.empty?
-    info "Setting the following environment variables in the docker image: #{build_env.keys}"
-    dockerfile = [
-      "FROM #{image.id}",
-      *build_env.map { |key, val| "ENV #{key.to_s.shellescape}=#{val.to_s.shellescape}" }
-    ].join("\n")
-    out_image = ::Docker::Image.build(dockerfile)
-    info "Successfully set the environment variables in the dockerfile"
-    out_image
-  end
-
   def add_git_archive(image)
     return image if git_archive.nil?
     info "adding the git archive"
@@ -202,7 +160,7 @@ class Dockly::Docker
     info "running custom build steps, starting with id: #{image.id}"
     out_image = ::Docker::Image.build("from #{image.id}\n#{build}")
     info "finished running custom build steps, result id: #{out_image.id}"
-    out_image.tap { |img| img.tag(repo: repo, tag: tag, force: true) }
+    out_image.tap { |img| img.tag(:repo => repo, :tag => tag) }
   end
 
   def repo
@@ -228,10 +186,10 @@ class Dockly::Docker
       container = image.run('true')
       info "created the container: #{container.id}"
 
-      if s3_bucket.nil?
-        output = File.open(tar_path, 'wb')
-      else
+      unless s3_bucket.nil?
         output = Dockly::AWS::S3Writer.new(connection, s3_bucket, s3_object)
+      else
+        output = File.open(tar_path, 'wb')
       end
 
       gzip_output = Zlib::GzipWriter.new(output)
@@ -250,7 +208,6 @@ class Dockly::Docker
     end
     raise
   ensure
-    container.tap(&:wait).remove if container
     gzip_output.close if gzip_output
   end
 
@@ -293,11 +250,9 @@ class Dockly::Docker
   end
 
   def s3_object
-    s3_object_for(Dockly::Util::Git.git_sha)
-  end
-
-  def s3_object_for(sha)
-    [s3_object_prefix, sha, '/', export_filename].join
+    output = "#{s3_object_prefix}"
+    output << "#{Dockly::Util::Git.git_sha}/"
+    output << "#{export_filename}"
   end
 
   def push_to_registry(image)

data/lib/dockly/rake_task.rb

@@ -53,131 +53,48 @@ namespace :dockly do
     raise "No dockly.rb found!" unless File.exist?('dockly.rb')
   end
 
-  prepare_targets = []
-  upload_targets = []
   build_targets = []
-  copy_targets = []
 
   namespace :deb do
     Dockly.debs.values.each do |inst|
-      namespace :prepare do
-        task inst.name => 'dockly:load' do |name|
-          inst.create_package!
-        end
-      end
-
-      namespace :upload do
-        deb inst.name => 'dockly:load' do |name|
-          inst.upload_to_s3
-        end
+      deb inst.name => 'dockly:load' do |name|
+        Thread.current[:rake_task] = name
+        inst.build
       end
-
-      namespace :copy do
-        task inst.name => 'dockly:load' do |name|
-          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
-        end
-      end
-
-      deb inst.name => [
-        'dockly:load',
-        "dockly:deb:prepare:#{inst.name}",
-        "dockly:deb:upload:#{inst.name}"
-      ]
-      prepare_targets << "dockly:deb:prepare:#{inst.name}"
-      upload_targets << "dockly:deb:upload:#{inst.name}"
-      copy_targets << "dockly:deb:copy:#{inst.name}"
       build_targets << "dockly:deb:#{inst.name}"
     end
   end
 
   namespace :rpm do
     Dockly.rpms.values.each do |inst|
-      namespace :prepare do
-        task inst.name => 'dockly:load' do |name|
-          inst.create_package!
-        end
-      end
-
-      namespace :upload do
-        rpm inst.name => 'dockly:load' do |name|
-          inst.upload_to_s3
-        end
+      rpm inst.name => 'dockly:load' do |name|
+        Thread.current[:rake_task] = name
+        inst.build
       end
-
-      namespace :copy do
-        task inst.name => 'dockly:load' do |name|
-          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
-        end
-      end
-
-      rpm inst.name => [
-        'dockly:load',
-        "dockly:rpm:prepare:#{inst.name}",
-        "dockly:rpm:upload:#{inst.name}"
-      ]
-      prepare_targets << "dockly:rpm:prepare:#{inst.name}"
-      upload_targets << "dockly:rpm:upload:#{inst.name}"
-      copy_targets << "dockly:rpm:copy:#{inst.name}"
       build_targets << "dockly:rpm:#{inst.name}"
     end
   end
 
   namespace :docker do
     Dockly.dockers.values.each do |inst|
-      # For backwards compatibility
-      namespace :noexport do
-        task inst.name => "dockly:docker:prepare:#{inst.name}"
+      docker inst.name => 'dockly:load' do
+        Thread.current[:rake_task] = inst.name
+        inst.generate!
       end
 
-      namespace :prepare do
+      namespace :noexport do
         task inst.name => 'dockly:load' do
           Thread.current[:rake_task] = inst.name
           inst.generate_build
         end
       end
 
-      namespace :upload do
-        task inst.name => 'dockly:load' do
-          Thread.current[:rake_task] = inst.name
-          inst.export_only
-        end
-      end
-
-      namespace :copy do
-        task inst.name => 'dockly:load' do
-          Thread.current[:rake_task] = inst.name
-          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
-        end
-      end
-
-      docker inst.name => [
-        'dockly:load',
-        "dockly:docker:prepare:#{inst.name}",
-        "dockly:docker:upload:#{inst.name}"
-      ]
-
       # Docker image will be generated by 'dockly:deb:package'
       unless inst.s3_bucket.nil?
-        prepare_targets << "dockly:docker:prepare:#{inst.name}"
-        upload_targets << "dockly:docker:upload:#{inst.name}"
-        copy_targets << "dockly:docker:copy:#{inst.name}"
         build_targets << "dockly:docker:#{inst.name}"
       end
     end
   end
 
-  multitask :prepare_all => prepare_targets
-  multitask :upload_all => upload_targets
   multitask :build_all => build_targets
-  multitask :copy_all => copy_targets
-
-  task :build_or_copy_all do
-    if Dockly::History.duplicate_build?
-      Rake::Task['dockly:copy_all'].invoke
-    else
-      Rake::Task['dockly:build_all'].invoke
-      Dockly::History.write_content_tag!
-      Dockly::History.push_content_tag!
-    end
-  end
 end

data/lib/dockly/version.rb

@@ -1,3 +1,3 @@
 module Dockly
-  VERSION = '1.13.0'
+  VERSION = '2.0.0'
 end

data/lib/dockly.rb

@@ -4,7 +4,6 @@ require 'dockly/util/git'
 require 'fog'
 require 'foreman/cli_fix'
 require 'foreman/export/base_fix'
-require 'rugged'
 
 module Dockly
   attr_reader :instance, :git_sha
@@ -16,7 +15,6 @@ module Dockly
   autoload :BuildCache, 'dockly/build_cache'
   autoload :Docker, 'dockly/docker'
   autoload :Deb, 'dockly/deb'
-  autoload :History, 'dockly/history'
   autoload :Rpm, 'dockly/rpm'
   autoload :TarDiff, 'dockly/tar_diff'
 

data/snippets/file_diff_docker_import.erb

@@ -1,7 +1,7 @@
 file_diff_docker_import_fn() {
   s3_path="<%= data[:base_image] %>"
   log "fetch: starting to fetch $s3_path"
-  s3cmd -f get $s3_path - 2> >(log)
+  aws s3 cp --quiet $s3_path - 2> >(log)
   log "fetch: successfully fetched $s3_path"
 }
 

data/snippets/get_and_install_deb.erb

@@ -5,7 +5,7 @@ log "fetch: starting to fetch $s3_path"
 for attempt in {1..200}; do
   [[ $worked != 0 ]] || break
   log "fetch: attempt ${attempt} to get $s3_path ..."
-  s3cmd -f get $s3_path $output_path 2> >(log) && worked=0 || (log "fetch: attempt failed, sleeping 30"; sleep 30)
+  aws s3 cp --quiet $s3_path $output_path 2> >(log) && worked=0 || (log "fetch: attempt failed, sleeping 30"; sleep 30)
 done
 [[ $worked != 0 ]] && fatal "fetch: failed to pull deb from S3"
 log "fetch: successfully fetched $s3_path"

data/snippets/get_from_s3.erb

@@ -5,7 +5,7 @@ log "fetch: starting to fetch $s3_path"
 for attempt in {1..200}; do
   [[ $worked != 0 ]] || break
   log "fetch: attempt ${attempt} to get $s3_path ..."
-  s3cmd -f get $s3_path $output_path 2> >(log) && worked=0 || (log "fetch: attempt failed, sleeping 30"; sleep 30)
+  aws s3 cp --quiet $s3_path $output_path 2> >(log) && worked=0 || (log "fetch: attempt failed, sleeping 30"; sleep 30)
 done
 [[ $worked != 0 ]] && fatal "fetch: failed to pull deb from S3"
 log "fetch: successfully fetched $s3_path"

data/snippets/s3_diff_docker_import.erb

@@ -5,14 +5,14 @@ base_image = "/opt/dockly/base_image.tar"
 s3_diff_docker_import_base_fn() {
   s3_path="<%= data[:base_image] %>"
   log "fetch: starting to fetch $s3_path"
-  s3cmd -f get $s3_path - 2> >(log)
+  aws s3 cp --quiet $s3_path - 2> >(log)
   log "fetch: successfully fetched $s3_path"
 }
 
 s3_diff_docker_import_diff_fn() {
   s3_path="<%= data[:diff_image] %>"
   log "fetch: starting to fetch $s3_path"
-  s3cmd -f get $s3_path - 2> >(log)
+  aws s3 cp --quiet $s3_path - 2> >(log)
   log "fetch: successfully fetched $s3_path"
 }
 
@@ -33,18 +33,6 @@ docker_import() {
   docker import - $repo:$tag > >(log) 2>&1 || fatal "docker failed to import"
 }
 
-remove_bad_imports() {
-  log "removing bad imports"
-  images=$(docker images | grep \<none | awk '{ print $3 }' || echo "")
-  log "images: $images"
-  if [[ "x$images" != "x" ]]; then
-    docker rmi $images > >(log)
-    log "bad import removed"
-  else
-    log "no bad images"
-  fi
-}
-
 worked=1
 for attempt in {1..200}; do
   [[ $worked != 0 ]] || break
@@ -56,7 +44,7 @@ log "fetch: successfully pulled base image"
 worked=1
 for attempt in {1..200}; do
   [[ $worked != 0 ]] || break
-  stream_image | docker_import && worked=0 || (remove_bad_imports; log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
+  stream_image | docker_import && worked=0 || (log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
 done
 [[ $worked != 0 ]] && fatal "fetch: failed to import diff image"
 log "fetch: successfully imported diff image"

data/snippets/s3_docker_import.erb

@@ -1,7 +1,7 @@
 s3_docker_import_fn() {
   s3_path="<%= data[:s3_url] %>"
   log "fetch: starting to fetch $s3_path"
-  s3cmd -f get $s3_path - 2> >(log)
+  aws s3 cp --quiet $s3_path - 2> >(log)
   log "fetch: successfully fetched $s3_path"
 }