dockly 1.5.8 → 1.5.9

data/Gemfile

@@ -4,3 +4,6 @@ source 'https://rubygems.org'
 gem 'foreman', :git => 'https://github.com/adamjt/foreman'
 
 gemspec
+
+gem 'fog-core'
+gem 'fog-json'

data/README.md

@@ -16,6 +16,8 @@ Usage
 -----
 
 Once a `deb` block has been defined by the DSL below, dockly is invoked by either `bundle exec dockly build #{deb block name}` or `bundle exec rake dockly:deb:#{deb block name}`.
+If looking to just build a `docker` block, run either `bundle exec dockly docker #{docker block name}` or `bundle exec rake dockly:docker:#{docker block name}`.
+To build without exporting, run add either `--no-export` or `:noexport` to the CLI program or Rake task
 
 The DSL
 -------
@@ -142,6 +144,18 @@ The `docker` DSL is used to define Docker containers. It has the following attri
     - required: `false`
     - default: `[]`
     - description: a listing of references to build caches to run
+- `s3_bucket`
+    - required: `false`
+    - default: `nil`
+    - description: S3 bucket for where the docker image is exported
+- `s3_object_prefix`
+    - required: `false`
+    - default: `nil`
+    - description: prefix to be added to S3 exported docker images
+- `tar_diff`
+    - required: `false`
+    - default: `false`
+    - description: after docker export, performs a diff between the base image and the new image
 
 In addition to the above attributes, `docker` has the following references:
 

data/Rakefile

@@ -16,4 +16,5 @@ end
 
 Cane::RakeTask.new(:quality) do |cane|
   cane.canefile = '.cane'
+  cane.abc_exclude = %w(TarDiff#process)
 end

data/dockly.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = %w{lib}
   gem.version       = Dockly::VERSION
   gem.add_dependency 'clamp', '~> 0.6'
-  gem.add_dependency 'docker-api', '>= 1.8.4'
+  gem.add_dependency 'docker-api', '>= 1.13.1'
   gem.add_dependency 'dockly-util', '~> 0.0.8'
   gem.add_dependency 'excon'
   gem.add_dependency 'fog', '~> 1.21.0'

data/lib/dockly.rb

@@ -11,9 +11,11 @@ module Dockly
 
   autoload :AWS, 'dockly/aws'
   autoload :Foreman, 'dockly/foreman'
+  autoload :BashBuilder, 'dockly/bash_builder'
   autoload :BuildCache, 'dockly/build_cache'
   autoload :Docker, 'dockly/docker'
   autoload :Deb, 'dockly/deb'
+  autoload :TarDiff, 'dockly/tar_diff'
 
   LOAD_FILE = 'dockly.rb'
 

data/lib/dockly/aws.rb

@@ -4,6 +4,8 @@ require 'fog/aws'
 module Dockly::AWS
   extend self
 
+  autoload :S3Writer, 'dockly/aws/s3_writer'
+
   def service(name, klass)
     define_method name do
       if val = instance_variable_get(:"@#{name}")

data/lib/dockly/aws/s3_writer.rb

@@ -0,0 +1,59 @@
+module Dockly
+  module AWS
+    class S3Writer
+      include Dockly::Util::Logger::Mixin
+
+      logger_prefix '[dockly s3writer]'
+
+      attr_accessor :buffer
+      attr_reader :connection, :s3_bucket, :s3_object, :upload_id
+
+      def initialize(connection, s3_bucket, s3_object)
+        @connection = connection
+        @s3_bucket = s3_bucket
+        @s3_object = s3_object
+        @parts = []
+        @closed = false
+        @buffer = ""
+
+        init_upload_res = connection.initiate_multipart_upload(s3_bucket, s3_object)
+        @upload_id = init_upload_res.body['UploadId']
+      end
+
+      def upload_buffer
+        res = connection.upload_part(s3_bucket, s3_object, upload_id, @parts.size + 1, buffer)
+        @parts << res.headers["ETag"]
+        debug "Writing a chunk"
+        @buffer = ""
+      end
+
+      def write(chunk)
+        self.buffer << chunk
+
+        upload_buffer if buffer.bytesize > 5242880
+
+        chunk.length
+      end
+
+      def close
+        return if @closed
+        upload_buffer unless buffer.empty?
+
+        res = connection.complete_multipart_upload(s3_bucket, s3_object, upload_id, @parts)
+        if res.body['Code'] || res.body['Message']
+          raise "Failed to upload to S3: #{res.body['Code']}: #{res.body['Message']}"
+        end
+        @closed = true
+      end
+
+      def abort
+        connection.abort_multipart_upload(s3_bucket, s3_object, upload_id)
+      end
+
+      def abort_unless_closed
+        abort unless @closed
+        @closed = true
+      end
+    end
+  end
+end

data/lib/dockly/bash_builder.rb

@@ -0,0 +1,29 @@
+require 'erb'
+require 'ostruct'
+
+module Dockly
+  class BashBuilder
+    SNIPPET_PATH = File.expand_path("../../../snippets", __FILE__)
+
+    def self.generate_snippet_for(name, opts, defaults={})
+      define_method(name) do |*args|
+        zipped_array = opts.zip(args).flatten
+        snippet = File.read(File.join(SNIPPET_PATH, "#{name}.erb"))
+        hash = Hash[*zipped_array].delete_if { |_,v| v.nil? }
+        data = defaults.merge(hash)
+        ERB.new(snippet).result(binding)
+      end
+    end
+
+    generate_snippet_for :normalize_for_dockly, []
+    generate_snippet_for :get_from_s3, [:s3_url, :output_path], { :output_path => "-" }
+    generate_snippet_for :install_package, [:path]
+    generate_snippet_for :get_and_install_deb, [:s3_url, :deb_path]
+    generate_snippet_for :docker_import, [:repo, :tag], { :tag => "latest" }
+    generate_snippet_for :file_docker_import, [:path, :repo, :tag]
+    generate_snippet_for :file_diff_docker_import, [:base_image, :diff_image, :repo, :tag]
+    generate_snippet_for :s3_docker_import, [:s3_url, :repo, :tag]
+    generate_snippet_for :s3_diff_docker_import, [:base_image, :diff_image, :repo, :tag]
+    generate_snippet_for :registry_import, [:repo, :tag], { :tag => "latest" }
+  end
+end

data/lib/dockly/build_cache.rb

@@ -1,11 +1,11 @@
 module Dockly::BuildCache
 end
 
-require 'dockly/build_cache/base'
-require 'dockly/build_cache/docker'
-require 'dockly/build_cache/local'
-
 module Dockly::BuildCache
+  autoload :Base, 'dockly/build_cache/base'
+  autoload :Docker, 'dockly/build_cache/docker'
+  autoload :Local, 'dockly/build_cache/local'
+
   class << self
     attr_writer :model
 

data/lib/dockly/build_cache/docker.rb

@@ -59,7 +59,7 @@ class Dockly::BuildCache::Docker < Dockly::BuildCache::Base
     container.wait(3600) # 1 hour max timeout
     debug 'Restarting the container to copy the cache\'s output'
     # Restart the container so we can copy its output
-    container = container.commit.run('sleep 3600').tap(&:start)
+    container = container.commit.run('sleep 3600')
     container.copy(output_directory) { |chunk| file.write(chunk.to_s) }
     container.kill
     file.tap(&:rewind)
@@ -88,7 +88,7 @@ class Dockly::BuildCache::Docker < Dockly::BuildCache::Base
     resp = ""
     debug "running command `#{command}` on image #{image.id}"
     container = image.run(["/bin/bash", "-lc", "cd #{command_directory} && #{command}"])
-    container.attach { |source,chunk| resp += chunk }
+    container.attach(logs: true) { |source,chunk| resp += chunk }
     status = container.wait['StatusCode']
     debug "`#{command}` returned the following output:"
     debug resp.strip

data/lib/dockly/cli.rb

@@ -30,10 +30,44 @@ class Dockly::BuildCommand < Dockly::AbstractCommand
   end
 end
 
+class Dockly::DockerCommand < Dockly::AbstractCommand
+  parameter 'DOCKER', 'the name to generate the docker image for', :attribute_name => :docker_name
+  option ['-f', '--force'], :flag, 'force the package build', :default => false, :attribute_name => :force
+  option ['-n', '--no-export'], :flag, 'do not export', :default => false, :attribute_name => :noexport
+
+  def execute
+    super
+    if docker = Dockly.dockers[docker_name.to_sym]
+      if force? || !docker.exists?
+        if noexport?
+          docker.generate_build
+        else
+          docker.generate!
+        end
+      else
+        puts "Package already exists!"
+      end
+    end
+  end
+end
+
 class Dockly::ListCommand < Dockly::AbstractCommand
   def execute
     super
-    Dockly.debs.each_with_index do |(name, package), index|
+    dockers = Dockly.dockers.dup
+    debs = Dockly.debs
+
+    puts "Debs" unless debs.empty?
+    debs.each_with_index do |(name, package), index|
+      puts "#{index + 1}. #{name}"
+      if package.docker
+        dockers.delete(package.docker.name)
+        puts " - Docker: #{package.docker.name}"
+      end
+    end
+
+    puts "Dockers" unless dockers.empty?
+    dockers.each_with_index do |(name, docker), index|
       puts "#{index + 1}. #{name}"
     end
   end
@@ -68,6 +102,7 @@ end
 
 class Dockly::Cli < Dockly::AbstractCommand
   subcommand ['build', 'b'], 'Create package', Dockly::BuildCommand
+  subcommand ['docker', 'd'], 'Generate docker image', Dockly::DockerCommand
   subcommand ['list', 'l'], 'List packages', Dockly::ListCommand
   subcommand ['build_cache', 'bc'], 'Build Cache commands', Dockly::BuildCacheCommand
 end

data/lib/dockly/deb.rb

@@ -6,8 +6,8 @@ class Dockly::Deb
 
   logger_prefix '[dockly deb]'
   dsl_attribute :package_name, :version, :release, :arch, :build_dir,
-                :pre_install, :post_install, :pre_uninstall, :post_uninstall,
-                :s3_bucket, :files, :app_user
+                :deb_build_dir, :pre_install, :post_install, :pre_uninstall,
+                :post_uninstall, :s3_bucket, :files, :app_user
 
   dsl_class_attribute :docker, Dockly::Docker
   dsl_class_attribute :foreman, Dockly::Foreman
@@ -15,7 +15,8 @@ class Dockly::Deb
   default_value :version, '0.0'
   default_value :release, '0'
   default_value :arch, 'x86_64'
-  default_value :build_dir, 'build/deb'
+  default_value :build_dir, 'build'
+  default_value :deb_build_dir, 'deb'
   default_value :files, []
   default_value :app_user, 'nobody'
 
@@ -24,8 +25,8 @@ class Dockly::Deb
   end
 
   def create_package!
-    ensure_present! :build_dir
-    FileUtils.mkdir_p(build_dir)
+    ensure_present! :build_dir, :deb_build_dir
+    FileUtils.mkdir_p(File.join(build_dir, deb_build_dir))
     FileUtils.rm(build_path) if File.exist?(build_path)
     debug "exporting #{package_name} to #{build_path}"
     build_package
@@ -46,8 +47,8 @@ class Dockly::Deb
   end
 
   def build_path
-    ensure_present! :build_dir
-    "#{build_dir}/#{output_filename}"
+    ensure_present! :build_dir, :deb_build_dir
+    File.join(build_dir, deb_build_dir, output_filename)
   end
 
   def exists?
@@ -63,7 +64,7 @@ class Dockly::Deb
 
   def upload_to_s3
     return if s3_bucket.nil?
-    create_package! unless File.exist?(build_path)
+    raise "Package wasn't created!" unless File.exist?(build_path)
     info "uploading package to s3"
     Dockly::AWS.s3.put_bucket(s3_bucket) rescue nil
     Dockly::AWS.s3.put_object(s3_bucket, s3_object_name, File.new(build_path))
@@ -81,16 +82,26 @@ class Dockly::Deb
     "#{package_name}_#{version}.#{release}_#{arch}.deb"
   end
 
+  def startup_script
+    scripts = []
+    bb = Dockly::BashBuilder.new
+    scripts << bb.normalize_for_dockly
+    scripts << bb.get_and_install_deb(s3_url, "/opt/dockly/#{File.basename(s3_url)}")
+
+    scripts.join("\n")
+  end
+
 private
   def build_package
     ensure_present! :package_name, :version, :release, :arch
 
     info "building #{package_name}"
     @dir_package = FPM::Package::Dir.new
-    add_docker(@dir_package)
     add_foreman(@dir_package)
     add_files(@dir_package)
     add_docker_auth_config(@dir_package)
+    add_docker(@dir_package)
+    add_startup_script(@dir_package)
 
     debug "converting to deb"
     @deb_package = @dir_package.convert(FPM::Package::Deb)
@@ -119,18 +130,6 @@ private
     package.attributes[:prefix] = nil
   end
 
-  def add_docker(package)
-    return if docker.nil?
-    info "adding docker image"
-    docker.generate!
-    return unless docker.registry.nil?
-    package.attributes[:prefix] = docker.package_dir
-    Dir.chdir(File.dirname(docker.tar_path)) do
-      package.input(File.basename(docker.tar_path))
-    end
-    package.attributes[:prefix] = nil
-  end
-
   def add_files(package)
     return if files.empty?
     info "adding files to package"
@@ -144,14 +143,71 @@ private
   end
 
   def add_docker_auth_config(package)
-    return if docker.nil? || (registry = docker.registry).nil? || !registry.authentication_required?
+    return if (registry = get_registry).nil? || !registry.authentication_required?
     info "adding docker config file"
     registry.generate_config_file!
 
-    package.attributes[:prefix] = docker.auth_config_file || "~#{app_user}/.dockercfg"
+    package.attributes[:prefix] = registry.auth_config_file || "~#{app_user}"
     Dir.chdir(File.dirname(registry.config_file)) do
       package.input(File.basename(registry.config_file))
     end
     package.attributes[:prefix] = nil
   end
+
+  def add_docker(package)
+    return if docker.nil? || docker.s3_bucket
+    info "adding docker image"
+    docker.generate!
+    return unless docker.registry.nil?
+    package.attributes[:prefix] = docker.package_dir
+    Dir.chdir(File.dirname(docker.tar_path)) do
+      package.input(File.basename(docker.tar_path))
+    end
+    package.attributes[:prefix] = nil
+  end
+
+  def get_registry
+    if docker && registry = docker.registry
+      registry
+    end
+  end
+
+  def post_startup_script
+    scripts = ["#!/bin/bash"]
+    bb = Dockly::BashBuilder.new
+    scripts << bb.normalize_for_dockly
+    if get_registry
+      scripts << bb.registry_import(docker.repo, docker.tag)
+    elsif docker
+      if docker.s3_bucket.nil?
+        docker_output = File.join(docker.package_dir, docker.export_filename)
+        if docker.tar_diff
+          scripts << bb.file_diff_docker_import(docker.import, docker_output, docker.name, docker.tag)
+        else
+          scripts << bb.file_docker_import(docker_output, docker.name, docker.tag)
+        end
+      else
+        if docker.tar_diff
+          scripts << bb.s3_diff_docker_import(docker.import, docker.s3_url, docker.name, docker.tag)
+        else
+          scripts << bb.s3_docker_import(docker.s3_url, docker.name, docker.tag)
+        end
+      end
+    end
+    scripts.join("\n")
+  end
+
+  def add_startup_script(package)
+    ensure_present! :build_dir
+    startup_script_path = File.join(build_dir, "dockly-startup.sh")
+    File.open(startup_script_path, 'w+') do |f|
+      f.write(post_startup_script)
+      f.chmod(0755)
+    end
+    package.attributes[:prefix] = "/opt/dockly"
+    Dir.chdir(build_dir) do
+      package.input("dockly-startup.sh")
+    end
+    package.attributes[:prefix] = nil
+  end
 end

data/lib/dockly/docker.rb

@@ -17,13 +17,16 @@ class Dockly::Docker
   dsl_class_attribute :registry, Dockly::Docker::Registry
 
   dsl_attribute :name, :import, :git_archive, :build, :tag, :build_dir, :package_dir,
-    :timeout, :cleanup_images, :auth_config_file
+    :timeout, :cleanup_images, :tar_diff, :s3_bucket, :s3_object_prefix
 
   default_value :tag, nil
   default_value :build_dir, 'build/docker'
   default_value :package_dir, '/opt/docker'
   default_value :cleanup_images, false
   default_value :timeout, 60
+  default_value :tar_diff, false
+  default_value :s3_bucket, nil
+  default_value :s3_object_prefix, ""
 
   def generate!
     image = generate_build
@@ -171,6 +174,10 @@ class Dockly::Docker
     end
   end
 
+  def s3_url
+    "s3://#{s3_bucket}/#{s3_object}"
+  end
+
   def export_image(image)
     ensure_present! :name
     if registry.nil?
@@ -178,22 +185,74 @@ class Dockly::Docker
       info "Exporting the image with id #{image.id} to file #{File.expand_path(tar_path)}"
       container = image.run('true')
       info "created the container: #{container.id}"
-      if ENV['SHELL_EXPORT']
-        command = "docker export #{container.id} | gzip > #{tar_path}"
-        info "running '#{command}' to export #{container.id}"
-        system(command)
-        raise "Could not export image, exit code: #{$?}" unless $?.success?
+
+      unless s3_bucket.nil?
+        output = Dockly::AWS::S3Writer.new(connection, s3_bucket, s3_object)
       else
-        Zlib::GzipWriter.open(tar_path) do |file|
-          container.export do |chunk, remaining, total|
-            file.write(chunk)
-          end
-        end
+        output = File.open(tar_path, 'wb')
+      end
+
+      gzip_output = Zlib::GzipWriter.new(output)
+
+      if tar_diff
+        export_image_diff(container, gzip_output)
+      else
+        export_image_whole(container, gzip_output)
       end
-      info "done writing the docker tar: #{export_filename}"
     else
       push_to_registry(image)
     end
+  rescue
+    if output && !s3_bucket.nil?
+      output.abort_unless_closed
+    end
+    raise
+  ensure
+    gzip_output.close if gzip_output
+  end
+
+  def export_image_whole(container, output)
+    container.export do |chunk, remaining, total|
+      output.write(chunk)
+    end
+  end
+
+  def export_image_diff(container, output)
+    rd, wr = IO.pipe(Encoding::ASCII_8BIT)
+
+    rd.binmode
+    wr.binmode
+
+    thread = Thread.new do
+      begin
+        if Dockly::Util::Tar.is_tar?(fetch_import)
+          base = File.open(fetch_import, 'rb')
+        else
+          base = Zlib::GzipReader.new(File.open(fetch_import, 'rb'))
+        end
+        td = Dockly::TarDiff.new(base, rd, output)
+        td.process
+        info "done writing the docker tar: #{export_filename}"
+      ensure
+        base.close if base
+        rd.close
+      end
+    end
+
+    begin
+      container.export do |chunk, remaining, total|
+        wr.write(chunk)
+      end
+    ensure
+      wr.close
+      thread.join
+    end
+  end
+
+  def s3_object
+    output = "#{s3_object_prefix}"
+    output << "#{Dockly::Util::Git.git_sha}/"
+    output << "#{export_filename}"
   end
 
   def push_to_registry(image)
@@ -235,6 +294,18 @@ class Dockly::Docker
     path
   end
 
+  def exists?
+    return false unless s3_bucket
+    debug "#{name}: checking for package: #{s3_url}"
+    Dockly::AWS.s3.head_object(s3_bucket, s3_object)
+    info "#{name}: found package: #{s3_url}"
+    true
+  rescue
+    info "#{name}: could not find package: " +
+         "#{s3_url}"
+    false
+  end
+
   def repository(value = nil)
     name(value)
   end