dockly 0.0.2

data/lib/dockly/build_cache.rb

@@ -0,0 +1,131 @@
+require 'tempfile'
+
+class Dockly::BuildCache
+  include Dockly::Util::DSL
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[dockly build_cache]'
+
+  attr_accessor :image
+  dsl_attribute :s3_bucket, :s3_object_prefix, :hash_command, :output_dir, :build_command,
+                :use_latest, :tmp_dir
+
+  default_value :use_latest, false
+  default_value :tmp_dir, '/tmp'
+
+  def execute!
+    ensure_present! :image
+    if up_to_date?
+      debug "build cache up to date, pulling from s3"
+      insert_cache
+    else
+      insert_latest
+      debug "build cache out of date, running build"
+      run_build
+    end
+    debug "finished build cache"
+    image
+  end
+
+  def insert_cache
+    push_cache(hash_output)
+  end
+
+  def insert_latest
+    if use_latest
+      debug "attempting to push latest"
+      if cache = push_cache("latest")
+        debug "pushed latest, removing local file"
+        File.delete(cache.path)
+      end
+    end
+  end
+
+  def run_build
+    container = image.run(build_command).tap(&:start)
+    cache = copy_output_dir(container)
+    debug "pushing #{output_dir} to s3"
+    push_to_s3(cache)
+    cache.close
+    self.image = container.commit
+  end
+
+  def push_cache(version)
+    ensure_present! :output_dir
+    if cache = pull_from_s3(version)
+      debug "inserting to #{output_dir}"
+      self.image = image.insert_local(
+        'localPath' => cache.path,
+        'outputPath' => File.dirname(output_dir)
+      )
+      cache.close
+    else
+      info "could not find #{s3_object(version)}"
+    end
+  end
+
+  def up_to_date?
+    ensure_present! :s3_bucket, :s3_object_prefix
+    connection.head_object(s3_bucket, s3_object(hash_output))
+    true
+  rescue Excon::Errors::NotFound
+    false
+  end
+
+  def pull_from_s3(version)
+    ensure_present! :s3_bucket, :s3_object_prefix
+
+    file_name = s3_object(version)
+    file_path = File.join(tmp_dir,file_name)
+
+    FileUtils.mkdir_p(File.dirname(file_path))
+    unless File.exist?(file_path)
+      object = connection.get_object(s3_bucket, file_name)
+
+      file = File.open(file_path, 'w+b')
+      file.write(object.body)
+      file.tap(&:rewind)
+    else
+      File.open(file_path, 'rb')
+    end
+  rescue Excon::Errors::NotFound
+    nil
+  end
+
+  def push_to_s3(file)
+    ensure_present! :s3_bucket, :s3_object_prefix
+    connection.put_object(s3_bucket, s3_object(hash_output), file.read)
+    connection.copy_object(s3_bucket, s3_object(hash_output), s3_bucket, s3_object("latest"))
+  end
+
+  def copy_output_dir(container)
+    ensure_present! :output_dir
+    file_path = File.join(tmp_dir,s3_object(hash_output))
+    FileUtils.mkdir_p(File.dirname(file_path))
+    file = File.open(file_path, 'w+b')
+    container.wait(3600) # 1 hour max timeout
+    container.copy(output_dir) { |chunk| file.write(chunk) }
+    file.tap(&:rewind)
+  end
+
+  def hash_output
+    ensure_present! :image, :hash_command
+    @hash_output ||= begin
+      resp = ""
+      image.run(hash_command).start.attach { |chunk| resp += chunk }
+      resp.strip
+    end
+  end
+
+  def file_output(file)
+    File.join(File.dirname(output_dir), File.basename(file.path))
+  end
+
+  def s3_object(file)
+    "#{s3_object_prefix}#{file}"
+  end
+
+  def connection
+    Dockly::AWS.s3
+  end
+end

data/lib/dockly/cli.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'dockly'
+require 'clamp'
+
+class Dockly::AbstractCommand < Clamp::Command
+  option ['-F', '--file'], 'FILE', 'dockly file to read', :default => 'dockly.rb', :attribute_name => :file
+
+  def execute
+    if File.exist?(file)
+      Dockly.setup(file)
+    else
+      raise 'Could not find a dockly file!'
+    end
+  end
+end
+
+class Dockly::BuildCommand < Dockly::AbstractCommand
+  parameter 'PACKAGE', 'the name to build the package for', :attribute_name => :package_name
+  option ['-f', '--force'], :flag, 'force the package build', :default => false, :attribute_name => :force
+
+  def execute
+    super
+    if package = Dockly::Deb.instances[package_name.to_sym]
+      if force? || !package.exists?
+        package.build
+      else
+        puts "Package already exists!"
+      end
+    end
+  end
+end
+
+class Dockly::ListCommand < Dockly::AbstractCommand
+  def execute
+    super
+    Dockly::Deb.instances.each_with_index do |(name, package), index|
+      puts "#{index + 1}. #{name}"
+    end
+  end
+end
+
+class Dockly::Cli < Dockly::AbstractCommand
+  subcommand ['build', 'b'], 'Create package', Dockly::BuildCommand
+  subcommand ['list', 'l'], 'List packages', Dockly::ListCommand
+end
+

data/lib/dockly/deb.rb

@@ -0,0 +1,141 @@
+require 'fpm'
+
+class Dockly::Deb
+  include Dockly::Util::DSL
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[dockly deb]'
+  dsl_attribute :package_name, :version, :release, :arch, :build_dir,
+                :pre_install, :post_install, :pre_uninstall, :post_uninstall,
+                :s3_bucket, :files
+  dsl_class_attribute :docker, Dockly::Docker
+  dsl_class_attribute :foreman, Dockly::Foreman
+
+  default_value :version, '0.0'
+  default_value :release, '0'
+  default_value :arch, 'x86_64'
+  default_value :build_dir, 'build/deb'
+  default_value :files, []
+
+  def file(source, destination)
+    @files << { :source => source, :destination => destination }
+  end
+
+  def create_package!
+    ensure_present! :build_dir
+    FileUtils.mkdir_p(build_dir)
+    FileUtils.rm(build_path) if File.exist?(build_path)
+    debug "exporting #{package_name} to #{build_path}"
+    build_package
+    if @deb_package
+      @deb_package.output(build_path)
+      info "exported #{package_name} to #{build_path}"
+    end
+  ensure
+    @dir_package.cleanup if @dir_package
+    @deb_package.cleanup if @deb_package
+  end
+
+  def build
+    info "creating package"
+    create_package!
+    info "uploading to s3"
+    upload_to_s3
+  end
+
+  def build_path
+    ensure_present! :build_dir
+    "#{build_dir}/#{output_filename}"
+  end
+
+  def exists?
+    debug "#{name}: checking for package: #{s3_url}"
+    Dockly::AWS.s3.head_object(s3_bucket, s3_object_name)
+    info "#{name}: found package: #{s3_url}"
+    true
+  rescue
+    info "#{name}: could not find package: " +
+         "#{s3_url}"
+    false
+  end
+
+  def upload_to_s3
+    return if s3_bucket.nil?
+    create_package! unless File.exist?(build_path)
+    info "uploading package to s3"
+    Dockly::AWS.s3.put_bucket(s3_bucket) rescue nil
+    Dockly::AWS.s3.put_object(s3_bucket, s3_object_name, File.new(build_path))
+  end
+
+  def s3_url
+    "s3://#{s3_bucket}/#{s3_object_name}"
+  end
+
+  def s3_object_name
+    "#{package_name}/#{Dockly::Util::Git.git_sha}/#{output_filename}"
+  end
+
+  def output_filename
+    "#{package_name}_#{version}.#{release}_#{arch}.deb"
+  end
+
+private
+  def build_package
+    ensure_present! :package_name, :version, :release, :arch
+
+    info "building #{package_name}"
+    @dir_package = FPM::Package::Dir.new
+    add_docker(@dir_package)
+    add_foreman(@dir_package)
+    add_files(@dir_package)
+
+    debug "converting to deb"
+    @deb_package = @dir_package.convert(FPM::Package::Deb)
+
+    @deb_package.scripts[:prein] = pre_install
+    @deb_package.scripts[:postin] = post_install
+    @deb_package.scripts[:preun] = pre_uninstall
+    @deb_package.scripts[:postun] = post_uninstall
+
+    @deb_package.name = package_name
+    @deb_package.version = version
+    @deb_package.iteration = release
+    @deb_package.architecture = arch
+
+    info "done building #{package_name}"
+  end
+
+  def add_foreman(package)
+    return if foreman.nil?
+    info "adding foreman export"
+    foreman.create!
+    package.attributes[:prefix] = foreman.init_dir
+    Dir.chdir(foreman.build_dir) do
+      package.input('.')
+    end
+    package.attributes[:prefix] = nil
+  end
+
+  def add_docker(package)
+    return if docker.nil?
+    info "adding docker image"
+    docker.generate!
+    package.attributes[:prefix] = docker.package_dir
+    Dir.chdir(File.dirname(docker.tar_path)) do
+      package.input(File.basename(docker.tar_path))
+    end
+    package.attributes[:prefix] = nil
+  end
+
+  def add_files(package)
+    return if files.empty?
+    info "adding files to package"
+    files.each do |file|
+      package.attributes[:prefix] = file[:destination]
+      Dir.chdir(File.dirname(file[:source])) do
+        package.input(File.basename(file[:source]))
+      end
+      package.attributes[:prefix] = nil
+    end
+  end
+end

data/lib/dockly/docker.rb

@@ -0,0 +1,169 @@
+require 'docker'
+require 'excon'
+require 'tempfile'
+require 'zlib'
+require 'rubygems/package'
+require 'fileutils'
+
+class Dockly::Docker
+  include Dockly::Util::DSL
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[dockly docker]'
+  dsl_attribute :import, :git_archive, :build, :repo, :tag, :build_dir, :package_dir,
+    :timeout, :cleanup_images, :build_caches
+
+  default_value :repo, 'dockly'
+  default_value :build_dir, 'build/docker'
+  default_value :package_dir, '/opt/docker'
+  default_value :build_caches, []
+  default_value :cleanup_images, false
+  default_value :timeout, 60
+
+  def generate!
+    Docker.options = { :read_timeout => timeout, :write_timeout => timeout }
+    docker_tar = File.absolute_path(ensure_tar(fetch_import))
+
+    import = import_base(docker_tar)
+
+    cleanup = add_git_archive(import)
+    cleanup = run_build_caches(cleanup)
+    cleanup = build_image(cleanup)
+
+    export_image(cleanup)
+
+    true
+  ensure
+    cleanup.remove if cleanup_images && !cleanup.nil?
+  end
+
+  def export_filename
+    "#{repo}-#{tag}-image.tgz"
+  end
+
+  def run_build_caches(image)
+    info "starting build caches"
+    build_caches.each do |cache|
+      cache.image = image
+      image = cache.execute!
+    end
+    info "finished build caches"
+    image
+  end
+
+  def tar_path
+    File.join(build_dir, export_filename)
+  end
+
+  def ensure_tar(file_name)
+    if Dockly::Util::Tar.is_tar?(file_name)
+      file_name
+    elsif Dockly::Util::Tar.is_gzip?(file_name)
+      file_name
+    else
+      raise "Expected a (possibly gzipped) tar: #{file_name}"
+    end
+  end
+
+  def make_git_archive
+    ensure_present! :tag, :git_archive
+    info "initializing"
+
+    prefix = git_archive
+    prefix += '/' unless prefix.end_with?('/')
+
+    FileUtils.rm_rf(git_archive_dir)
+    FileUtils.mkdir_p(git_archive_dir)
+    info "archiving #{Dockly::Util::Git.git_sha}"
+    Grit::Git.with_timeout(120) do
+      Dockly::Util::Git.git_repo.archive_to_file(Dockly::Util::Git.git_sha, prefix, git_archive_path, 'tar', 'cat')
+    end
+    git_archive_path
+  end
+
+  def git_archive_dir
+    @git_archive_dir ||= File.join(build_dir, "gitarc")
+  end
+
+  def git_archive_path
+    "#{git_archive_dir}/#{name}.tar"
+  end
+
+  def git_archive_tar
+    git_archive && File.absolute_path(make_git_archive)
+  end
+
+  def import_base(docker_tar)
+    info "importing the docker image from #{docker_tar}"
+    image = ::Docker::Image.import(docker_tar)
+    info "imported docker image: #{image.id}"
+    image
+  end
+
+  def add_git_archive(image)
+    return image if git_archive.nil?
+
+    image.insert_local(
+      'localPath' => git_archive_tar,
+      'outputPath' => '/'
+    )
+  end
+
+  def build_image(image)
+    ensure_present! :repo, :tag, :build
+    info "starting build from #{image.id}"
+    out_image = ::Docker::Image.build("from #{image.id}\n#{build}")
+    info "built the image: #{out_image.id}"
+    out_image.tag(:repo => repo, :tag => tag)
+    out_image
+  end
+
+  def export_image(image)
+    ensure_present! :repo, :tag, :build_dir
+    container = ::Docker::Container.create('Image' => image.id, 'Cmd' => %w[true])
+    info "created the container: #{container.id}"
+    Zlib::GzipWriter.open(tar_path) do |file|
+      container.export do |chunk, remaining, total|
+        file.write(chunk)
+      end
+    end
+    info "done writing the docker tar: #{export_filename}"
+  end
+
+  def fetch_import
+    ensure_present! :tag, :import
+    path = "/tmp/dockly-docker-import.#{name}.#{File.basename(import)}"
+
+    if File.exist?(path)
+      debug "already fetched #{import}"
+    else
+      debug "fetching #{import}"
+      File.open("#{path}.tmp", 'wb') do |file|
+        case import
+          when /^s3:\/\/(?<bucket_name>.+?)\/(?<object_path>.+)$/
+            connection.get_object(Regexp.last_match[:bucket_name],
+                                  Regexp.last_match[:object_path]) do |chunk, remaining, total|
+              file.write(chunk)
+            end
+          when /^https?:\/\//
+            Excon.get(import, :response_block => lambda { |chunk, remaining, total|
+              file.write(chunk)
+            })
+          else
+            raise "You can only import from S3 or a public url"
+        end
+      end
+      FileUtils.mv("#{path}.tmp", path, :force => true)
+    end
+    path
+  end
+
+  def build_cache(&block)
+    build_caches << Dockly::BuildCache.new(&block)
+  end
+
+private
+  def connection
+    Dockly::AWS.s3
+  end
+end