dry-dock 0.1.0

data/examples/ruby-node-app-dsl.rb

@@ -0,0 +1,128 @@
+#!/usr/bin/env drydock
+
+APP_ROOT   = '/app'
+BUILD_ROOT = '/build'
+REPO_NAME  = 'ripta/test'
+TAG_NAME   = 'v1.0'
+
+
+from 'gliderlabs/alpine', '3.2'
+
+with Plugins::APK do |pkg|
+  pkg.update
+  pkg.upgrade
+  pkg.add 'ruby', 'ruby-dev'
+  pkg.add 'nodejs', 'nodejs-dev'
+  pkg.add 'musl', 'musl-dev'
+  pkg.add 'linux-headers'
+  pkg.add 'gcc', 'g++'
+  pkg.add 'make'
+  pkg.add 'curl', 'curl-dev'
+  pkg.add 'openssh'
+  pkg.add 'libffi-dev', 'libxml2-dev', 'libxslt-dev'
+  pkg.add 'git'
+end
+
+download_once 'https://github.com/tianon/gosu/releases/download/1.3/gosu-amd64', '/bin/gosu', chmod: 0755
+
+with Plugins::Rubygems do |g|
+  # g.source.add 'https://s3.amazonaws.com/production.s3.rubygems.org/'
+  # g.source.remove 'https://rubygems.org/'
+  g.update_system(document: false)
+  g.install('bundler', document: false)
+  g.install('unicorn', document: false)
+end
+
+run 'bundle config --global frozen 1'
+run 'bundle config --global build.nokogiri --use-system-libraries'
+
+with Plugins::NPM do |npm|
+  npm.install('bower', 'gulp', global: true)
+end
+
+one_week_old = CachingStrategies::TimeLimited.new(
+  max_age:   1.week,
+  max_depth: 1
+)
+
+derive do
+  env 'BUILD_ROOT', BUILD_ROOT
+  mkdir BUILD_ROOT
+
+  env 'APP_ROOT', APP_ROOT
+  mkdir APP_ROOT
+
+  gem_image = derive(label: 'rubygems', strategy: one_week_old) do
+    copy 'Gemfile', BUILD_ROOT
+    copy 'Gemfile.lock', BUILD_ROOT
+
+    cd BUILD_ROOT do
+      run 'bundle --path vendor'
+    end
+  end
+
+  npm_image = if File.exist?('package.json')
+    derive(label: 'npm', strategy: one_week_old) do
+      copy 'package.json', BUILD_ROOT
+      cd BUILD_ROOT do
+        with(Plugins::NPM).install
+      end
+    end
+  end
+
+  import 'vendor',       from: BUILD_ROOT, in: gem_image, to: APP_ROOT
+  import 'node_modules', from: BUILD_ROOT, in: npm_image, to: APP_ROOT
+  # import_stream gem_image.export_stream(BUILD_ROOT / 'vendor'), APP_ROOT / 'vendor'
+  # import_stream npm_image.export_stream(BUILD_ROOT / 'node_modules'), APP_ROOT / 'node_modules'
+  copy '.', BUILD_ROOT
+
+  # import_stream BUILD_ROOT do |writer|
+  #   gem_image.export_stream(BUILD_ROOT + '/vendor') do |reader|
+  #     while chunk = reader.read(32 * 1024)
+  #       writer.write(chunk)
+  #     end
+  #   end
+  # end
+
+  # tag REPO_NAME, TAG_NAME
+end
+
+# # Drydock.using(dd) { |base| ... }
+# #   or
+# # base = Drydock.from(dd.id)
+# #   ...
+# Drydock.using(dd) do |base|
+#   base.env('BUILD_ROOT', build_root)
+#   base.mkdir(build_root)
+
+#   # dd.snapshot('name') do |base|
+#   #   ...
+#   # end
+#   #   or
+#   # base = Drydock.from('name') || Drydock.from(dd.id)
+#   #   ...
+#   # base.tag('name')
+#   bundle_image = base.snapshot('rubygems', max_age: Date.beginning_of_week) do |build|
+#     build.copy('Gemfile', build_root)
+#     build.copy('Gemfile.lock', build_root)
+#     build.in(build_root).run('bundle --path vendor')
+#   end
+
+#   npm_image = base.snapshot('npm', max_age: Date.beginning_of_week) do |build|
+#     build.copy('package.json', build_root)
+#     build.in(build_root).npm.install
+#   end
+
+#   Drydock.using(dd) do |build|
+#     build.env('APPLICATION_ROOT', app_root)
+#     build.mkdir(app_root)
+  
+#     build.copy('.', app_root)
+#     build.import(bundle_image.export(build_root), app_root)
+#     build.import(npm_image.export(build_root), app_root)
+
+#     build.tag('ripta/dumplings', 'v1.0')
+#   end
+# end
+
+

data/examples/test-dsl.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env drydock
+
+set :event_handler do |event|
+  puts event
+end
+
+from 'gliderlabs/alpine', '3.2'
+run 'apk update'
+puts latest_image.id

data/examples/test.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler'
+
+Bundler.setup
+require 'docker'
+
+Docker.options[:read_timeout] = 300
+
+estream = Thread.new do
+  Docker::Event.stream do |event|
+    puts event
+  end
+end
+
+stream_chunk_proc = proc do |stream, chunk|
+  puts "#{stream} #{chunk}"
+end
+
+i1 = Docker::Image.create(fromImage: 'gliderlabs/alpine', tag: '3.2')
+puts i1.id
+
+c1 = Docker::Container.create(Image: i1.id, Cmd: ['/bin/sh', '-c', 'apk update'], Tty: true)
+c1.tap(&:start).attach(&stream_chunk_proc)
+c1.wait
+puts "  #{c1.changes.size} changed files"
+
+i2 = c1.commit
+puts i2.id
+
+Thread.kill(estream)
+exit
+
+c2 = Docker::Container.create(Image: i2.id, Cmd: ['/bin/sh', '-c', 'apk add ruby ruby-dev'], Tty: true)
+c2.tap(&:start).attach(&stream_chunk_proc)
+c2.wait
+puts "  #{c2.changes.size} changed files"
+
+i3 = c2.commit
+puts i3.id
+
+##image.run('apk update')
+##image.run('apk add ruby ruby-dev')
+##image.run('apk add nodejs nodejs-dev')
+

data/lib/drydock/cli_flags.rb

@@ -0,0 +1,46 @@
+
+module Drydock
+  class CliFlags
+
+    def initialize(flags = {})
+      @flags = flags
+    end
+
+    def to_s
+      return '' if flags.nil? || flags.empty?
+
+      buffer = StringIO.new
+      flags.each_pair do |k, v|
+        buffer << process_flag(k, v)
+      end
+
+      buffer.string
+    end
+
+    private
+    attr_reader :flags
+
+    def process_flag(key, value)
+      key = key.to_s
+      if key.size == 1
+        "-#{key} "
+      else
+        key = key.gsub(/_/, '-')
+        case value
+        when TrueClass
+          "--#{key} "
+        when FalseClass
+          "--no-#{key} "
+        else
+          "--#{key} #{process_value(value)}"
+        end
+      end
+    end
+
+    def process_value(value)
+      value = value.to_s
+      value.match(/\s/) ? value.inspect : value
+    end
+
+  end
+end

data/lib/drydock/container_config.rb

@@ -0,0 +1,75 @@
+
+module Drydock
+  class ContainerConfig < ::Hash
+
+    DEFAULTS = {
+      'OpenStdin'    => false,
+      'AttachStdin'  => false,
+      'AttachStdout' => false,
+      'AttachStderr' => false,
+      'User'         => '',
+      'Tty'          => false,
+      'Cmd'          => nil,
+      'Env'          => ['PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'],
+      'Labels'       => nil,
+      'Entrypoint'   => nil,
+      'ExposedPorts' => nil,
+      'Volumes'      => nil
+    }
+
+    def self.from(hash)
+      return nil if hash.nil?
+
+      self.new.tap do |cfg|
+        DEFAULTS.each_pair do |k, v|
+          cfg[k] = v
+        end
+        hash.each_pair do |k, v|
+          cfg[k] = v
+        end
+      end
+    end
+
+    # Logic taken from https://github.com/docker/docker/blob/master/runconfig/compare.go
+    def ==(other)
+      return false if other.nil?
+
+      return false if self['OpenStdin'] || other['OpenStdin']
+      return false if self['AttachStdout'] != other['AttachStdout']
+      return false if self['AttachStderr'] != other['AttachStderr']
+
+      return false if self['User'] != other['User']
+      return false if self['Tty'] != other['Tty']
+
+      return false if self['Cmd'] != other['Cmd']
+      return false if Array(self['Env']).sort != Array(other['Env']).sort
+      return false if self['Labels'] != other['Labels']
+      return false if self['Entrypoint'] != other['Entrypoint']
+
+      my_ports = self['ExposedPorts'] || {}
+      other_ports = other['ExposedPorts'] || {}
+      return false if my_ports.keys.size != other_ports.keys.size
+      my_ports.keys.each do |my_port|
+        return false unless other_ports.key?(my_port)
+      end
+
+      my_vols = self['Volumes'] || {}
+      other_vols = other['Volumes'] || {}
+      return false if my_vols.keys.size != other_vols.keys.size
+      my_vols.keys.each do |my_vol|
+        return false unless other_vols.key?(my_vol)
+      end
+
+      return true
+    end
+
+    def [](key)
+      super(key.to_s)
+    end
+
+    def []=(key, value)
+      super(key.to_s, value)
+    end
+
+  end
+end

data/lib/drydock/docker_api_patch.rb

@@ -0,0 +1,176 @@
+
+module Docker
+
+  class Connection
+
+    def raw_request(*args, &block)
+      request = compile_request_params(*args, &block)
+      log_request(request)
+      resource.request(request)
+    rescue Excon::Errors::BadRequest => ex
+      raise ClientError, ex.response.body
+    rescue Excon::Errors::Unauthorized => ex
+      raise UnauthorizedError, ex.response.body
+    rescue Excon::Errors::NotFound => ex
+      raise NotFoundError, ex.response.body
+    rescue Excon::Errors::Conflict => ex
+      raise ConflictError, ex.response.body
+    rescue Excon::Errors::InternalServerError => ex
+      raise ServerError, ex.response.body
+    rescue Excon::Errors::Timeout => ex
+      raise TimeoutError, ex.message
+    end
+
+  end
+
+  class Container
+
+    def archive_get(path = '/', &blk)
+      query = { 'path' => path }
+      connection.get(path_for(:archive), query, response_block: blk)
+      self
+    end
+
+    def archive_head(path = '/', &blk)
+      query = { 'path' => path }
+      response = connection.raw_request(:head, path_for(:archive), query, response_block: blk)
+
+      return if response.nil?
+      return if response.headers.empty?
+      return unless response.headers.key?('X-Docker-Container-Path-Stat')
+
+      ContainerPathStat.new(response.headers['X-Docker-Container-Path-Stat'])
+    rescue Docker::Error::NotFoundError
+      nil
+    end
+
+    def archive_put(path = '/', overwrite: false, &blk)
+      headers = { 'Content-Type' => 'application/x-tar' }
+      query   = { 'path' => path, 'noOverwriteDirNonDir' => overwrite }
+
+      output = StringIO.new
+      blk.call(output)
+      output.rewind
+
+      connection.put(path_for(:archive), query, headers: headers, body: output)
+      self
+    end
+
+  end
+
+  class ContainerPathStat
+
+    def initialize(definition)
+      @data = JSON.parse(Base64.decode64(definition))
+    end
+
+    def link_target
+      @data.fetch('linkTarget')
+    end
+
+    def method_missing(method_name, *method_args, &blk)
+      if mode.respond_to?(method_name)
+        mode.public_send(method_name, *method_args, &blk)
+      else
+        super
+      end
+    end
+
+    def mode
+      @mode ||= UniversalFileMode.new(@data.fetch('mode'))
+    end
+
+    def mtime
+      @mtime ||= DateTime.parse(@data.fetch('mtime'))
+    end
+
+    def name
+      @data.fetch('name')
+    end
+
+    def respond_to?(method_name)
+      mode.respond_to?(method_name) || super
+    end
+
+    def size
+      @data.fetch('size')
+    end
+
+  end
+
+  # Go implementation of cross-system file modes: https://golang.org/pkg/os/#FileMode
+  class UniversalFileMode
+
+    BIT_FIELDS = [
+      {directory:        'd'},
+      {append_only:      'a'},
+      {exclusive:        'l'},
+      {temporary:        'T'},
+      {link:             'L'},
+      {device:           'D'},
+      {named_pipe:       'p'},
+      {socket:           'S'},
+      {setuid:           'u'},
+      {setgid:           'g'},
+      {character_device: 'c'},
+      {sticky:           't'}
+    ]
+
+    def self.bit_for(name)
+      32 - 1 - BIT_FIELDS.index { |field| field.keys.first == name }
+    end
+
+    def self.flags
+      BIT_FIELDS.map { |field| field.keys.first }
+    end
+
+    def self.file_mode_mask
+      0777
+    end
+
+    def self.short_flag_for(name)
+      BIT_FIELDS.find { |field| field.keys.first == name }.values.first
+    end
+
+    def self.type_mode_mask
+      value_for(:directory) | value_for(:link) | value_for(:named_pipe) | value_for(:socket) | value_for(:device)
+    end
+
+    def self.value_for(name)
+      1 << bit_for(name)
+    end
+
+    def initialize(value)
+      @value = value
+    end
+
+    def file_mode
+      (@value & self.class.file_mode_mask)
+    end
+
+    def flags
+      self.class.flags.select { |name| send("#{name}?") }
+    end
+
+    def regular?
+      (@value & self.class.type_mode_mask) == 0
+    end
+
+    def short_flags
+      flags.map { |flag| self.class.short_flag_for(flag) }
+    end
+
+    def to_s
+      short_flags.join
+    end
+
+    flags.each do |name|
+      define_method("#{name}?") do
+        bit_value = self.class.value_for(name)
+        (@value & bit_value) == bit_value
+      end
+    end
+
+  end
+
+end

data/lib/drydock/drydock.rb

@@ -0,0 +1,65 @@
+
+module Drydock
+
+  def self.banner
+    "Drydock v#{Drydock.version}"
+  end
+
+  def self.build(opts = {}, &blk)
+    Project.new(opts).tap do |project|
+      dryfile, dryfilename = yield
+
+      Dir.chdir(File.dirname(dryfilename))
+      Drydock.logger.info("Working directory set to #{Dir.pwd}")
+
+      begin
+        catch :done do
+          project.instance_eval(dryfile, dryfilename)
+        end
+      rescue => e
+        Drydock.logger.error("Error processing #{dryfilename}:")
+        Drydock.logger.error(message: "#{e.class}: #{e.message}")
+        e.backtrace.each do |backtrace|
+          Drydock.logger.debug(message: "#{backtrace}", indent: 1)
+        end
+      ensure
+        Drydock.logger.info("Cleaning up")
+        project.finalize!
+      end
+    end
+  end
+
+  def self.build_on_chain(chain, opts = {}, &blk)
+    Project.new(opts.merge(chain: chain)).tap do |project|
+      project.instance_eval(&blk) if blk
+    end
+  end
+
+  def self.from(repo, opts = {}, &blk)
+    opts = opts.clone
+    tag  = opts.delete(:tag, 'latest')
+
+    build(opts).tap do |project|
+      project.from(repo, tag)
+      yield project
+    end
+  end
+
+  def self.logger
+    @logger ||= Logger.new(File.new('/dev/null', 'w+'))
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.using(project)
+    raise NotImplementedError, "TODO(rpasay)"
+  end
+
+  def self.version
+    version_file = File.join(File.dirname(__FILE__), '..', '..', 'VERSION')
+    File.exist?(version_file) ? File.read(version_file).chomp : ""
+  end
+
+end

data/lib/drydock/errors.rb

@@ -0,0 +1,6 @@
+
+module Drydock
+  class OperationError < StandardError; end
+
+  class InvalidInstructionError < OperationError; end
+end

data/lib/drydock/file_manager.rb

@@ -0,0 +1,26 @@
+
+module Drydock
+  class FileManager
+
+    def self.find(path, ignorefile, prepend_path: false, recursive: true)
+      path = path.sub(%r{/$}, '')
+
+      [].tap do |results|
+        ::Find.find(path) do |subpath|
+          subpath = subpath.sub(%r{^#{path}/}, '')
+
+          Find.prune if ignorefile.match?(subpath)
+
+          if File.directory?(subpath)
+            Find.prune if path != subpath && !recursive
+          elsif prepend_path
+            results << File.join(path, subpath)
+          else
+            results << subpath
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/drydock/formatters.rb

@@ -0,0 +1,13 @@
+module Drydock
+  module Formatters
+
+    DELIMITER_PATTERN = /(\d)(?=(\d\d\d)+(?!\d))/
+
+    def self.number(value, delimiter: ',', separator: '.')
+      integers, decimals = value.to_s.split('.')
+      integers.gsub!(DELIMITER_PATTERN) { |digits| "#{digits}#{delimiter}" }
+      [integers, decimals].compact.join(separator)
+    end
+
+  end
+end

data/lib/drydock/ignorefile_definition.rb

@@ -0,0 +1,61 @@
+
+module Drydock
+  class IgnorefileDefinition
+    class Rule < Struct.new(:pattern, :exclude)
+      alias_method :exclude?, :exclude
+
+      def match?(test)
+        File.fnmatch?(pattern, test)
+      end
+    end
+
+    extend Forwardable
+
+    def_delegators :@rules, :count, :length, :size
+
+    def initialize(file_or_filename, dotfiles: false)
+      @dotfiles = dotfiles
+
+      if file_or_filename.respond_to?(:readlines)
+        patterns = Array(file_or_filename.readlines)
+      else
+        patterns = File.exist?(file_or_filename) ? File.readlines(file_or_filename) : []
+      end
+
+      @rules = patterns.map do |pattern|
+        pattern = pattern.chomp
+        if pattern.start_with?('!')
+          Rule.new(pattern.slice(1..-1), true)
+        else
+          Rule.new(pattern, false)
+        end
+      end
+    end
+
+    def match?(filename)
+      return false if excludes?(filename)
+      return true if includes?(filename)
+      return true if is_dotfile?(filename)
+      return false
+    end
+
+    private
+
+    def excludes?(filename)
+      @rules.select { |rule| rule.exclude? }.any? do |rule|
+        rule.match?(filename)
+      end
+    end
+
+    def includes?(filename)
+      @rules.select { |rule| !rule.exclude? }.any? do |rule|
+        rule.match?(filename)
+      end
+    end
+
+    def is_dotfile?(filename)
+      @dotfiles && filename.start_with?('.') && filename.size > 1
+    end
+
+  end
+end

data/lib/drydock/image_repository.rb

@@ -0,0 +1,50 @@
+
+module Drydock
+  class ImageRepository
+
+    class <<self
+      include Enumerable
+    end
+
+    @image_cache = {}
+
+    def self.all
+      image_count = @image_cache.size
+      Docker::Image.all(all: 1).map do |image|
+        @image_cache[image.id] ||= Docker::Image.get(image.id)
+      end
+    ensure
+      delta_count = @image_cache.size - image_count
+      if delta_count > 0
+        Drydock.logger.info(message: "Loaded metadata for #{delta_count} images from docker cache")
+      end
+    end
+
+    def self.dangling
+      filters = {dangling: ["true"]}
+      Docker::Image.all(filters: filters.to_json)
+    end
+
+    def self.each(&blk)
+      self.all.each(&blk)
+    end
+
+    def self.find_by_config(config)
+      base_image = config['Image']
+      candidates = self.select_by_config(config)
+
+      possibles = candidates.select do |image|
+        image.info['Parent'] == base_image || image.info['ContainerConfig']['Image'] == base_image
+      end
+
+      possibles.sort_by { |image| image.info['Created'] }.last
+    end
+
+    def self.select_by_config(config)
+      # we want to look at 'ContainerConfig', because we're interesting in how
+      # the image was built, not how the image will run
+      self.select { |image| config == ContainerConfig.from(image.info['ContainerConfig']) }
+    end
+
+  end
+end