dry-dock 0.1.0

data/lib/drydock/logger.rb

@@ -0,0 +1,61 @@
+
+require 'logger'
+
+module Drydock
+  class Logger < ::Logger
+
+    def add(severity, message = nil, progname = nil, &block)
+      annotation = nil
+      indent     = 0
+
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+          progname = @progname
+        end
+      end
+
+      if message.respond_to?(:key?)
+        indent     = message[:indent].to_i + 1
+        annotation = message.fetch(:annotation, '-->')
+        if message.key?(:message)
+          messages = Array(message[:message])
+        elsif message.key?(:messages)
+          messages = Array(message[:messages])
+        end
+      else
+        messages = [message]
+      end
+
+      annotation << " " if annotation
+      indentation = '    ' * indent
+
+      messages.each do |m|
+        m.to_s.split(/\n/).each do |line|
+          if annotation
+            super(severity, "#{indentation}#{annotation} #{line}", progname)
+          else
+            super(severity, line, progname)
+          end
+        end
+      end
+    end
+
+    alias log add
+
+  end
+
+  class Formatter < ::Logger::Formatter
+
+    def call(severity, time, program, message)
+      "%s [%s] %s\n" % [
+        severity.slice(0, 1),
+        time.strftime('%H:%M:%S'),
+        msg2str(message)
+      ]
+    end
+
+  end
+end

data/lib/drydock/object_caches/base.rb

@@ -0,0 +1,24 @@
+
+module Drydock
+  module ObjectCaches
+    class Base
+
+      def clear
+        raise NotImplementedError, '#clear must be overridden in the subclass'
+      end
+
+      def fetch(key, &blk)
+        raise NotImplementedError, '#fetch must be overridden in the subclass'
+      end
+
+      def get(key, &blk)
+        raise NotImplementedError, '#get must be overridden in the subclass'
+      end
+
+      def set(key, value = nil, &blk)
+        raise NotImplementedError, '#set must be overridden in the subclass'
+      end
+
+    end
+  end
+end

data/lib/drydock/object_caches/filesystem_cache.rb

@@ -0,0 +1,88 @@
+
+require_relative 'base'
+
+module Drydock
+  module ObjectCaches
+    class FilesystemCache < Base
+
+      def initialize(dir = "~/.drydock")
+        @dir = File.expand_path(File.join(dir.to_s, 'cache'))
+        FileUtils.mkdir_p(@dir)
+      end
+
+      def clear
+        begin
+          FileUtils.remove_entry(@dir)
+          true
+        rescue => e
+          Drydock.logger.error("Cannot clear #{self.class} at #{@dir.inspect}: #{e}")
+          return false
+        end
+      end
+
+      def fetch(key, &blk)
+        filename = build_path(key)
+
+        if File.exist?(filename)
+          File.read(filename)
+        else
+          dirname = File.dirname(filename)
+          FileUtils.mkdir_p(dirname)
+
+          blk.call.tap do |contents|
+            File.open(filename, 'w') do |file|
+              file.write contents
+            end
+          end
+        end
+      end
+
+      def get(key, &blk)
+        filename = build_path(key)
+        if File.exist?(filename)
+          if blk.nil?
+            File.read(filename)
+          else
+            File.open(filename) do |file|
+              blk.call file
+            end
+          end
+        else
+          nil
+        end
+      end
+
+      def key?(key)
+        File.exist?(build_path(key))
+      end
+
+      def set(key, value = nil, &blk)
+        filename = build_path(key)
+        dirname = File.dirname(filename)
+        FileUtils.mkdir_p(dirname)
+
+        File.open(filename, 'w') do |file|
+          if blk.nil?
+            file.write value
+          else
+            blk.call file
+          end
+        end
+
+        nil
+      end
+
+      private
+      attr_reader :dir
+
+      def build_path(key)
+        digest   = Digest::SHA2.hexdigest(key)
+        subdir1  = digest.slice(0, 2)
+        subdir2  = digest.slice(2, 2)
+        filename = digest.slice(4, digest.length - 4)
+        File.join(dir, subdir1, subdir2, filename)
+      end
+
+    end
+  end
+end

data/lib/drydock/object_caches/in_memory_cache.rb

@@ -0,0 +1,52 @@
+
+require_relative 'base'
+
+module Drydock
+  module ObjectCaches
+    class InMemoryCache < Base
+
+      def initialize
+        @mem = {}
+      end
+
+      def clear
+        @mem.clear
+        true
+      end
+
+      def fetch(key, &blk)
+        @mem.fetch(key, &blk)
+      end
+
+      def get(key, &blk)
+        if @mem.key?(key)
+          if blk.nil?
+            @mem[key]
+          else
+            blk.call(StringIO.new(@mem[key]))
+          end
+        else
+          nil
+        end
+      end
+
+      def key?(key)
+        @mem.key?(key)
+      end
+
+      def set(key, value = nil, &blk)
+        if blk
+          buffer = StringIO.new
+          blk.call buffer
+          buffer.rewind
+          @mem[key] = buffer.string
+        else
+          @mem[key] = value
+        end
+
+        nil
+      end
+
+    end
+  end
+end

data/lib/drydock/object_caches/no_cache.rb

@@ -0,0 +1,38 @@
+
+require_relative 'base'
+
+module Drydock
+  module ObjectCaches
+    class NoCache < Base
+
+      def clear
+        true
+      end
+
+      def fetch(key, &blk)
+        blk.call
+      end
+
+      def get(key, &blk)
+        nil
+      end
+
+      def key?(key)
+        false
+      end
+
+      def set(key, value = nil, &blk)
+        if blk
+          File.open('/dev/null', 'w') do |file|
+            blk.call file
+          end
+        else
+          # :noop:
+        end
+
+        nil
+      end
+
+    end
+  end
+end

data/lib/drydock/phase.rb

@@ -0,0 +1,50 @@
+
+module Drydock
+  class Phase < Struct.new(:source_image, :build_container, :result_image)
+
+    alias_method :build,   :build_container
+    alias_method :build=,  :build_container=
+
+    alias_method :result,  :result_image
+    alias_method :result=, :result_image=
+
+    alias_method :source,  :source_image
+    alias_method :source=, :source_image=
+
+    def self.from(hsh)
+      h = hsh.to_h
+      extra_keys = h.keys - members
+      raise ArgumentError, "unknown options: #{extra_keys.join(', ')}" unless extra_keys.empty?
+      new(*h.values_at(*members))
+    end
+
+    def built?
+      !cached?
+    end
+
+    def cached?
+      build_container.nil?
+    end
+
+    def destroy!(force: false)
+      if result_image
+        begin
+          result_image.remove(force: force)
+        rescue Docker::Error::NotFoundError => e
+          # Ignore, because the image could have been deleted by another phase in
+          # another derived chain.
+        end
+      end
+
+      build_container.remove(force: force) if built?
+      self
+    end
+
+    def finalize!(force: false)
+      return self unless built?
+      build_container.remove(force: force)
+      self
+    end
+
+  end
+end

data/lib/drydock/phase_chain.rb

@@ -0,0 +1,233 @@
+
+module Drydock
+  class PhaseChain
+    extend Forwardable
+    include Enumerable
+
+    def_delegators :@chain, :<<, :at, :empty?, :last, :length, :push, :size
+
+    def self.build_commit_opts(opts = {})
+      {}.tap do |commit|
+        if opts.key?(:command)
+          commit['run'] = {
+            Cmd: opts[:command]
+          }
+        end
+
+        commit[:author]  = opts.fetch(:author, '')  if opts.key?(:author)
+        commit[:comment] = opts.fetch(:comment, '') if opts.key?(:comment)
+      end
+    end
+
+    def self.build_container_opts(image_id, cmd, opts = {})
+      cmd = ['/bin/sh', '-c', cmd.to_s] unless cmd.is_a?(Array)
+
+      ContainerConfig.from(
+        Cmd: cmd,
+        Tty: opts.fetch(:tty, false),
+        Image: image_id
+      ).tap do |cc|
+        env = Array(opts[:env])
+        cc[:Env].push(*env) unless env.empty?
+
+        if opts.key?(:expose)
+          cc[:ExposedPorts] ||= {}
+          opts[:expose].each do |port|
+            cc[:ExposedPorts][port] = {}
+          end
+        end
+
+        (cc[:OnBuild] ||= []).push(opts[:on_build]) if opts.key?(:on_build)
+      end
+    end
+
+    def self.build_pull_opts(repo, tag = nil)
+      if tag
+        {fromImage: repo, tag: tag}
+      else
+        {fromImage: repo}
+      end
+    end
+
+    def self.create_container(cfg, timeout: nil, &blk)
+      timeout ||= Excon.defaults[:read_timeout]
+
+      Docker::Container.create(cfg).tap do |c|
+        t = Thread.new do
+          begin
+            c.attach(stream: true, stdout: true, stderr: true) do |stream, chunk|
+              case stream
+              when :stdout
+                Drydock.logger.info(message: chunk, annotation: '(O)')
+              when :stderr
+                Drydock.logger.info(message: chunk, annotation: '(E)')
+              else
+                Drydock.logger.info(message: chunk, annotation: '(?)')
+              end
+            end
+          rescue Docker::Error::TimeoutError
+            Drydock.logger.warn(message: "Lost connection to stream; retrying")
+            retry
+          end
+        end
+
+        c.start
+
+        blk.call(c) if blk
+
+        c.wait(timeout)
+        t.join
+      end
+    end
+
+    def self.from_repo(repo, tag = 'latest')
+      new(Docker::Image.create(build_pull_opts(repo, tag)))
+    end
+
+    def initialize(from, parent = nil)
+      @chain  = []
+      @from   = from
+      @parent = parent
+      @children = []
+
+      @ephemeral_containers = []
+
+      if parent
+        parent.children << self
+      end
+    end
+
+    def children
+      @children
+    end
+
+    def containers
+      map(&:build_container)
+    end
+
+    def depth
+      @parent ? @parent.depth + 1 : 1
+    end
+
+    def derive
+      self.class.new(last_image, self)
+    end
+
+    def destroy!(force: false)
+      return self if frozen?
+      children.reverse_each { |c| c.destroy!(force: force) } if children
+      ephemeral_containers.map { |c| c.remove(force: force) }
+
+      reverse_each { |c| c.destroy!(force: force) }
+      freeze
+    end
+
+    def each(&blk)
+      @chain.each(&blk)
+    end
+
+    def ephemeral_containers
+      @ephemeral_containers
+    end
+
+    def finalize!(force: false)
+      return self if frozen?
+
+      children.map { |c| c.finalize!(force: force) } if children
+      ephemeral_containers.map { |c| c.remove(force: force) }
+
+      Drydock.logger.info("##{serial}: Final image ID is #{last_image.id}") unless empty?
+      map { |p| p.finalize!(force: force) }
+      freeze
+    end
+
+    def images
+      [root_image] + map(&:result_image)
+    end
+
+    def last_image
+      @chain.last ? @chain.last.result_image : nil
+    end
+
+    def root_image
+      @from
+    end
+
+    def run(cmd, opts = {}, &blk)
+      src_image = last ? last.result_image : @from
+      no_commit = opts.fetch(:no_commit, false)
+
+      no_cache = opts.fetch(:no_cache, false)
+      no_cache = true if no_commit
+
+      build_config = self.class.build_container_opts(src_image.id, cmd, opts)
+      Drydock.logger.info(build_config.inspect)
+      cached_image = ImageRepository.find_by_config(build_config)
+
+      if cached_image && !no_cache
+        Drydock.logger.info(message: "Using cached image ID #{cached_image.id.slice(0, 12)}")
+
+        if no_commit
+          Drydock.logger.info(message: "Skipping commit phase")
+        else
+          self << Phase.from(
+            source_image: src_image,
+            result_image: cached_image
+          )
+        end
+      else
+        if cached_image && no_commit
+          Drydock.logger.info(message: "Found cached image ID #{cached_image.id.slice(0, 12)}, but skipping due to :no_commit")
+        elsif cached_image && no_cache
+          Drydock.logger.info(message: "Found cached image ID #{cached_image.id.slice(0, 12)}, but skipping due to :no_cache")
+        end
+
+        container = self.class.create_container(build_config)
+        yield container if block_given?
+
+        if no_commit
+          Drydock.logger.info(message: "Skipping commit phase")
+          ephemeral_containers << container
+        else
+          if opts.key?(:command)
+            Drydock.logger.info("Command override: #{opts[:command].inspect}")
+          else
+            src_image.refresh!
+            if src_image.info && src_image.info.key?('Config')
+              src_image_config = src_image.info['Config']
+              opts[:command]   = src_image_config['Cmd'] if src_image_config.key?('Cmd')
+            end
+
+            Drydock.logger.info("Command retrieval: #{opts[:command].inspect}")
+            Drydock.logger.info("Source image info: #{src_image.info.class} #{src_image.info.inspect}")
+            Drydock.logger.info("Source image config: #{src_image.info['Config'].inspect}")
+          end
+
+          commit_config = self.class.build_commit_opts(opts)
+          Drydock.logger.info(opts.inspect)
+          Drydock.logger.info(commit_config.inspect)
+
+          result = container.commit(commit_config)
+          Drydock.logger.info(message: "Committed image ID #{result.id.slice(0, 12)}")
+
+          self << Phase.from(
+            source_image:    src_image,
+            build_container: container,
+            result_image:    result
+          )
+        end
+      end
+
+      self
+    end
+
+    def serial
+      @parent ? "#{@parent.serial}.#{@parent.children.index(self) + 1}.#{size + 1}" : "#{size + 1}"
+    end
+
+    def tag(repo, tag = 'latest', force: false)
+      last_image.tag(repo: repo, tag: tag, force: force)
+    end
+
+  end
+end

data/lib/drydock/plugins/apk.rb

@@ -0,0 +1,31 @@
+
+require_relative 'package_manager'
+
+module Drydock
+  module Plugins
+    class APK < PackageManager
+
+      def add(*pkgs)
+        opts = pkgs.last.is_a?(Hash) ? pkgs.pop : {}
+        project.run "apk add #{pkgs.join(' ')}", opts
+      end
+
+      def clean
+        project.run "rm -rf /var/cache/apk/*"
+      end
+
+      def remove(*pkgs)
+        project.run "apk del #{pkgs.join(' ')}"
+      end
+
+      def update
+        project.run "apk update"
+      end
+
+      def upgrade
+        project.run "apk upgrade"
+      end
+
+    end
+  end
+end

data/lib/drydock/plugins/base.rb

@@ -0,0 +1,15 @@
+
+module Drydock
+  module Plugins
+    class Base
+
+      attr_reader :project
+
+      def initialize(project)
+        @project = project
+      end
+
+    end
+  end
+end
+

data/lib/drydock/plugins/npm.rb

@@ -0,0 +1,16 @@
+
+require_relative 'base'
+
+module Drydock
+  module Plugins
+    class NPM < Base
+
+      def install(*pkgs)
+        opts = pkgs.last.is_a?(Hash) ? pkgs.pop : {}
+        flags = CliFlags.new(opts)
+        project.run("npm install #{flags}#{pkgs.join(' ')}")
+      end
+
+    end
+  end
+end

data/lib/drydock/plugins/package_manager.rb

@@ -0,0 +1,30 @@
+
+require_relative 'base'
+
+module Drydock
+  module Plugins
+    class PackageManager < Base
+
+      def add(*pkgs)
+        raise NotImplementedError, '#add must be overridde in the subclass'
+      end
+
+      def clean
+        raise NotImplementedError, '#clean must be overridde in the subclass'
+      end
+
+      def remove(*pkgs)
+        raise NotImplementedError, '#remove must be overridde in the subclass'
+      end
+
+      def update
+        raise NotImplementedError, '#update must be overridde in the subclass'
+      end
+
+      def upgrade
+        raise NotImplementedError, '#upgrade must be overridde in the subclass'
+      end
+
+    end
+  end
+end

data/lib/drydock/plugins/rubygems.rb

@@ -0,0 +1,30 @@
+
+require_relative 'base'
+
+module Drydock
+  module Plugins
+    class Rubygems < Base
+
+      def add_source(uri)
+        project.run("gem sources --add #{uri}")
+      end
+
+      def install(pkg, opts = {})
+        timeout = opts.delete(:timeout) || 120
+        flags   = CliFlags.new(opts)
+        project.run("gem install #{pkg} #{flags}", timeout: timeout)
+      end
+
+      def remove_source(uri)
+        project.run("gem sources --remove #{uri}")
+      end
+
+      def update_system(opts = {})
+        timeout = opts.delete(:timeout) || 300
+        flags   = CliFlags.new(opts)
+        project.run("gem update --system #{flags}", timeout: timeout)
+      end
+
+    end
+  end
+end