hoosegow 1.2.0

data/lib/hoosegow/docker.rb

@@ -0,0 +1,236 @@
+require 'yajl'
+require 'docker'
+require 'stringio'
+
+require_relative 'exceptions'
+
+class Hoosegow
+  # Minimal API client for Docker, allowing attaching to container
+  # stdin/stdout/stderr.
+  class Docker
+    ::Docker.options[:read_timeout]  = 3600
+    ::Docker.options[:write_timeout] = 3600
+    DEFAULT_HOST   = "127.0.0.1"
+    DEFAULT_PORT   = 4243
+    DEFAULT_SOCKET = "/var/run/docker.sock"
+
+    # Initialize a new Docker API client.
+    #
+    # options - Connection options.
+    #           :host   - IP or hostname to connect to (unless using Unix
+    #                     socket).
+    #           :port   - TCP port to connect to (unless using Unix socket).
+    #           :socket - Path to local Unix socket (unless using host and
+    #                     port).
+    #           :after_create - A proc that will be called after a container is created.
+    #           :after_start  - A proc that will be called after a container is started.
+    #           :after_stop   - A proc that will be called after a container stops.
+    #           :prestart - Start a new container after each `run_container` call.
+    #           :volumes  - A mapping of volumes to mount in the container. e.g.
+    #                       if the Dockerfile has `VOLUME /work`, where the container will
+    #                       write data, and `VOLUME /config` where read-only configuration
+    #                       is, you might use
+    #                         :volumes => {
+    #                           "/config" => "/etc/shared-config",
+    #                           "/work"   => "/data/work:rw",
+    #                         }
+    #                       `:volumes => { "/work" => "/home/localuser/work/to/do" }`
+    #           :Other - any option with a capitalized key will be passed on
+    #                    to the 'create container' call. See http://docs.docker.io/en/latest/reference/api/docker_remote_api_v1.9/#create-a-container
+    def initialize(options = {})
+      ::Docker.url       = docker_url options
+      @after_create      = options[:after_create]
+      @after_start       = options[:after_start]
+      @after_stop        = options[:after_stop]
+      @volumes           = options[:volumes]
+      @prestart          = options.fetch(:prestart, true)
+      @container_options = options.select { |k,v| k =~ /\A[A-Z]/ }
+    end
+
+    # Public: Create and start a Docker container if one hasn't been started
+    # already, then attach to it its stdin/stdout.
+    #
+    # image    - The image to run.
+    # data     - The data to pipe to the container's stdin.
+    #
+    # Returns the data from the container's stdout.
+    def run_container(image, data, &block)
+      unless @prestart && @container
+        create_container(image)
+        start_container
+      end
+
+      begin
+        attach_container(data, &block)
+      ensure
+        wait_container
+        delete_container
+        if @prestart
+          create_container(image)
+          start_container
+        end
+      end
+      nil
+    end
+
+    # Public: Create a container using the specified image.
+    #
+    # image - The name of the image to start the container with.
+    #
+    # Returns nothing.
+    def create_container(image)
+      @container = ::Docker::Container.create @container_options.merge(
+        :StdinOnce => true,
+        :OpenStdin => true,
+        :Volumes   => volumes_for_create,
+        :Image     => image
+      )
+      callback @after_create
+    end
+
+    # Public: Start a Docker container.
+    #
+    # Returns nothing.
+    def start_container
+      @container.start :Binds => volumes_for_bind
+      callback @after_start
+    end
+
+    # Attach to a container, writing data to container's STDIN.
+    #
+    # Returns combined STDOUT/STDERR from container.
+    def attach_container(data, &block)
+      stdin = StringIO.new data
+      @container.attach :stdin => stdin, &block
+    end
+
+    # Public: Wait for a container to finish.
+    #
+    # Returns nothing.
+    def wait_container
+      @container.wait
+      callback @after_stop
+    end
+
+    # Public: Stop the running container.
+    #
+    # Returns response body or nil if no container is running.
+    def stop_container
+      return unless @container
+      @container.stop :timeout => 0
+      callback @after_stop
+    end
+
+    # Public: Delete the last started container.
+    #
+    # Returns response body or nil if no container was started.
+    def delete_container
+      return unless @container
+      @container.delete
+    end
+
+    # Public: Build a new image.
+    #
+    # name    - The name to give the image.
+    # tarfile - Tarred data for creating image. See http://docs.docker.io/en/latest/api/docker_remote_api_v1.5/#build-an-image-from-dockerfile-via-stdin
+    #
+    # Returns Array of build result objects from the Docker API.
+    def build_image(name, tarfile)
+      # Setup parser to receive chunks and yield parsed JSON objects.
+      ret = []
+      error = nil
+      parser = Yajl::Parser.new
+      parser.on_parse_complete = Proc.new do |obj|
+        ret << obj
+        error = Hoosegow::ImageBuildError.new(obj) if obj["error"]
+        yield obj if block_given?
+      end
+
+      # Make API call to create image.
+      opts = {:t => name, :rm => '1'}
+      ::Docker::Image.build_from_tar StringIO.new(tarfile), opts do |chunk|
+        parser << chunk
+      end
+
+      raise error if error
+
+      # Return Array of received objects.
+      ret
+    end
+
+    # Check if a Docker image exists.
+    #
+    # name - The name of the image to check for.
+    #
+    # Returns true/false.
+    def image_exist?(name)
+      ::Docker::Image.exist? name
+    end
+
+  private
+    # Private: Get the URL to use for communicating with Docker. If a host and/or
+    # port a present, a TCP socket URL will be generated. Otherwise a Unix
+    # socket will be used.
+    #
+    # options - A Hash of options for building the URL.
+    #           :host   - The hostname or IP of a remote Docker daemon
+    #                     (optional).
+    #           :port   - The TCP port of the remote Docker daemon (optional).
+    #           :socket - The path of a local Unix socket (optional).
+    #
+    # Returns a String url.
+    def docker_url(options)
+      if options[:host] || options[:port]
+        host = options[:host] || DEFAULT_HOST
+        port = options[:port] || DEFAULT_PORT
+        "tcp://#{host}:#{port}"
+      else
+        path = options[:socket] || DEFAULT_SOCKET
+        "unix://#{path}"
+      end
+    end
+
+    # Private: Generate the `Volumes` argument for creating a container.
+    #
+    # Given a hash of container_path => local_path in @volumes, generate a
+    # hash of container_path => {}.
+    def volumes_for_create
+      result = {}
+      each_volume do |container_path, local_path, permissions|
+        result[container_path] = {}
+      end
+      result
+    end
+
+    # Private: Generate the `Binds` argument for starting a container.
+    #
+    # Given a hash of container_path => local_path in @volumes, generate an
+    # array of "local_path:container_path:rw".
+    def volumes_for_bind
+      result = []
+      each_volume do |container_path, local_path, permissions|
+        result << "#{local_path}:#{container_path}:#{permissions}"
+      end
+      result
+    end
+
+    # Private: Yields information about each `@volume`.
+    #
+    #   each_volume do |container_path, local_path, permissions|
+    #   end
+    def each_volume
+      if @volumes
+        @volumes.each do |container_path, local_path|
+          local_path, permissions = local_path.split(':', 2)
+          permissions ||= "ro"
+          yield container_path, local_path, permissions
+        end
+      end
+    end
+
+    def callback(callback_proc)
+      callback_proc.call(@container.info) if callback_proc
+    rescue Object
+    end
+  end
+end

data/lib/hoosegow/exceptions.rb

@@ -0,0 +1,27 @@
+class Hoosegow
+  # General error for others to inherit from.
+  class Error < StandardError; end
+
+  # Errors while building the Docker image.
+  class ImageBuildError < Error
+    def initialize(message)
+      if message.is_a?(Hash)
+        @detail = message['errorDetail']
+        message = message['error']
+      end
+      super(message)
+    end
+
+    # The error details from docker.
+    #
+    # Example:
+    #     {"code" => 127, "message" => "The command [/bin/sh -c boom] returned a non-zero code: 127"}
+    attr_reader :detail
+  end
+
+  # Errors while importing dependencies
+  class InmateImportError < Error; end
+
+  # Errors while running an inmate
+  class InmateRuntimeError < Error; end
+end

data/lib/hoosegow/image_bundle.rb

@@ -0,0 +1,111 @@
+require 'fileutils'
+
+class Hoosegow
+  class ImageBundle
+    # Public: The source for the Dockerfile. Defaults to Dockerfile in the hoosegow gem.
+    attr_accessor :dockerfile
+
+    # Public: The ruby version to install on the container.
+    attr_accessor :ruby_version
+
+    # Public: Include files in the bundle.
+    #
+    # To add all files in "root" to the root of the bundle:
+    #     add("root/*")
+    #
+    # To add all files other than files that start with "." to the root of the bundle:
+    #     add("root/*", :ignore_hidden => true)
+    #
+    # To add all files in "lib" to "vendor/lib":
+    #     add("lib/*", :prefix => "vendor/lib")
+    #     add("lib", :prefix => "vendor")
+    def add(glob, options)
+      definition << options.merge(:glob => glob)
+    end
+
+    # Public: Exclude files from the bundle.
+    #
+    # To exclude "Gemfile.lock":
+    #     exclude("Gemfile.lock")
+    def exclude(path)
+      excludes << path
+    end
+
+    # Public: The default name of the docker image, based on the tarball's hash.
+    def image_name
+      (tarball && @image_name)
+    end
+
+    # Tarball of this gem and the inmate file. Used for building an image.
+    #
+    # Returns the tar file's bytes.
+    def tarball
+      return @tarball if defined? @tarball
+
+      require 'open3'
+      Dir.mktmpdir do |tmpdir|
+        definition.each do |options|
+          glob          = options.fetch(:glob)
+          prefix        = options[:prefix]
+          ignore_hidden = options[:ignore_hidden]
+
+          files = Dir[glob]
+          files.reject! { |f| f.start_with?('.') } if ignore_hidden
+
+          dest = prefix ? File.join(tmpdir, prefix) : tmpdir
+
+          FileUtils.mkpath(dest)
+          FileUtils.cp_r(files, dest)
+        end
+
+        excludes.each do |path|
+          full_path = File.join(tmpdir, path)
+          if File.file?(full_path)
+            File.unlink(File.join(tmpdir, path))
+          end
+        end
+
+        # Specify the correct ruby version in the Dockerfile.
+        bundle_dockerfile = File.join(tmpdir, "Dockerfile")
+        content = IO.read(bundle_dockerfile)
+        content = content.gsub("{{ruby_version}}", ruby_version)
+        IO.write bundle_dockerfile, content
+
+        if dockerfile
+          File.unlink bundle_dockerfile
+          FileUtils.cp dockerfile, bundle_dockerfile
+        end
+
+        # Find hash of all files we're sending over.
+        digest = Digest::SHA1.new
+        Dir[File.join(tmpdir, '**/*')].each do |path|
+          if File.file? path
+            open path, 'r' do |file|
+              digest.update file.read
+            end
+          end
+        end
+        @image_name = "hoosegow:#{digest.hexdigest}"
+
+        # Create tarball of the tmpdir.
+        stdout, stderr, status = Open3.capture3 'tar', '-c', '-C', tmpdir, '.'
+
+        raise Hoosegow::ImageBuildError, stderr unless stderr.empty?
+
+        @tarball = stdout
+      end
+    end
+
+    private
+
+    # The things to include in the tarfile.
+    def definition
+      @definition ||= []
+    end
+
+    # The things to exclude from the tarfile
+    def excludes
+      @excludes ||= []
+    end
+  end
+end

data/lib/hoosegow/protocol.rb

@@ -0,0 +1,133 @@
+require 'msgpack'
+require 'thread'
+
+class Hoosegow
+  # See docs/dispatch.md for more information.
+  module Protocol
+    # Hoosegow, on the app side of the proxy.
+    #
+    # Sends data to and from an inmate, via a Docker container running `bin/hoosegow`.
+    class Proxy
+      # Options:
+      # * (optional) :yield - a block to call when the inmate yields
+      # * (optional) :stdout - an IO for writing STDOUT from the inmate
+      # * (optional) :stderr - an IO for writing STDERR from the inmate
+      def initialize(options)
+        @yield_block = options.fetch(:yield, nil)
+        @stdout      = options.fetch(:stdout, $stdout)
+        @stderr      = options.fetch(:stderr, $stderr)
+      end
+
+      # Encodes a "send" method call for an inmate.
+      def encode_send(method_name, args)
+        MessagePack.pack([method_name, args])
+      end
+
+      # The return value
+      attr_reader :return_value
+
+      # Decodes a message from an inmate via docker.
+      def receive(type, msg)
+        if type == :stdout
+          @unpacker ||= MessagePack::Unpacker.new
+          @unpacker.feed_each(msg) do |decoded|
+            inmate_type, inmate_value = decoded
+            case inmate_type.to_s
+            when 'yield'
+              @yield_block.call(*inmate_value) if @yield_block
+            when 'return'
+              @return_value = inmate_value
+            when 'raise'
+              raise(*raise_args(inmate_value))
+            when 'stdout'
+              @stdout.write(inmate_value)
+            end
+          end
+        elsif type == :stderr
+          @stderr.write(msg)
+        end
+      end
+
+      def raise_args(remote_error)
+        exception_class = Hoosegow::InmateRuntimeError
+        exception_message = "#{remote_error['class']}: #{remote_error['message']}"
+        if remote_backtrace = remote_error['backtrace']
+          exception_message << ("\n" + Array(remote_backtrace).join("\n"))
+        end
+        [exception_class, exception_message]
+      end
+    end
+
+    # bin/hoosegow client (where the inmate code runs)
+    #
+    # Translates stdin into a method call on on inmate.
+    # Encodes yields and the return value onto a stream.
+    class Inmate
+      def self.run(options)
+        o = new(options)
+        o.intercepting do
+          o.run
+        end
+      end
+
+      # Options:
+      # * :stdout - real stdout, where we can write things that our parent process will see
+      # * :intercepted - where this process or child processes write STDOUT to
+      # * (optional) :inmate - the hoosegow instance to use as the inmate.
+      # * (optional) :stdin - where to read the encoded method call data.
+      def initialize(options)
+        @inmate       = options.fetch(:inmate) { Hoosegow.new(:no_proxy => true) }
+        @stdin        = options.fetch(:stdin, $stdin)
+        @stdout       = options.fetch(:stdout)
+        @intercepted  = options.fetch(:intercepted)
+        @stdout_mutex = Mutex.new
+      end
+
+      def run
+        name, args = MessagePack::Unpacker.new(@stdin).read
+        result = @inmate.send(name, *args) do |*yielded|
+          report(:yield, yielded)
+          nil # Don't return anything from the inmate's `yield`.
+        end
+        report(:return, result)
+      rescue => e
+        report(:raise, {:class => e.class.name, :message => e.message, :backtrace => e.backtrace})
+      end
+
+      def intercepting
+        start_intercepting
+        yield
+      ensure
+        stop_intercepting
+      end
+
+      def start_intercepting
+        @intercepting = true
+        @intercept_thread = Thread.new do
+          begin
+            loop do
+              if IO.select([@intercepted], nil, nil, 0.1)
+                report(:stdout, @intercepted.read_nonblock(100000))
+              elsif ! @intercepting
+                break
+              end
+            end
+          rescue EOFError
+            # stdout is closed, so we can stop checking it.
+          end
+        end
+      end
+
+      def stop_intercepting
+        @intercepting = false
+        @intercept_thread.join
+      end
+
+      private
+
+      def report(type, data)
+        @stdout_mutex.synchronize { @stdout.write(MessagePack.pack([type, data])) }
+      end
+    end
+  end
+end