docker_toolkit 0.1.2 → 0.1.3

data/lib/docker_toolkit/childprocess/tools/generator.rb

@@ -0,0 +1,148 @@
+require 'fileutils'
+
+module DockerToolkit
+module ChildProcess
+  module Tools
+    class Generator
+      EXE_NAME         = "childprocess-sizeof-generator"
+      TMP_PROGRAM      = "childprocess-sizeof-generator.c"
+      DEFAULT_INCLUDES = %w[stdio.h stddef.h]
+
+      def self.generate
+        new.generate
+      end
+
+      def initialize
+        @cc        = ENV['CC'] || 'gcc'
+        @out       = File.expand_path("../../unix/platform/#{ChildProcess.platform_name}.rb", __FILE__)
+        @sizeof    = {}
+        @constants = {}
+      end
+
+      def generate
+        fetch_size 'posix_spawn_file_actions_t', :include => "spawn.h"
+        fetch_size 'posix_spawnattr_t', :include => "spawn.h"
+        fetch_size 'sigset_t', :include => "signal.h"
+
+        fetch_constant 'POSIX_SPAWN_RESETIDS',   :include  => 'spawn.h'
+        fetch_constant 'POSIX_SPAWN_SETPGROUP',  :include  => 'spawn.h'
+        fetch_constant 'POSIX_SPAWN_SETSIGDEF',  :include  => 'spawn.h'
+        fetch_constant 'POSIX_SPAWN_SETSIGMASK', :include  => 'spawn.h'
+
+        if ChildProcess.linux?
+          fetch_constant 'POSIX_SPAWN_USEVFORK', :include => 'spawn.h', :define => {'_GNU_SOURCE' => nil}
+        end
+
+        write
+      end
+
+      def write
+        FileUtils.mkdir_p(File.dirname(@out))
+        File.open(@out, 'w') do |io|
+          io.puts result
+        end
+
+        puts "wrote #{@out}"
+      end
+
+      def fetch_size(type_name, opts = {})
+        print "sizeof(#{type_name}): "
+        src = <<-EOF
+int main() {
+  printf("%d", (unsigned int)sizeof(#{type_name}));
+  return 0;
+}
+        EOF
+
+        output = execute(src, opts)
+
+        if output.to_i < 1
+          raise "sizeof(#{type_name}) == #{output.to_i} (output=#{output})"
+        end
+
+        size = output.to_i
+        @sizeof[type_name] = size
+
+        puts size
+      end
+
+      def fetch_constant(name, opts)
+        print "#{name}: "
+        src = <<-EOF
+int main() {
+  printf("%d", (unsigned int)#{name});
+  return 0;
+}
+        EOF
+
+        output = execute(src, opts)
+        value  = Integer(output)
+        @constants[name] = value
+
+        puts value
+      end
+
+
+      def execute(src, opts)
+        program = Array(opts[:define]).map do |key, value|
+          <<-SRC
+#ifndef #{key}
+#define #{key} #{value}
+#endif
+          SRC
+        end.join("\n")
+        program << "\n"
+
+        includes = Array(opts[:include]) + DEFAULT_INCLUDES
+        program << includes.map { |include| "#include <#{include}>" }.join("\n")
+        program << "\n#{src}"
+
+        File.open(TMP_PROGRAM, 'w') do |file|
+          file << program
+        end
+
+        cmd = "#{@cc} #{TMP_PROGRAM} -o #{EXE_NAME}"
+        system cmd
+        unless $?.success?
+          raise "failed to compile program: #{cmd.inspect}\n#{program}"
+        end
+
+        output = `./#{EXE_NAME} 2>&1`
+
+        unless $?.success?
+          raise "failed to run program: #{cmd.inspect}\n#{output}"
+        end
+
+        output.chomp
+      ensure
+        File.delete TMP_PROGRAM if File.exist?(TMP_PROGRAM)
+        File.delete EXE_NAME if File.exist?(EXE_NAME)
+      end
+
+      def result
+        if @sizeof.empty? && @constants.empty?
+          raise "no data collected, nothing to do"
+        end
+
+        out =  ['module ChildProcess::Unix::Platform']
+        out << '  SIZEOF = {'
+
+        max = @sizeof.keys.map { |e| e.length }.max
+        @sizeof.each_with_index do |(type, size), idx|
+          out << "     :#{type.ljust max} => #{size}#{',' unless idx == @sizeof.size - 1}"
+        end
+        out << '  }'
+
+        max = @constants.keys.map { |e| e.length }.max
+        @constants.each do |name, val|
+          out << "  #{name.ljust max} = #{val}"
+        end
+        out << 'end'
+
+        out.join "\n"
+      end
+
+    end
+  end
+end
+end

data/lib/docker_toolkit/childprocess/unix/fork_exec_process.rb

@@ -0,0 +1,72 @@
+module DockerToolkit
+module ChildProcess
+  module Unix
+    class ForkExecProcess < Process
+      private
+
+      def launch_process
+        if @io
+          stdout = @io.stdout
+          stderr = @io.stderr
+        end
+
+        # pipe used to detect exec() failure
+        exec_r, exec_w = ::IO.pipe
+        ChildProcess.close_on_exec exec_w
+
+        if duplex?
+          reader, writer = ::IO.pipe
+        end
+
+        @pid = Kernel.fork {
+          # Children of the forked process will inherit its process group
+          # This is to make sure that all grandchildren dies when this Process instance is killed
+          ::Process.setpgid 0, 0 if leader?
+
+          if @cwd
+            Dir.chdir(@cwd)
+          end
+
+          exec_r.close
+          set_env
+
+          STDOUT.reopen(stdout || "/dev/null")
+          STDERR.reopen(stderr || "/dev/null")
+
+          if duplex?
+            STDIN.reopen(reader)
+            writer.close
+          end
+
+          executable, *args = @args
+
+          begin
+            Kernel.exec([executable, executable], *args)
+          rescue SystemCallError => ex
+            exec_w << ex.message
+          end
+        }
+
+        exec_w.close
+
+        if duplex?
+          io._stdin = writer
+          reader.close
+        end
+
+        # if we don't eventually get EOF, exec() failed
+        unless exec_r.eof?
+          raise LaunchError, exec_r.read || "executing command with #{@args.inspect} failed"
+        end
+
+        ::Process.detach(@pid) if detach?
+      end
+
+      def set_env
+        @environment.each { |k, v| ENV[k.to_s] = v.nil? ? nil : v.to_s }
+      end
+
+    end # Process
+  end # Unix
+end # ChildProcess
+end

data/lib/docker_toolkit/childprocess/unix/io.rb

@@ -0,0 +1,22 @@
+module DockerToolkit
+module ChildProcess
+  module Unix
+    class IO < AbstractIO
+      private
+
+      def check_type(io)
+        unless io.respond_to? :to_io
+          raise ArgumentError, "expected #{io.inspect} to respond to :to_io"
+        end
+
+        result = io.to_io
+        unless result && result.kind_of?(::IO)
+          raise TypeError, "expected IO, got #{result.inspect}:#{result.class}"
+        end
+      end
+
+    end # IO
+  end # Unix
+end # ChildProcess
+end
+

data/lib/docker_toolkit/childprocess/unix/lib.rb

@@ -0,0 +1,188 @@
+module DockerToolkit
+module ChildProcess
+  module Unix
+    module Lib
+      extend FFI::Library
+      ffi_lib FFI::Library::LIBC
+
+      if ChildProcess.os == :macosx
+        attach_function :_NSGetEnviron, [], :pointer
+        def self.environ
+          _NSGetEnviron().read_pointer
+        end
+      elsif respond_to? :attach_variable
+        attach_variable :environ, :pointer
+      end
+
+      attach_function :strerror, [:int], :string
+      attach_function :chdir, [:string], :int
+      attach_function :fcntl, [:int, :int, :int], :int # fcntl actually takes varags, but we only need this version.
+
+      # int posix_spawnp(
+      #   pid_t *restrict pid,
+      #   const char *restrict file,
+      #   const posix_spawn_file_actions_t *file_actions,
+      #   const posix_spawnattr_t *restrict attrp,
+      #   char *const argv[restrict],
+      #   char *const envp[restrict]
+      # );
+
+      attach_function :posix_spawnp, [
+        :pointer,
+        :string,
+        :pointer,
+        :pointer,
+        :pointer,
+        :pointer
+      ], :int
+
+      # int posix_spawn_file_actions_init(posix_spawn_file_actions_t *file_actions);
+      attach_function :posix_spawn_file_actions_init, [:pointer], :int
+
+      # int posix_spawn_file_actions_destroy(posix_spawn_file_actions_t *file_actions);
+      attach_function :posix_spawn_file_actions_destroy, [:pointer], :int
+
+      # int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *file_actions, int filedes);
+      attach_function :posix_spawn_file_actions_addclose, [:pointer, :int], :int
+
+      # int posix_spawn_file_actions_addopen(
+      #   posix_spawn_file_actions_t *restrict file_actions,
+      #   int filedes,
+      #   const char *restrict path,
+      #   int oflag,
+      #   mode_t mode
+      # );
+      attach_function :posix_spawn_file_actions_addopen, [:pointer, :int, :string, :int, :mode_t], :int
+
+      # int posix_spawn_file_actions_adddup2(
+      #   posix_spawn_file_actions_t *file_actions,
+      #   int filedes,
+      #   int newfiledes
+      # );
+      attach_function :posix_spawn_file_actions_adddup2, [:pointer, :int, :int], :int
+
+      # int posix_spawnattr_init(posix_spawnattr_t *attr);
+      attach_function :posix_spawnattr_init, [:pointer], :int
+
+      # int posix_spawnattr_destroy(posix_spawnattr_t *attr);
+      attach_function :posix_spawnattr_destroy, [:pointer], :int
+
+      # int posix_spawnattr_setflags(posix_spawnattr_t *attr, short flags);
+      attach_function :posix_spawnattr_setflags, [:pointer, :short], :int
+
+      # int posix_spawnattr_getflags(const posix_spawnattr_t *restrict attr, short *restrict flags);
+      attach_function :posix_spawnattr_getflags, [:pointer, :pointer], :int
+
+      # int posix_spawnattr_setpgroup(posix_spawnattr_t *attr, pid_t pgroup);
+      attach_function :posix_spawnattr_setpgroup, [:pointer, :pid_t], :int
+
+      # int posix_spawnattr_getpgroup(const posix_spawnattr_t *restrict attr, pid_t *restrict pgroup);
+      attach_function :posix_spawnattr_getpgroup, [:pointer, :pointer], :int
+
+      # int posix_spawnattr_setsigdefault(posix_spawnattr_t *restrict attr, const sigset_t *restrict sigdefault);
+      attach_function :posix_spawnattr_setsigdefault, [:pointer, :pointer], :int
+
+      # int posix_spawnattr_getsigdefault(const posix_spawnattr_t *restrict attr, sigset_t *restrict sigdefault);
+      attach_function :posix_spawnattr_getsigdefault, [:pointer, :pointer], :int
+
+      # int posix_spawnattr_setsigmask(posix_spawnattr_t *restrict attr, const sigset_t *restrict sigmask);
+      attach_function :posix_spawnattr_setsigmask, [:pointer, :pointer], :int
+
+      # int posix_spawnattr_getsigmask(const posix_spawnattr_t *restrict attr, sigset_t *restrict sigmask);
+      attach_function :posix_spawnattr_getsigmask, [:pointer, :pointer], :int
+
+      def self.check(errno)
+        if errno != 0
+          raise Error, Lib.strerror(FFI.errno)
+        end
+      end
+
+      class FileActions
+        def initialize
+          @ptr = FFI::MemoryPointer.new(1, Platform::SIZEOF.fetch(:posix_spawn_file_actions_t), false)
+          Lib.check Lib.posix_spawn_file_actions_init(@ptr)
+        end
+
+        def add_close(fileno)
+          Lib.check Lib.posix_spawn_file_actions_addclose(
+            @ptr,
+            fileno
+          )
+        end
+
+        def add_open(fileno, path, oflag, mode)
+          Lib.check Lib.posix_spawn_file_actions_addopen(
+            @ptr,
+            fileno,
+            path,
+            oflag,
+            mode
+          )
+        end
+
+        def add_dup(fileno, new_fileno)
+          Lib.check Lib.posix_spawn_file_actions_adddup2(
+            @ptr,
+            fileno,
+            new_fileno
+          )
+        end
+
+        def free
+          Lib.check Lib.posix_spawn_file_actions_destroy(@ptr)
+          @ptr = nil
+        end
+
+        def to_ptr
+          @ptr
+        end
+      end # FileActions
+
+      class Attrs
+        def initialize
+          @ptr = FFI::MemoryPointer.new(1, Platform::SIZEOF.fetch(:posix_spawnattr_t), false)
+          Lib.check Lib.posix_spawnattr_init(@ptr)
+        end
+
+        def free
+          Lib.check Lib.posix_spawnattr_destroy(@ptr)
+          @ptr = nil
+        end
+
+        def flags=(flags)
+          Lib.check Lib.posix_spawnattr_setflags(@ptr, flags)
+        end
+
+        def flags
+          ptr = FFI::MemoryPointer.new(:short)
+          Lib.check Lib.posix_spawnattr_getflags(@ptr, ptr)
+
+          ptr.read_short
+        end
+
+        def pgroup=(pid)
+          self.flags |= Platform::POSIX_SPAWN_SETPGROUP
+          Lib.check Lib.posix_spawnattr_setpgroup(@ptr, pid)
+        end
+
+        def to_ptr
+          @ptr
+        end
+      end # Attrs
+
+    end
+  end
+end
+end
+
+# missing on rubinius
+class FFI::MemoryPointer
+  unless method_defined?(:from_string)
+    def self.from_string(str)
+      ptr = new(1, str.bytesize + 1)
+      ptr.write_string("#{str}\0")
+
+      ptr
+    end
+  end
+end if defined?(FFI)

data/lib/docker_toolkit/childprocess/unix/platform/i386-linux.rb

@@ -0,0 +1,14 @@
+module DockerToolkit
+module ChildProcess::Unix::Platform
+  SIZEOF = {
+     :posix_spawn_file_actions_t => 76,
+     :posix_spawnattr_t          => 336,
+     :sigset_t                   => 128
+  }
+  POSIX_SPAWN_RESETIDS   = 1
+  POSIX_SPAWN_SETPGROUP  = 2
+  POSIX_SPAWN_SETSIGDEF  = 4
+  POSIX_SPAWN_SETSIGMASK = 8
+  POSIX_SPAWN_USEVFORK   = 64
+end
+end

data/lib/docker_toolkit/childprocess/unix/platform/i386-solaris.rb

@@ -0,0 +1,13 @@
+module DockerToolkit
+module ChildProcess::Unix::Platform
+  SIZEOF = {
+     :posix_spawn_file_actions_t => 4,
+     :posix_spawnattr_t          => 4,
+     :sigset_t                   => 16
+  }
+  POSIX_SPAWN_RESETIDS   = 1
+  POSIX_SPAWN_SETPGROUP  = 2
+  POSIX_SPAWN_SETSIGDEF  = 4
+  POSIX_SPAWN_SETSIGMASK = 8
+end
+end

data/lib/docker_toolkit/childprocess/unix/platform/x86_64-linux.rb

@@ -0,0 +1,14 @@
+module DockerToolkit
+module ChildProcess::Unix::Platform
+  SIZEOF = {
+     :posix_spawn_file_actions_t => 80,
+     :posix_spawnattr_t          => 336,
+     :sigset_t                   => 128
+  }
+  POSIX_SPAWN_RESETIDS   = 1
+  POSIX_SPAWN_SETPGROUP  = 2
+  POSIX_SPAWN_SETSIGDEF  = 4
+  POSIX_SPAWN_SETSIGMASK = 8
+  POSIX_SPAWN_USEVFORK   = 64
+end
+end

data/lib/docker_toolkit/childprocess/unix/platform/x86_64-macosx.rb

@@ -0,0 +1,13 @@
+module DockerToolkit
+module ChildProcess::Unix::Platform
+  SIZEOF = {
+     :posix_spawn_file_actions_t => 8,
+     :posix_spawnattr_t          => 8,
+     :sigset_t                   => 4
+  }
+  POSIX_SPAWN_RESETIDS   = 1
+  POSIX_SPAWN_SETPGROUP  = 2
+  POSIX_SPAWN_SETSIGDEF  = 4
+  POSIX_SPAWN_SETSIGMASK = 8
+end
+end

data/lib/docker_toolkit/childprocess/unix/posix_spawn_process.rb

@@ -0,0 +1,135 @@
+require 'thread'
+
+module DockerToolkit
+module ChildProcess
+  module Unix
+    class PosixSpawnProcess < Process
+      private
+
+      @@cwd_lock = Mutex.new
+
+      def launch_process
+        pid_ptr = FFI::MemoryPointer.new(:pid_t)
+        actions = Lib::FileActions.new
+        attrs   = Lib::Attrs.new
+
+        if io.stdout
+          actions.add_dup fileno_for(io.stdout), fileno_for(STDOUT)
+        else
+          actions.add_open fileno_for(STDOUT), "/dev/null", File::WRONLY, 0644
+        end
+
+        if io.stderr
+          actions.add_dup fileno_for(io.stderr), fileno_for(STDERR)
+        else
+          actions.add_open fileno_for(STDERR), "/dev/null", File::WRONLY, 0644
+        end
+
+        if duplex?
+          reader, writer = ::IO.pipe
+          actions.add_dup fileno_for(reader), fileno_for(STDIN)
+          actions.add_close fileno_for(writer)
+        end
+
+        attrs.pgroup = 0 if leader?
+        attrs.flags |= Platform::POSIX_SPAWN_USEVFORK if defined? Platform::POSIX_SPAWN_USEVFORK
+
+        # wrap in helper classes in order to avoid GC'ed pointers
+        argv = Argv.new(@args)
+        envp = Envp.new(ENV.to_hash.merge(@environment))
+
+        ret = 0
+        @@cwd_lock.synchronize do
+          Dir.chdir(@cwd || Dir.pwd) do
+            if ChildProcess.jruby?
+              # on JRuby, the current working directory is for some reason not inherited.
+              # We'll work around it by making a chdir call through FFI.
+              # TODO: report this to JRuby
+              Lib.chdir Dir.pwd
+            end
+
+            ret = Lib.posix_spawnp(
+              pid_ptr,
+              @args.first, # TODO: not sure this matches exec() behaviour
+              actions,
+              attrs,
+              argv,
+              envp
+            )
+          end
+        end
+
+        if duplex?
+          io._stdin = writer
+          reader.close
+        end
+
+        actions.free
+        attrs.free
+
+        if ret != 0
+          raise LaunchError, "#{Lib.strerror(ret)} (#{ret})"
+        end
+
+        @pid = pid_ptr.read_int
+        ::Process.detach(@pid) if detach?
+      end
+
+      if ChildProcess.jruby?
+        def fileno_for(obj)
+          ChildProcess::JRuby.posix_fileno_for(obj)
+        end
+      else
+        def fileno_for(obj)
+          obj.fileno
+        end
+      end
+
+      class Argv
+        def initialize(args)
+          @ptrs = args.map do |e|
+            if e.include?("\0")
+              raise ArgumentError, "argument cannot contain null bytes: #{e.inspect}"
+            end
+
+            FFI::MemoryPointer.from_string(e.to_s)
+          end
+
+          @ptrs << FFI::Pointer.new(0)
+        end
+
+        def to_ptr
+          argv = FFI::MemoryPointer.new(:pointer, @ptrs.size)
+          argv.put_array_of_pointer(0, @ptrs)
+
+          argv
+        end
+      end # Argv
+
+      class Envp
+        def initialize(env)
+          @ptrs = env.map do |key, val|
+            next if val.nil?
+
+            if key =~ /=|\0/ || val.include?("\0")
+              raise InvalidEnvironmentVariable, "#{key.inspect} => #{val.inspect}"
+            end
+
+            FFI::MemoryPointer.from_string("#{key}=#{val}")
+          end.compact
+
+          @ptrs << FFI::Pointer.new(0)
+        end
+
+        def to_ptr
+          env = FFI::MemoryPointer.new(:pointer, @ptrs.size)
+          env.put_array_of_pointer(0, @ptrs)
+
+          env
+        end
+      end # Envp
+
+    end
+  end
+end
+end