trusted-sandbox 0.0.2.pre

data/lib/trusted_sandbox/defaults.rb

@@ -0,0 +1,33 @@
+module TrustedSandbox
+  class Defaults < Config
+
+    def initialize
+      self.docker_options = {}
+      self.docker_image_name = 'vaharoni/trusted_sandbox:2.1.2.v1'
+      self.memory_limit = 50 * 1024 * 1024
+      self.memory_swap_limit = 50 * 1024 * 1024
+      self.cpu_shares = 1
+      self.execution_timeout = 15
+      self.network_access = false
+      self.enable_swap_limit = false
+      self.enable_quotas = false
+      self.host_code_root_path = 'tmp/code_dirs'
+      self.host_uid_pool_lock_path = 'tmp/uid_pool_lock'
+
+      self.docker_url = ENV['DOCKER_HOST']
+      self.docker_cert_path = ENV['DOCKER_CERT_PATH']
+
+      # Note, changing these may require changing Dockerfile and run.rb and rebuilding the docker image
+      self.container_code_path = '/home/sandbox/src'
+      self.container_input_filename = 'input'
+      self.container_output_filename = 'output'
+
+      # Note, changing these requires running `rake trusted_sandbox:set_quotas`
+      self.pool_min_uid = 20000
+      self.pool_size = 5000
+
+      self.keep_code_folders = false
+    end
+
+  end
+end

data/lib/trusted_sandbox/errors.rb

@@ -0,0 +1,8 @@
+module TrustedSandbox
+  class InvocationError < StandardError ; end
+  class PoolTimeoutError < StandardError; end
+  class ContainerError < StandardError; end
+  class UserCodeError < StandardError; end
+  class InternalError < StandardError; end
+  class ExecutionTimeoutError < StandardError; end
+end

data/lib/trusted_sandbox/request_serializer.rb

@@ -0,0 +1,53 @@
+module TrustedSandbox
+  class RequestSerializer
+
+    attr_reader :host_code_dir_path, :input_file_name
+
+    # @param host_code_dir_path [String] path to the folder where the argument value needs to be stored
+    # @param input_file_name [String] name of input file inside the host_code_dir_path
+    def initialize(host_code_dir_path, input_file_name)
+      @host_code_dir_path = host_code_dir_path
+      @input_file_name = input_file_name
+    end
+
+    # @param klass [Class] class name to be serialized
+    # @param args [Array] the array of argument values
+    # @return [String] full path of the argument that was stored
+    def serialize(klass, *args)
+      self.klass = klass
+      copy_code_file
+
+      data = Marshal.dump([klass.name, dest_file_name, args])
+      File.binwrite input_file_path, data
+    end
+
+    private
+
+    def input_file_path
+      File.join host_code_dir_path, input_file_name
+    end
+
+    # = Methods depending on @klass
+
+    attr_accessor :klass
+
+    def source_file_path
+      file, _line = klass.instance_method(:initialize).source_location
+      raise InvocationError.new("Cannot find location of class #{klass.name}") unless File.exist?(file.to_s)
+      file
+    end
+
+    def dest_file_name
+      File.basename(source_file_path)
+    end
+
+    def dest_file_path
+      File.join host_code_dir_path, dest_file_name
+    end
+
+    def copy_code_file
+      FileUtils.cp source_file_path, dest_file_path
+    end
+
+  end
+end

data/lib/trusted_sandbox/response.rb

@@ -0,0 +1,63 @@
+module TrustedSandbox
+  class Response
+
+    attr_reader :host_code_dir_path, :output_file_name, :stdout, :stderr,
+                :raw_response, :status, :error, :error_to_raise, :output
+
+    # @param host_code_dir_path [String] path to the folder where the argument value needs to be stored
+    # @param output_file_name [String] name of output file inside the host_code_dir_path
+    def initialize(host_code_dir_path, output_file_name, stdout, stderr)
+      @host_code_dir_path = host_code_dir_path
+      @output_file_name = output_file_name
+      @stdout = stdout
+      @stderr = stderr
+      parse_output_file
+    end
+
+    def valid?
+      status == 'success'
+    end
+
+    def output!
+      propagate_errors!
+      output
+    end
+
+    private
+
+    def output_file_path
+      File.join(host_code_dir_path, output_file_name)
+    end
+
+    def parse_output_file
+      begin
+        data = File.binread output_file_path
+        @raw_response = Marshal.load(data)
+      rescue => e
+        @status = 'error'
+        @error = e
+        @error_to_raise = ContainerError.new(e)
+        return
+      end
+
+      unless ['success', 'error'].include? @raw_response[:status]
+        @status = 'error'
+        @error = ContainerError.new('Output file has invalid format')
+        @error_to_raise = @error
+        return
+      end
+
+      @status = @raw_response[:status]
+      @output = @raw_response[:output]
+      @error = @raw_response[:error]
+      @error_to_raise = UserCodeError.new(@error) if @error
+    end
+
+    def propagate_errors!
+      return if valid?
+      raise InternalError.new 'Response object is invalid but no errors were recorded.' unless error
+      raise error_to_raise
+    end
+
+  end
+end

data/lib/trusted_sandbox/runner.rb

@@ -0,0 +1,129 @@
+module TrustedSandbox
+  class Runner
+
+    attr_reader :uid_pool, :config
+
+    # @param config [Config]
+    # @param uid_pool [UidPool]
+    # @param config_override [Hash] allows overriding configurations for a specific invocation
+    def initialize(config, uid_pool, config_override={})
+      @config = config.override(config_override)
+      @uid_pool = uid_pool
+    end
+
+    # @param klass [Class] the class object that should be run
+    # @param *args [Array] arguments to send to klass#initialize
+    # @return [Response]
+    def run(klass, *args)
+      create_code_dir
+      serialize_request(klass, *args)
+      create_container
+      start_container
+    ensure
+      release_uid
+      remove_code_dir unless config.keep_code_folders
+      remove_container
+    end
+
+    # @param klass [Class] the class object that should be run
+    # @param *args [Array] arguments to send to klass#initialize
+    # @return [Response]
+    # @raise [InternalError, UserCodeError, ContainerError]
+    def run!(klass, *args)
+      run(klass, *args).output!
+    end
+
+    private
+
+    def obtain_uid
+      @uid ||= uid_pool.lock
+    end
+
+    def release_uid
+      uid_pool.release(@uid) if @uid
+    end
+
+    def code_dir_path
+      @code_dir_path ||= File.join config.host_code_root_path, obtain_uid.to_s
+    end
+
+    def remove_code_dir
+      FileUtils.rm_rf code_dir_path
+    end
+
+    def create_code_dir
+      FileUtils.mkdir_p code_dir_path
+    end
+
+    def serialize_request(klass, *args)
+      serializer = RequestSerializer.new(code_dir_path, config.container_input_filename)
+      serializer.serialize(klass, *args)
+    end
+
+    def create_container
+      @container = Docker::Container.create create_container_request
+    end
+
+    def start_container
+      @container.start start_container_request
+      stdout, stderr = nil, nil
+      Timeout.timeout(config.execution_timeout) do
+        stdout, stderr = @container.attach(stream: true, stdin: nil, stdout: true, stderr: true, logs: true, tty: false)
+      end
+      TrustedSandbox::Response.new code_dir_path, config.container_output_filename, stdout, stderr
+    rescue Timeout::Error => e
+      raise TrustedSandbox::ExecutionTimeoutError.new(e)
+    end
+
+    def remove_container
+      return unless @container
+      @container.delete force: true
+    end
+
+    def create_container_request
+      basic_request = {
+          # 'Hostname' => '',
+          # 'Domainname' => '',
+          # 'User' => '',
+          'CpuShares' => config.cpu_shares,
+          'Memory' => config.memory_limit,
+          # 'Cpuset' => '0,1',
+          'AttachStdin' => false,
+          'AttachStdout' => true,
+          'AttachStderr' => true,
+          # 'PortSpecs' => null,
+          'Tty' => false,
+          'OpenStdin' => false,
+          'StdinOnce' => false,
+          'Cmd' => [@uid.to_s],
+          'Image' => config.docker_image_name,
+          'Volumes' => {
+              config.container_code_path => {}
+          },
+          # 'WorkingDir' => '',
+          'NetworkDisabled' => !config.network_access,
+          # 'ExposedPorts' => {
+          #     '22/tcp' => {}
+          # }
+      }
+      basic_request.merge!('MemorySwap' => config.memory_swap_limit) if config.enable_swap_limit
+      basic_request.merge!('Env' => ['USE_QUOTAS=1']) if config.enable_quotas
+      basic_request
+    end
+
+    def start_container_request
+      {
+          'Binds' => ["#{code_dir_path}:#{config.container_code_path}"],
+          # 'Links' => ['redis3:redis'],
+          # 'LxcConf' => {'lxc.utsname' => 'docker'},
+          # 'PortBindings' => {'22/tcp' => [{'HostPort' => '11022'}]},
+          # 'PublishAllPorts' => false,
+          # 'Privileged' => false,
+          # 'Dns' => ['8.8.8.8'],
+          # 'VolumesFrom' => ['parent', 'other:ro'],
+          # 'CapAdd' => ['NET_ADMIN'],
+          # 'CapDrop' => ['MKNOD']
+      }
+    end
+  end
+end

data/lib/trusted_sandbox/server_images/2.1.2/Dockerfile

@@ -0,0 +1,23 @@
+FROM ubuntu:14.04
+MAINTAINER Amit Aharoni <amit.sites@gmail.com>
+
+RUN apt-get -y update
+RUN apt-get -y install build-essential zlib1g-dev libssl-dev libreadline6-dev libyaml-dev wget
+RUN cd /tmp && wget http://ftp.ruby-lang.org/pub/ruby/2.1/ruby-2.1.2.tar.gz && tar -xvzf ruby-2.1.2.tar.gz
+RUN cd /tmp/ruby-2.1.2/ && ./configure --prefix=/usr/local && make && make install
+
+RUN groupadd app && useradd -m -G app -d /home/sandbox sandbox
+
+RUN gem install bundler
+ADD Gemfile /home/sandbox/Gemfile
+ADD bundle_config /home/sandbox/.bundle/config
+RUN chown sandbox /home/sandbox/Gemfile && \
+    chown sandbox /home/sandbox/.bundle && \
+    chown sandbox /home/sandbox/.bundle/config && \
+    sudo -u sandbox -i bundle install
+
+ADD entrypoint.sh entrypoint.sh
+ADD run.rb /home/sandbox/run.rb
+RUN chown sandbox /home/sandbox/run.rb
+
+ENTRYPOINT ["/bin/bash", "entrypoint.sh"]

data/lib/trusted_sandbox/server_images/2.1.2/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'activesupport'

data/lib/trusted_sandbox/server_images/2.1.2/bundle_config

@@ -0,0 +1,3 @@
+---
+BUNDLE_PATH: .bundle
+BUNDLE_DISABLE_SHARED_GEMS: "1"

data/lib/trusted_sandbox/server_images/2.1.2/entrypoint.sh

@@ -0,0 +1,15 @@
+uid=$1
+
+if [ -z $uid ]; then
+  echo "you must provide a uid"
+  exit 1
+fi
+
+echo "127.0.0.1 $(hostname)" >> /etc/hosts
+
+if [ -n "$USE_QUOTAS" -a "$USE_QUOTAS" != "0" -a "$USE_QUOTAS" != "false" ]; then
+  usermod -u $uid sandbox
+  chown sandbox /home/sandbox/src
+fi
+
+sudo -u sandbox -i bundle exec ruby run.rb

data/lib/trusted_sandbox/server_images/2.1.2/run.rb

@@ -0,0 +1,18 @@
+begin
+  require 'active_support'
+
+  input_file_path = '/home/sandbox/src/input'
+  output_file_path = '/home/sandbox/src/output'
+
+  data = File.binread(input_file_path)
+  klass_name, file_name, args = Marshal.load(data)
+  require File.join('/home/sandbox/src', file_name)
+  klass = ActiveSupport::Inflector.constantize klass_name
+
+  obj = klass.new(*args)
+  output = obj.run
+
+  File.binwrite output_file_path, Marshal.dump(status: 'success', output: output)
+rescue => e
+  File.binwrite output_file_path, Marshal.dump(status: 'error', error: e)
+end

data/lib/trusted_sandbox/uid_pool.rb

@@ -0,0 +1,153 @@
+module TrustedSandbox
+
+  # Offers intra-server inter-process pool of Uids. In other words:
+  #   - Every server has its own pool. Since Docker containers live within a server, this is what we want.
+  #   - Processes within the same server share the pool.
+  #
+  # Usage:
+  #   The following will behave the same when different processes try to perform #lock and #release.
+  #
+  #   pool = UidPool.new 100, 101
+  #   pool.lock
+  #   # => 100
+  #
+  #   pool.lock
+  #   # => 101
+  #
+  #   pool.lock
+  #   # => RuntimeError: No available UIDs in the pool. Please try again later.
+  #
+  #   pool.release(100)
+  #   # => 100
+  #
+  #   pool.lock
+  #   # => 100
+  #
+  #   pool.release_all
+  #
+  class UidPool
+
+    attr_reader :lock_dir, :master_lock_file, :lower, :upper, :timeout, :retries, :delay
+
+    # @param lower [Integer] lower bound of the pool
+    # @param upper [Integer] upper bound of the pool
+    # @option timeout [Integer] number of seconds to wait for the lock
+    # @option retries [Integer] number of attempts to retry to acquire a uid
+    # @option delay [Float] delay between retries
+    def initialize(lock_dir, lower, upper, timeout: nil, retries: nil, delay: nil)
+      @lock_dir = lock_dir
+      FileUtils.mkdir_p(lock_dir)
+
+      @master_lock_file = lock_file_path_for('master')
+      @lower = lower
+      @upper = upper
+      @timeout = timeout || 3
+      @retries = retries || 5
+      @delay = delay || 0.5
+    end
+
+    def inspect
+      "#<TrustedSandbox::UidPool used: #{used}, available: #{available}, used_uids: #{used_uids}>"
+    end
+
+    # @return [Integer]
+    def lock
+      retries.times do
+        atomically(timeout) do
+          uid = available_uid
+          if uid
+            lock_uid uid
+            return uid.to_i
+          end
+        end
+        sleep(delay)
+      end
+      raise PoolTimeoutError.new('No available UIDs in the pool. Please try again later.')
+    end
+
+    # Releases all UIDs
+    # @return [UidPool] self
+    def release_all
+      all_uids.each do |uid|
+        release uid
+      end
+      self
+    end
+
+    # @param uid [Integer]
+    def release(uid)
+      atomically(timeout) do
+        release_uid uid
+      end
+    end
+
+    # @return [Integer] number of used UIDs
+    def used
+      used_uids.length
+    end
+
+    # @return [Integer] number of availabld UIDs
+    def available
+      available_uids.length
+    end
+
+    # @return [Array<Integer>] all taken uids
+    def used_uids
+      uids = Dir.entries(lock_dir) - %w(. .. master)
+      uids.map(&:to_i)
+    end
+
+    # @return [Array<Integer>] all non taken uids
+    def available_uids
+      all_uids - used_uids
+    end
+
+    private
+
+    # @return [Array<Integer>] all uids in range
+    def all_uids
+      [*lower..upper]
+    end
+
+    # @param uid [Integer]
+    # @return [String] full path for the UID lock file
+    def lock_file_path_for(uid)
+      File.join lock_dir, uid.to_s
+    end
+
+    # Creates a UID lock file in the lock_dir
+    #
+    # @param uid [Integer]
+    # @return [Integer] the UID locked
+    def lock_uid(uid)
+      File.open lock_file_path_for(uid), 'w'
+      uid
+    end
+
+    # Removes a UID lock file from the lock_dir
+    #
+    # @param uid [Integer]
+    # @return [Integer] the UID removed
+    def release_uid(uid)
+      FileUtils.rm lock_file_path_for(uid), force: true
+      uid
+    end
+
+    # @param timeout [Integer]
+    # @return yield return value
+    def atomically(timeout)
+      Timeout.timeout(timeout) do
+        File.open(master_lock_file, File::RDWR|File::CREAT, 0644) do |f|
+          f.flock File::LOCK_EX
+          yield
+        end
+      end
+    end
+
+    # @return [Integer, nil] one available uid or nil if none is available
+    def available_uid
+      available_uids.first
+    end
+
+  end
+end

data/lib/trusted_sandbox/version.rb

@@ -0,0 +1,3 @@
+module TrustedSandbox
+  VERSION = '0.0.2.pre'
+end

data/lib/trusted_sandbox.rb

@@ -0,0 +1,59 @@
+module TrustedSandbox
+
+  require 'yaml'
+  require 'docker'
+  require 'trusted_sandbox/config'
+  require 'trusted_sandbox/defaults'
+  require 'trusted_sandbox/errors'
+  require 'trusted_sandbox/request_serializer'
+  require 'trusted_sandbox/response'
+  require 'trusted_sandbox/runner'
+  require 'trusted_sandbox/uid_pool'
+  require 'trusted_sandbox/version'
+
+  # Usage:
+  #   TrustedSandbox.config do |c|
+  #     c.pool_size = 10
+  #     # ...
+  #   end
+  def self.config
+    @config ||= Defaults.send(:new).override config_overrides_from_file
+    yield @config if block_given?
+    @config.finished_configuring
+  end
+
+  def self.config_overrides_from_file(env = nil)
+    yaml_path = %w(trusted_sandbox.yml config/trusted_sandbox.yml).find {|x| File.exist?(x)}
+    return {} unless yaml_path
+
+    env ||= ENV['TRUSTED_SANDBOX_ENV'] || ENV['RAILS_ENV'] || 'development'
+    YAML.load_file(yaml_path)[env]
+  end
+
+  def self.uid_pool
+    @uid_pool ||= UidPool.new config.host_uid_pool_lock_path, config.pool_min_uid, config.pool_max_uid,
+                              timeout: config.pool_timeout, retries: config.pool_retries, delay: config.pool_delay
+  end
+
+  # @param config_override [Hash] allows overriding configurations for a specific invocation
+  def self.with_options(config_override={})
+    yield new_runner(config_override)
+  end
+
+  # @param klass [Class] the class to be instantiated in the safe sandbox
+  # @param *args [Array] arguments to send to klass#new
+  def self.run(klass, *args)
+    new_runner.run(klass, *args)
+  end
+
+  def self.run!(klass, *args)
+    new_runner.run!(klass, *args)
+  end
+
+  def self.new_runner(config_override = {})
+    Runner.new(config, uid_pool, config_override)
+  end
+end
+
+# Run the configuration block
+TrustedSandbox.config

data/trusted-sandbox.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'trusted_sandbox/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'trusted-sandbox'
+  spec.version       = TrustedSandbox::VERSION
+  spec.authors       = ['Amit Aharoni']
+  spec.email         = ['amit.sites@gmail.com']
+  spec.description   = %q{Trusted Sandbox makes it simple to execute Ruby classes that eval untrusted code in a resource-controlled docker container}
+  spec.summary       = %q{Run untrusted Ruby code in a contained sandbox using Docker}
+  spec.homepage      = 'https://github.com/vaharoni/trusted-sandbox'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+
+  spec.add_runtime_dependency 'docker-api', '~> 1.13'
+  spec.add_runtime_dependency 'thor', '~> 0.19'
+end