contained_mr 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 92996839ff48bb11fd7dd367fe661163a7b33c25
+  data.tar.gz: f44046ae3b61c5c10fc9dfbe0311a48b725e26fd
+SHA512:
+  metadata.gz: b570c230d4c6c5a2a74bffbda9e88667bfe8f2a420e6b992b8bc4368d94e4fecc12b5541b6090d5dba5e6376cf1abffb342ed0dd0da1ef254fe5f3ee97f261d0
+  data.tar.gz: c296cc24c02982f2018f968f6338db78af78a32c9140a0c885c2ca2f1fd711a62b8c39166539021ffee870de044a26b45aa3337967b8f371bcb23af5200a92cb

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,15 @@
+source 'https://rubygems.org'
+
+gem 'docker-api', '>= 1.22.3', require: 'docker'
+gem 'rubyzip', '>= 1.1.7', require: 'zip'
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "minitest", ">= 0"
+  gem "yard", ">= 0.7"
+  gem "rdoc", ">= 3.12"
+  gem "bundler", ">= 1.6.1"
+  gem "jeweler", ">= 2.0.1"
+  gem "simplecov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,76 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    builder (3.2.2)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    docile (1.1.5)
+    docker-api (1.22.3)
+      excon (>= 0.38.0)
+      json
+    excon (0.45.4)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.9.1)
+    github_api (0.12.4)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.4)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.6)
+      oauth2
+    hashie (3.4.2)
+    highline (1.7.3)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.3)
+    jwt (1.5.1)
+    mini_portile (0.6.2)
+    minitest (5.8.0)
+    multi_json (1.11.2)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.6.4)
+    rake (10.4.2)
+    rdoc (4.2.0)
+    rubyzip (1.1.7)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    thread_safe (0.3.5)
+    yard (0.8.7.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.6.1)
+  docker-api (>= 1.22.3)
+  jeweler (>= 2.0.1)
+  minitest
+  rdoc (>= 3.12)
+  rubyzip (>= 1.1.7)
+  simplecov
+  yard (>= 0.7)
+
+BUNDLED WITH
+   1.10.6

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 Victor Costan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,17 @@
+= contained_mr
+
+Map-Reduce where both the mappers and the reducer run inside Docker containers.
+
+== Contributing to contained_mr
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2015 Victor Costan. See LICENSE.txt for further details.

data/Rakefile

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "contained_mr"
+  gem.homepage = "http://github.com/pwnall/contained_mr"
+  gem.license = "MIT"
+  gem.summary = %Q{Map-Reduce with Docker containers}
+  gem.description = %Q{Plumbing for running mappers and reducers inside Docker containers}
+  gem.email = "victor@costan.us"
+  gem.authors = ["Victor Costan"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/contained_mr/cleaner.rb

@@ -0,0 +1,47 @@
+require 'json'
+
+# Cleans up left over Docker images and containers.
+class ContainedMr::Cleaner
+  attr_reader :name_prefix
+
+  # Sets up a cleaner.
+  #
+  # @param {String} name_prefix should match the value given to Template
+  #   instances
+  def initialize(name_prefix)
+    @name_prefix = name_prefix
+    @label_value = name_prefix
+  end
+
+  # Removes all images and containers matching this cleaner's name prefix.
+  def destroy_all!
+    destroy_all_containers!
+    destroy_all_images!
+  end
+
+  def destroy_all_containers!
+    containers = Docker::Container.all all: true,
+                                       filters: container_filters.to_json
+    containers.each do |container|
+      container.delete force: false, volumes: true
+    end
+  end
+  private :destroy_all_containers!
+
+  def destroy_all_images!
+    tag_prefix = "#{@name_prefix}/"
+    images = Docker::Image.all
+    images.each do |image|
+      image_tags = image.info['RepoTags'] || []
+      next unless image_tags.any? { |tag| tag.start_with? tag_prefix }
+      image.delete
+    end
+  end
+  private :destroy_all_images!
+
+  # @return { Hash<Symbol, Array<String>> } filters used to identify Docker
+  #   containers started by this controller
+  def container_filters
+    { label: [ "contained_mr.ctl=#{@label_value}" ] }
+  end
+end

data/lib/contained_mr/job.rb

@@ -0,0 +1,236 @@
+require 'json'
+require 'rubygems'  # For tar operations.
+require 'stringio'
+require 'yaml'
+
+require 'docker'
+
+# A map-reduce job.
+class ContainedMr::Job
+  attr_reader :id, :item_count, :mapper_image_id, :reducer_image_id
+
+  # Sets up the job.
+  #
+  # @param {Template} template data used to spawn this job
+  # @param {String} id the job's unique ID
+  # @param {Hash<String, Object>} json_options job options, extracted from JSON
+  def initialize(template, id, json_options)
+    @id = id
+    @template = template
+    @name_prefix = template.name_prefix
+    @item_count = template.item_count
+
+    @mapper_image_id = nil
+    @reducer_image_id = nil
+
+    @mappers = Array.new @item_count
+    @reducer = nil
+    @mapper_options = nil
+    @reducer_options = nil
+    parse_options json_options
+  end
+
+  # Tears down the job's state.
+  #
+  # This removes the job's containers, as well as the mapper and reducer Docker
+  # images, if they still exist.
+  def destroy!
+    @mappers.each do |runner|
+      next if runner.nil? or runner.container_id.nil?
+      container = Docker::Container.get runner.container_id
+      container.delete force: true
+    end
+
+    unless @reducer.nil? or @reducer.container_id.nil?
+      container = Docker::Container.get @reducer.container_id
+      container.delete force: true
+    end
+
+    unless @mapper_image_id.nil?
+      image = Docker::Image.get @mapper_image_id
+      image.remove
+      @mapper_image_id = nil
+    end
+
+    unless @reducer_image_id.nil?
+      image = Docker::Image.get @reducer_image_id
+      image.remove
+      @reducer_image_id = nil
+    end
+  end
+
+  # Returns the runner used for a mapper.
+  #
+  # @param {Number} i the mapper number
+  # @return {ContainedMr::Runner} the runner used for the given mapper; nil if
+  #   the given mapper was not started
+  def mapper_runner(i)
+    @mappers[i - 1]
+  end
+
+  # Returns the runner used for the reducer.
+  #
+  # @return {ContainedMr::Runner} the runner used for reducer; nil if the
+  #   reducer was not started
+  def reducer_runner
+    @reducer
+  end
+
+  # Builds the Docker image used to run this job's mappers.
+  #
+  # @param {String} mapper_input data passed to the mappers
+  # @return {String} the newly built Docker image's ID
+  def build_mapper_image(mapper_input)
+    tar_io = mapper_tar_context mapper_input
+    image = Docker::Image.build_from_tar tar_io, t: mapper_image_tag
+    @mapper_image_id = image.id
+  end
+
+  # Builds the Docker image used to run this job's reducer.
+  #
+  # @return {String} the newly built Docker image's ID
+  def build_reducer_image
+    tar_io = reducer_tar_context
+    image = Docker::Image.build_from_tar tar_io, t: reducer_image_tag
+    @reducer_image_id = image.id
+  end
+
+  # Runs one of the job's mappers.
+  #
+  # @param {Number} i the mapper to run
+  # @return {ContainedMr::Runner} the runner used by the mapper
+  def run_mapper(i)
+    mapper = ContainedMr::Runner.new mapper_container_options(i),
+        @mapper_options[:wait_time], @template.mapper_output_path
+    @mappers[i - 1] = mapper
+    mapper.perform
+  end
+
+  # Runs one the job's reducer.
+  #
+  # @return {ContainedMr::Runner} the runner used by the reducer
+  def run_reducer
+    reducer = ContainedMr::Runner.new reducer_container_options,
+        @reducer_options[:wait_time], @template.reducer_output_path
+    @reducer = reducer
+    @reducer.perform
+  end
+
+  # @return {String} tag applied to the Docker image used by the job's mappers
+  def mapper_image_tag
+    "#{@name_prefix}/mapper.#{@id}"
+  end
+
+  # @return {String} tag applied to the Docker image used by the job's reducers
+  def reducer_image_tag
+    "#{@name_prefix}/reducer.#{@id}"
+  end
+
+  # @return {Hash<String, Object>} params used to create a mapper container
+  def mapper_container_options(i)
+    ulimits = @mapper_options[:ulimits].map do |k, v|
+      { "Name" => k.to_s, "Soft" => v, "Hard" => v }
+    end
+
+    {
+      'name' => "#{@name_prefix}_mapper.#{@id}.#{i}",
+      'Image' => @mapper_image_id,
+      'Hostname' => "#{i}.mapper", 'Domainname' => '',
+      'Labels' => { 'contained_mr.ctl' => @name_prefix },
+      'Env' => @template.mapper_env(i), 'Ulimits' => ulimits,
+      'NetworkDisabled' => true, 'ExposedPorts' => {},
+    }
+  end
+
+  # @return {Hash<String, Object>} params used to create a reducer container
+  def reducer_container_options
+    ulimits = @reducer_options[:ulimits].map do |k, v|
+      { "Name" => k.to_s, "Soft" => v, "Hard" => v }
+    end
+
+    {
+      'name' => "#{@name_prefix}_reducer.#{@id}",
+      'Image' => @reducer_image_id,
+      'Hostname' => 'reducer', 'Domainname' => '',
+      'Labels' => { 'contained_mr.ctl' => @name_prefix },
+      'Env' => @template.reducer_env, 'Ulimits' => ulimits,
+      'NetworkDisabled' => true, 'ExposedPorts' => {},
+    }
+  end
+
+  # Reads in JSON options and sets defaults.
+  def parse_options(json_options)
+    mapper = json_options['mapper'] || {}
+    mapper_ulimits = mapper['ulimits'] || {}
+    @mapper_options = {
+      wait_time: mapper['wait_time'] || 60,
+      ulimits: {
+        cpu: mapper_ulimits['cpu'] || 60,  # seconds
+        rss: mapper_ulimits['rss'] || 500_000,  # pages
+      }
+    }
+
+    reducer = json_options['reducer'] || {}
+    reducer_ulimits = reducer['ulimits'] || {}
+    @reducer_options = {
+      wait_time: reducer['wait_time'] || 60,
+      ulimits: {
+        cpu: reducer_ulimits['cpu'] || 60,
+        rss: reducer_ulimits['rss'] || 500_000,
+      }
+    }
+  end
+  private :parse_options
+
+  # Builds the .tar context used to create the mapper's Docker image.
+  #
+  # @param {String} mapper_input data passed to the mappers
+  # @return {IO} an IO implementation that sources the .tar data
+  def mapper_tar_context(mapper_input)
+    tar_buffer = StringIO.new
+    Gem::Package::TarWriter.new tar_buffer do |tar|
+      tar.add_file 'Dockerfile', 0644 do |docker_io|
+        docker_io.write @template.mapper_dockerfile
+      end
+      tar.add_file 'input', 0644 do |input_io|
+        input_io.write mapper_input
+      end
+    end
+    tar_buffer.rewind
+    tar_buffer
+  end
+  private :mapper_tar_context
+
+  # Builds the .tar context used to create the mapper's Docker image.
+  #
+  # @return {IO} an IO implementation that sources the .tar file
+  def reducer_tar_context
+    tar_buffer = StringIO.new
+    Gem::Package::TarWriter.new tar_buffer do |tar|
+      tar.add_file 'Dockerfile', 0644 do |docker_io|
+        docker_io.write @template.reducer_dockerfile
+      end
+      @mappers.each_with_index do |mapper, index|
+        i = index + 1
+
+        if mapper.output
+          tar.add_file "#{i}.out", 0644 do |io|
+            io.write mapper.output
+          end
+        end
+        tar.add_file("#{i}.stdout", 0644) { |io| io.write mapper.stdout }
+        tar.add_file("#{i}.stderr", 0644) { |io| io.write mapper.stderr }
+
+        status = {
+          ran_for: mapper.ran_for,
+          exit_code: mapper.status_code,
+          timed_out: mapper.timed_out,
+        }
+        tar.add_file("#{i}.json", 0644) { |io| io.write status.to_json }
+      end
+    end
+    tar_buffer.rewind
+    tar_buffer
+  end
+  private :mapper_tar_context
+end

data/lib/contained_mr/runner.rb

@@ -0,0 +1,125 @@
+require 'rubygems'  # For tar operations.
+require 'stringio'
+
+require 'docker'
+
+# Handles running a single mapper or reducer.
+class ContainedMr::Runner
+  attr_reader :container_id
+  attr_reader :started_at, :ended_at, :status_code, :timed_out
+  attr_reader :stdout, :stderr, :output
+
+  # C
+  def initialize(container_options, time_limit, output_path)
+    @container_options = container_options
+    @time_limit = time_limit
+    @output_path = output_path
+
+    @container_id = nil
+    @started_at = @ended_at = nil
+    @status_code = nil
+    @timed_out = nil
+    @stdout = @stderr = nil
+    @output = nil
+  end
+
+
+  # Performs a full mapper / reducer step.
+  def perform
+    container = create
+    @container_id = container.id
+
+    execute container
+    fetch_console_output container
+    fetch_file_output container
+    destroy container
+    self
+  end
+
+  # @return {Number} the container's running time, in seconds
+  def ran_for
+    @started_at && @ended_at && (@ended_at - @started_at)
+  end
+
+  # Creates a container for running a mapper or reducer.
+  #
+  # @return {Docker::Container} newly created container
+  def create
+    Docker::Container.create @container_options
+  end
+  private :create
+
+  # Runs the process inside the container, kills it if takes too long.
+  #
+  # @param {Docker::Container} container the container that holds the process
+  def execute(container)
+    container.start
+    @started_at = Time.now
+    begin
+      wait_status = container.wait @time_limit
+      @status_code = wait_status['StatusCode']
+      @timed_out = false
+    rescue Docker::Error::TimeoutError
+      @status_code = false
+      @timed_out = true
+      container.kill
+    end
+    @ended_at = Time.now
+  end
+  private :execute
+
+  # Extracts console output from a container.
+  #
+  # @param {Docker::Container} container the mapper / reducer's container
+  def fetch_console_output(container)
+    messages = container.attach stream: false, logs: true, stdin: nil,
+                                stdout: true, stderr: true
+    @stdout = messages[0].join ''
+    @stderr = messages[1].join ''
+  end
+  private :fetch_console_output
+
+  # Extracts the mapper / reducer's output file from a container.
+  #
+  # @param {Docker::Container} container the mapper / reducer's container
+  def fetch_file_output(container)
+    begin
+      tar_buffer = fetch_tar_output container
+    rescue Docker::Error::ServerError
+      @output = false
+      return
+    end
+
+    Gem::Package::TarReader.new tar_buffer do |tar|
+      tar.each do |entry|
+        next unless entry.file?
+        @output = entry.read
+        return
+      end
+    end
+    @output = false
+  end
+  private :fetch_file_output
+
+  # Extracts the mapper / reducer's output, as a .tar, from a container.
+  #
+  # @param {Docker::Container} container the mapper / reducer's container
+  # @return {IO} an IO implementation that sources the .tar data
+  def fetch_tar_output(container)
+    tar_buffer = StringIO.new
+    container.copy @output_path do |data|
+      tar_buffer << data
+    end
+    tar_buffer.rewind
+    tar_buffer
+  end
+  private :fetch_tar_output
+
+  # Removes the container used to run a mapper / reducer.
+  #
+  # @param {Docker::Container} container the mapper / reducer's container
+  def destroy(container)
+    container.delete
+    @container_id = nil
+  end
+end

data/lib/contained_mr/template.rb

@@ -0,0 +1,138 @@
+require 'json'
+require 'rubygems'  # For tar operations.
+require 'yaml'
+
+require 'docker'
+require 'zip'
+
+# A template is used to spawn multiple Map-Reduce jobs.
+class ContainedMr::Template
+  attr_reader :name_prefix, :item_count, :image_id
+
+  # Sets up the template and builds its Docker base image.
+  #
+  # @param {String} name_prefix prepended to Docker objects, for identification
+  #   purposes
+  # @param {String} id the job's unique identifier
+  # @param {String} zip_io IO implementation that produces the template .zip
+  def initialize(name_prefix, id, zip_io)
+    @name_prefix = name_prefix
+    @id = id
+    @image_id = nil
+    @definition = nil
+    @item_count = nil
+
+    tar_buffer = StringIO.new
+    process_zip zip_io, tar_buffer
+    tar_buffer.rewind
+    build_image tar_buffer
+  end
+
+  # Tears down the template's state.
+  #
+  # This removes the template's base Docker image.
+  def destroy!
+    unless @image_id.nil?
+      image = Docker::Image.get @image_id
+      image.remove
+      @image_id = nil
+    end
+  end
+
+  # Computes the Dockerfile used to build a job's mapper image.
+  #
+  # @return {String} the Dockerfile
+  def mapper_dockerfile
+    job_dockerfile @definition['mapper'] || {}, 'input'
+  end
+
+  # Computes the Dockerfile used to build a job's reducer image.
+  #
+  # @return {String} the Dockerfile
+  def reducer_dockerfile
+    job_dockerfile @definition['reducer'] || {}, '.'
+  end
+
+  # @return {String} tag applied to the template's base Docker image
+  def image_tag
+    "#{@name_prefix}/base.#{@id}"
+  end
+
+  # Computes the environment variables to be set in a mapper container.
+  #
+  # @param {Number} i the mapper number
+  # @return {Array<String>} environment variables to be set in the mapper
+  def mapper_env(i)
+    [ "ITEM=#{i}", "ITEMS=#{@item_count.to_s}" ]
+  end
+
+  # Computes the environment variables to be set in the reducer container.
+  #
+  # @return {Array<String>} environment variables to be set in the mapper
+  def reducer_env
+    [ "ITEMS=#{@item_count.to_s}" ]
+  end
+
+  # @return {String} the map output's path in the mapper Docker container
+  def mapper_output_path
+    (@definition['mapper'] || {})['output'] || '/output'
+  end
+
+  # @return {String} the reducer output's path in the reducer Docker container
+  def reducer_output_path
+    (@definition['reducer'] || {})['output'] || '/output'
+  end
+
+  # @private common code from mapper_dockerfile and reducer_dockerfile
+  def job_dockerfile(job_definition, input_source)
+    <<DOCKER_END
+FROM #{@image_id}
+COPY #{input_source} #{job_definition['input'] || '/input'}
+WORKDIR #{job_definition['chdir'] || '/'}
+ENTRYPOINT #{JSON.dump(job_definition['cmd'] || ['/bin/sh'])}
+DOCKER_END
+  end
+  private :job_dockerfile
+
+  # Reads the template .zip and parses the definition.
+  #
+  # @param {IO} zip_io IO implementation that produces the .zip file
+  # @param {IO} tar_io IO implementation that will receive the .tar file
+  def process_zip(zip_io, tar_io)
+    Gem::Package::TarWriter.new tar_io do |tar|
+      # TODO(pwnall): zip_io.read -> zip_io after rubyzip releases 1.1.8
+      Zip::File.open_buffer zip_io.read do |zip|
+        zip.each do |zip_entry|
+          file_name = zip_entry.name
+          if file_name == 'mapreduced.yml'
+            read_definition zip_entry.get_input_stream
+            next
+          end
+          tar.add_file file_name, 0644 do |tar_file_io|
+            IO.copy_stream zip_entry.get_input_stream, tar_file_io
+          end
+        end
+      end
+    end
+  end
+
+  # Reads the template's definition, using data at the given path.
+  #
+  # @param {IO} yaml_io IO implementation that produces the .yaml file
+  #   containing the definition
+  def read_definition(yaml_io)
+    @definition = YAML.load yaml_io.read
+
+    @item_count = @definition['items'] || 1
+  end
+  private :read_definition
+
+  # Builds the template's Docker image, using data at the given path.
+  #
+  # @param {IO} tar_io IO implementation that produces the image's .tar file
+  def build_image(tar_io)
+    image = Docker::Image.build_from_tar tar_io, t: image_tag
+    @image_id = image.id
+  end
+  private :build_image
+end

data/lib/contained_mr.rb

@@ -0,0 +1,8 @@
+# Namespace.
+module ContainedMr
+end
+
+require_relative 'contained_mr/cleaner.rb'
+require_relative 'contained_mr/job.rb'
+require_relative 'contained_mr/runner.rb'
+require_relative 'contained_mr/template.rb'

data/test/helper.rb

@@ -0,0 +1,45 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_profile 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'minitest/autorun'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'contained_mr'
+
+class MiniTest::Test
+end
+
+File.unlink 'testdata/hello.zip' if File.file?('testdata/hello.zip')
+Zip::File.open('testdata/hello.zip', Zip::File::CREATE) do |zip|
+  files = Dir.chdir('testdata/hello') { Dir.glob('**/*') }.sort
+  files.each do |file|
+    path = File.join 'testdata/hello', file
+    next unless File.file?(path)
+    zip.add file, path
+  end
+end
+
+MiniTest.autorun

data/test/test_job.rb

@@ -0,0 +1,153 @@
+require 'helper'
+
+class TestJob < MiniTest::Test
+  def setup
+    @template = ContainedMr::Template.new 'contained_mrtests', 'hello',
+        StringIO.new(File.binread('testdata/hello.zip'))
+    @job = ContainedMr::Job.new @template, 'testjob',
+                           JSON.load(File.read('testdata/job.hello'))
+  end
+
+  def teardown
+    @job.destroy!
+    @template.destroy!
+  end
+
+  def test_mapper_container_options
+    @job.build_mapper_image File.read('testdata/input.hello')
+
+    golden = {
+      'name' => 'contained_mrtests_mapper.testjob.2',
+      'Image' => @job.mapper_image_id,
+      'Hostname' => '2.mapper',
+      'Domainname' => '',
+      'Labels' => { 'contained_mr.ctl' => 'contained_mrtests' },
+      'Env' => [ 'ITEM=2', 'ITEMS=3' ],
+      'Ulimits' => [
+        { 'Name' => 'cpu', 'Hard' => 3, 'Soft' => 3 },
+        { 'Name' => 'rss', 'Hard' => 1000000, 'Soft' => 1000000 },
+      ],
+      'NetworkDisabled' => true, 'ExposedPorts' => {},
+    }
+    assert_equal golden, @job.mapper_container_options(2)
+  end
+
+  def test_reducer_container_options
+    golden = {
+      'name' => 'contained_mrtests_reducer.testjob',
+      'Image' => @job.mapper_image_id,
+      'Hostname' => 'reducer',
+      'Domainname' => '',
+      'Labels' => { 'contained_mr.ctl' => 'contained_mrtests' },
+      'Env' => [ 'ITEMS=3' ],
+      'Ulimits' => [
+        { 'Name' => 'cpu', 'Hard' => 2, 'Soft' => 2 },
+        { 'Name' => 'rss', 'Hard' => 100000, 'Soft' => 100000 },
+      ],
+      'NetworkDisabled' => true, 'ExposedPorts' => {},
+    }
+    assert_equal golden, @job.reducer_container_options
+  end
+
+  def test_build_mapper_image
+    assert_equal 'contained_mrtests/mapper.testjob', @job.mapper_image_tag
+
+    @job.build_mapper_image File.read('testdata/input.hello')
+    image = Docker::Image.get @job.mapper_image_tag
+    assert image, 'Docker::Image'
+    assert_operator image.id, :start_with?, @job.mapper_image_id
+
+    1.upto 3 do |i|
+      assert_nil @job.mapper_runner(i), "Mapper #{i} started prematurely"
+    end
+    assert_nil @job.reducer_runner, "Reducer started prematurely"
+  end
+
+  def test_run_mapper_stderr
+    @job.build_mapper_image File.read('testdata/input.hello')
+    @job.run_mapper 2
+
+    assert_nil @job.mapper_runner(1), 'Mapper 1 started prematurely'
+    assert_nil @job.mapper_runner(3), 'Mapper 3 started prematurely'
+    assert_nil @job.reducer_runner, 'Reducer started prematurely'
+
+    mapper = @job.mapper_runner 2
+    assert mapper, 'Mapper 2 not started'
+    assert_equal nil, mapper.container_id, 'Mapper container still running'
+    assert_equal "2 3\n", mapper.stderr, 'Stderr: $ITEM + $ITEMS'
+  end
+
+  def test_run_mapper_stdout
+    @job.build_mapper_image File.read('testdata/input.hello')
+    mapper = @job.run_mapper 2
+
+    assert_equal nil, mapper.container_id, 'Mapper container still running'
+    assert_equal "2\nmapper input file\nHello world!\n", mapper.stdout,
+                 'Stdout: $ITEM + mapper input file + data file'
+  end
+
+  def test_run_mapper_output
+    @job.build_mapper_image File.read('testdata/input.hello')
+    mapper = @job.run_mapper 2
+
+    assert_equal nil, mapper.container_id, 'Mapper container still running'
+    assert_equal "2\n", mapper.output, 'Output: ITEM env variable'
+  end
+
+  def test_build_reducer_image
+    assert_equal 'contained_mrtests/reducer.testjob', @job.reducer_image_tag
+
+    @job.build_mapper_image File.read('testdata/input.hello')
+    1.upto(3) { |i| @job.run_mapper i }
+
+    @job.build_reducer_image
+    image = Docker::Image.get @job.reducer_image_tag
+    assert image, 'Docker::Image'
+    assert_operator image.id, :start_with?, @job.reducer_image_id
+
+    assert_nil @job.reducer_runner, "Reducer started prematurely"
+  end
+
+  def test_run_reducer
+    @job.build_mapper_image File.read('testdata/input.hello')
+    1.upto(3) { |i| @job.run_mapper i }
+    @job.build_reducer_image
+    reducer = @job.run_reducer
+
+    assert @job.reducer_runner, 'reducer_runner return'
+    assert_equal "3 /\n", reducer.stderr, 'Stderr: $ITEMS + $PWD'
+
+    output_gold = "1\n2\n3\n" +
+        "1\nmapper input file\nHello world!\n" +
+        "2\nmapper input file\nHello world!\n" +
+        "3\nmapper input file\nHello world!\n" +
+        "1 3\n2 3\n3 3\n"
+    assert_equal output_gold, reducer.output, 'Stderr: mappers out/stdout/err'
+
+    json_texts = reducer.stdout.split("\n")
+    assert_equal 3, json_texts.length
+
+    jsons = json_texts.map { |t| JSON.load t }
+    assert_equal [false, 0, 42], jsons.map { |j| j['exit_code'] }
+    assert_equal [true, false, false], jsons.map { |j| j['timed_out'] }
+    assert_operator jsons[0]['ran_for'], :>, 2, 'ran_for in mapper 0'
+    assert_operator jsons[1]['ran_for'], :<, 2, 'ran_for in mapper 1'
+    assert_operator jsons[2]['ran_for'], :<, 2, 'ran_for in mapper 2'
+  end
+
+  def test_destroy
+    @job.build_mapper_image File.read('testdata/input.hello')
+    1.upto(3) { |i| @job.run_mapper i }
+    @job.build_reducer_image
+
+    @job.destroy!
+
+    assert_raises Docker::Error::NotFoundError do
+      Docker::Image.get @job.mapper_image_tag
+    end
+
+    assert_raises Docker::Error::NotFoundError do
+      Docker::Image.get @job.reducer_image_tag
+    end
+  end
+end

data/test/test_runner.rb

@@ -0,0 +1,56 @@
+require 'helper'
+
+class TestRunner < MiniTest::Test
+  def setup
+    @template = ContainedMr::Template.new 'contained_mrtests', 'hello',
+        StringIO.new(File.binread('testdata/hello.zip'))
+    @job = ContainedMr::Job.new @template, 'testjob',
+                           JSON.load(File.read('testdata/job.hello'))
+    @job.build_mapper_image File.read('testdata/input.hello')
+  end
+
+  def teardown
+    @job.destroy!
+    @template.destroy!
+  end
+
+  def test_perform_happy_path
+    runner = ContainedMr::Runner.new @job.mapper_container_options(2), 2.5,
+                                @template.mapper_output_path
+    runner.perform
+
+    assert_equal nil, runner.container_id, 'container still running'
+    assert_operator runner.ended_at - runner.started_at, :<, 1, 'running time'
+    assert_equal 0, runner.status_code, 'status code'
+    assert_equal false, runner.timed_out, 'timed out'
+    assert_equal "2 3\n", runner.stderr, 'Stderr: $ITEM + $ITEMS'
+    assert_equal "2\nmapper input file\nHello world!\n", runner.stdout,
+                 'Stdout: $ITEM + mapper input file + data file'
+    assert_equal "2\n", runner.output, 'Output: ITEM env variable'
+  end
+
+  def test_perform_exit_code
+    runner = ContainedMr::Runner.new @job.mapper_container_options(3), 2.5,
+                                @template.mapper_output_path
+    runner.perform
+
+    assert_equal nil, runner.container_id, 'container still running'
+    assert_operator runner.ended_at - runner.started_at, :<, 1, 'running time'
+    assert_equal 42, runner.status_code, 'status code'
+    assert_equal false, runner.timed_out, 'timed out'
+  end
+
+  def test_perform_timeout
+    runner = ContainedMr::Runner.new @job.mapper_container_options(1), 2.5,
+                                @template.mapper_output_path
+    runner.perform
+
+    assert_equal nil, runner.container_id, 'container still running'
+    assert_equal false, runner.status_code, 'status code'
+    assert_equal true, runner.timed_out, 'timed out'
+    assert_operator runner.ended_at - runner.started_at, :>, 2.2,
+                    'running time'
+    assert_operator runner.ended_at - runner.started_at, :<, 2.8,
+                    'running time'
+  end
+end

data/test/test_template.rb

@@ -0,0 +1,50 @@
+require 'helper'
+
+class TestTemplate < MiniTest::Test
+  def setup
+    @template = ContainedMr::Template.new 'contained_mrtests', 'hello',
+        StringIO.new(File.binread('testdata/hello.zip'))
+  end
+
+  def teardown
+    @template.destroy!
+  end
+
+  def test_image_id_matches_created_image
+    image = Docker::Image.get @template.image_tag
+    assert image, 'Docker::Image'
+    assert_operator image.id, :start_with?, @template.image_id
+  end
+
+  def test_image_tag
+    assert_equal 'contained_mrtests/base.hello', @template.image_tag
+  end
+
+  def test_dockerfiles
+    golden = File.read 'testdata/Dockerfile.hello.mapper'
+    golden.sub! 'contained_mrtests/base.hello', @template.image_id
+    assert_equal golden, @template.mapper_dockerfile, 'mapper Dockerfile'
+
+    golden = File.read 'testdata/Dockerfile.hello.reducer'
+    golden.sub! 'contained_mrtests/base.hello', @template.image_id
+    assert_equal golden, @template.reducer_dockerfile, 'reducer Dockerfile'
+  end
+
+  def test_paths
+    assert_equal '/usr/mrd/map-output', @template.mapper_output_path
+    assert_equal '/usr/mrd/reduce-output', @template.reducer_output_path
+  end
+
+  def test_envs
+    assert_equal 3, @template.item_count
+    assert_equal ['ITEM=2', 'ITEMS=3'], @template.mapper_env(2)
+    assert_equal ['ITEMS=3'], @template.reducer_env
+  end
+
+  def test_destroy
+    @template.destroy!
+    assert_raises Docker::Error::NotFoundError do
+      Docker::Image.get @template.image_tag
+    end
+  end
+end

data/testdata/Dockerfile.hello.mapper

@@ -0,0 +1,4 @@
+FROM contained_mrtests/base.hello
+COPY input /usr/mrd/map-input
+WORKDIR /usr/mrd
+ENTRYPOINT ["/bin/sh","/usr/mrd/mapper.sh"]

data/testdata/Dockerfile.hello.reducer

@@ -0,0 +1,4 @@
+FROM contained_mrtests/base.hello
+COPY . /usr/mrd/
+WORKDIR /
+ENTRYPOINT ["/bin/sh","/usr/mrd/reducer.sh"]

data/testdata/hello/Dockerfile

@@ -0,0 +1,4 @@
+FROM busybox
+
+RUN mkdir -p /usr/mrd && chmod 777 /usr/mrd
+COPY . /usr/mrd/

data/testdata/hello/data/goodbye.txt

@@ -0,0 +1 @@
+Goodbye, cruel world!

data/testdata/hello/data/hello.txt

@@ -0,0 +1 @@
+Hello world!

data/testdata/hello/mapper.sh

@@ -0,0 +1,9 @@
+echo $ITEM > map-output
+cat map-output map-input data/hello.txt
+>&2 echo $ITEM $ITEMS
+if [ "$ITEM" -eq "3" ]; then
+  exit 42
+fi
+if [ "$ITEM" -eq "1" ]; then
+  sleep 100
+fi

data/testdata/hello/mapreduced.yml

@@ -0,0 +1,18 @@
+---
+items: 3
+mapper:
+  input: /usr/mrd/map-input
+  output: /usr/mrd/map-output
+  chdir: /usr/mrd
+  env: ITEM
+  cmd:
+    - /bin/sh
+    - /usr/mrd/mapper.sh
+reducer:
+  input: /usr/mrd/
+  output: /usr/mrd/reduce-output
+  chdir: /
+  env: ITEMS
+  cmd:
+    - /bin/sh
+    - /usr/mrd/reducer.sh

data/testdata/hello/reducer.sh

@@ -0,0 +1,10 @@
+>&2 echo $ITEMS $PWD
+cd /usr/mrd
+cat 1.out 2.out 3.out > reduce-output
+cat 1.stdout 2.stdout 3.stdout >> reduce-output
+cat 1.stderr 2.stderr 3.stderr >> reduce-output
+cat 1.json
+echo
+cat 2.json
+echo
+cat 3.json

data/testdata/input.hello

@@ -0,0 +1 @@
+mapper input file

data/testdata/job.hello

@@ -0,0 +1,10 @@
+{
+  "mapper": {
+    "wait_time": 2.5,
+    "ulimits": { "cpu": 3, "rss": 1000000 }
+  },
+  "reducer": {
+    "wait_time": 2,
+    "ulimits": { "cpu": 2, "rss": 100000 }
+  }
+}

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification
+name: contained_mr
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Victor Costan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.22.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.22.3
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.7
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Plumbing for running mappers and reducers inside Docker containers
+email: victor@costan.us
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.rdoc
+files:
+- ".document"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/contained_mr.rb
+- lib/contained_mr/cleaner.rb
+- lib/contained_mr/job.rb
+- lib/contained_mr/runner.rb
+- lib/contained_mr/template.rb
+- test/helper.rb
+- test/test_job.rb
+- test/test_runner.rb
+- test/test_template.rb
+- testdata/Dockerfile.hello.mapper
+- testdata/Dockerfile.hello.reducer
+- testdata/hello/Dockerfile
+- testdata/hello/data/goodbye.txt
+- testdata/hello/data/hello.txt
+- testdata/hello/mapper.sh
+- testdata/hello/mapreduced.yml
+- testdata/hello/reducer.sh
+- testdata/input.hello
+- testdata/job.hello
+homepage: http://github.com/pwnall/contained_mr
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Map-Reduce with Docker containers
+test_files: []