contained_mr 0.1.2 → 0.2.0

data/lib/contained_mr/mock/runner.rb

@@ -0,0 +1,86 @@
+# @see {ContainedMr::Runner}
+class ContainedMr::Mock::Runner
+  # @return {Hash<String, Object>} the options passed to the constructor
+  attr_reader :_container_options
+  # @return {Hash<String, Object>} the time limit passed to the constructor
+  attr_reader :_time_limit
+  # @return {Hash<String, Object>} the output path passed to the constructor
+  attr_reader :_output_path
+
+  # @return {Boolean} true if {#perform} was called
+  def performed?
+    @performed
+  end
+
+  # @return {Boolean} true if {#destroy!} was called
+  def destroyed?
+    @destroyed
+  end
+
+  include ContainedMr::RunnerLogic
+
+  # @see {ContainedMr::Runner#initialize}
+  def initialize(container_options, time_limit, output_path)
+    @_container_options = container_options
+    @_time_limit = time_limit
+    @_output_path = output_path
+
+    @container_id = nil
+    @started_at = @ended_at = nil
+    @status_code = nil
+    @timed_out = nil
+    @stdout = @stderr = nil
+    @output = nil
+
+    @performed = false
+    @destroyed = false
+  end
+
+  # @see {ContainedMr::Runner#perform}
+  def perform
+    @performed = true
+    self
+  end
+
+  # @see {ContainedMr::Runner#destroy!}
+  def destroy!
+    @destroyed = true
+    self
+  end
+
+  # Sets the container execution data returned by the mock.
+  #
+  # @param {Hash<Symbol, Object>} attributes values describing the result of
+  #   running the job
+  # @return {ContainedMr::Runner} self
+  def _mock_set(attributes)
+    @started_at = attributes[:started_at]
+    @ended_at = attributes[:ended_at]
+    @status_code = attributes[:status_code]
+    @timed_out = attributes[:timed_out]
+    @stdout = attributes[:stdout]
+    @stderr = attributes[:stderr]
+    @output = attributes[:output]
+    self
+  end
+
+  # Convenience method for looking up an ulimit in the container options.
+  #
+  # @param {String} name the ulimit's name, such as 'cpu' or 'rss'
+  # @return {Number} the ulimit's hard and soft value, or nil if the ulimit was
+  #   not found
+  # @raise {RuntimeError} if the ulimit's hard and soft values don't match
+  def _ulimit(name)
+    return nil unless ulimits = @_container_options['Ulimits']
+
+    ulimits.each do |ulimit|
+      if ulimit['Name'] == name
+        if ulimit['Hard'] != ulimit['Soft']
+          raise RuntimeError, "Hard/soft ulimit mismatch for #{name}"
+        end
+        return ulimit['Hard']
+      end
+    end
+    nil
+  end
+end

data/lib/contained_mr/mock/template.rb

@@ -0,0 +1,62 @@
+# @see {ContainedMr::Template}
+class ContainedMr::Mock::Template
+  # @return {Hash<String, Object>} YAML-parsed mapreduced.yml
+  attr_reader :_definition
+
+  # @return {Hash<String, Symbol|String>} maps file names in the template .zip
+  #   to their contents, and maps directory entries to the :directory symbol
+  attr_reader :_zip_contents
+
+  include ContainedMr::TemplateLogic
+
+  # @return {Boolean} true if {#destroy!} was called
+  def destroyed?
+    @destroyed
+  end
+
+  # @see {ContainedMr::Template#initialize}
+  def initialize(name_prefix, id, zip_io)
+    @name_prefix = name_prefix
+    @id = id
+    @image_id = 'mock-template-image-id'
+    @item_count = nil
+    @_definition = nil
+
+    @destroyed = false
+    @_zip_contents = {}
+
+    process_zip zip_io
+  end
+
+  # @see {ContainedMr::Template#destroy!}
+  def destroy!
+    @destroyed = true
+    self
+  end
+
+  # @see {ContainedMr::Template#new_job}
+  def job_class
+    ContainedMr::Mock::Job
+  end
+
+  # Reads the template .zip and parses the definition.
+  def process_zip(zip_io)
+    # TODO(pwnall): zip_io.read -> zip_io after rubyzip releases 1.1.8
+    Zip::File.open_buffer zip_io.read do |zip|
+      zip.each do |zip_entry|
+        file_name = zip_entry.name
+        if zip_entry.directory?
+          @_zip_contents[file_name] = :directory
+        elsif zip_entry.file?
+          if file_name == 'mapreduced.yml'
+            read_definition zip_entry.get_input_stream
+            next
+          end
+          @_zip_contents[file_name] = zip_entry.get_input_stream.read
+        end
+      end
+    end
+    @_zip_contents.freeze
+  end
+  private :process_zip
+end

data/lib/contained_mr/mock.rb

@@ -0,0 +1,3 @@
+# Namespace for mock classes that facilitate writing tests against ContainedMr.
+module ContainedMr::Mock
+end

data/lib/contained_mr/namespace.rb

@@ -0,0 +1,24 @@
+# Namespace and factory for templates.
+module ContainedMr
+  # Sets up the template and builds its Docker base image.
+  #
+  # This method should be used instead of calling {ContainedMr::Template.new}
+  # directly. This way, tests can stub {ContainedMr.template_class} to have it
+  # return {ContainedMr::Mock::Template}.
+  #
+  # @param {String} name_prefix prepended to Docker objects, for identification
+  #   purposes
+  # @param {String} id the template's unique identifier
+  # @param {String} zip_io IO implementation that produces the template .zip
+  def self.new_template(name_prefix, id, zip_io)
+    template_class.new name_prefix, id, zip_io
+  end
+
+  # The class instantiated by {ContainedMr.new_template}.
+  #
+  # @return {Class} by default {ContainedMr::Template}; tests should stub this
+  #   method and have it return {ContainedMr::Mock::Template}
+  def self.template_class
+    ContainedMr::Template
+  end
+end

data/lib/contained_mr/runner.rb

@@ -5,15 +5,20 @@ require 'docker'
 
 # Handles running a single mapper or reducer.
 class ContainedMr::Runner
-  attr_reader :container_id
-  attr_reader :started_at, :ended_at, :status_code, :timed_out
-  attr_reader :stdout, :stderr, :output
+  include ContainedMr::RunnerLogic
 
-  # C
+  # Initialize a runner.
+  #
+  # @param {Hash<String, Object>} container_options docker container creation
+  #   options, passed to {Docker::Container.create} without modification
+  # @param {Number} time_limit maximum number of seconds that the runner's
+  #   container is allowed to execute before being terminated
+  # @param {String} output_path the location of the file inside the container
+  #   whose output will be saved
   def initialize(container_options, time_limit, output_path)
-    @container_options = container_options
-    @time_limit = time_limit
-    @output_path = output_path
+    @_container_options = container_options
+    @_time_limit = time_limit
+    @_output_path = output_path
 
     @container_id = nil
     @started_at = @ended_at = nil
@@ -23,8 +28,9 @@ class ContainedMr::Runner
     @output = nil
   end
 
-
   # Performs a full mapper / reducer step.
+  #
+  # @return {ContainedMr::Runner} self
   def perform
     container = create
     @container_id = container.id
@@ -32,20 +38,29 @@ class ContainedMr::Runner
     execute container
     fetch_console_output container
     fetch_file_output container
-    destroy container
-    self
+    destroy! container
   end
 
-  # @return {Number} the container's running time, in seconds
-  def ran_for
-    @started_at && @ended_at && (@ended_at - @started_at)
+  # Removes the container used to run a mapper / reducer.
+  #
+  # @param {Docker::Container} container the mapper / reducer's container; if
+  #   not supplied, an extra Docker API query is performed to obtain the
+  #   container object
+  # @return {ContainedMr::Runner} self
+  def destroy!(container = nil)
+    unless @container_id.nil?
+      container ||= Docker::Container.get @container_id
+      container.delete force: true
+      @container_id = nil
+    end
+    self
   end
 
   # Creates a container for running a mapper or reducer.
   #
   # @return {Docker::Container} newly created container
   def create
-    Docker::Container.create @container_options
+    Docker::Container.create @_container_options
   end
   private :create
 
@@ -56,7 +71,7 @@ class ContainedMr::Runner
     container.start
     @started_at = Time.now
     begin
-      wait_status = container.wait @time_limit
+      wait_status = container.wait @_time_limit
       @status_code = wait_status['StatusCode']
       @timed_out = false
     rescue Docker::Error::TimeoutError
@@ -107,19 +122,11 @@ class ContainedMr::Runner
   # @return {IO} an IO implementation that sources the .tar data
   def fetch_tar_output(container)
     tar_buffer = StringIO.new
-    container.copy @output_path do |data|
+    container.copy @_output_path do |data|
       tar_buffer << data
     end
     tar_buffer.rewind
     tar_buffer
   end
   private :fetch_tar_output
-
-  # Removes the container used to run a mapper / reducer.
-  #
-  # @param {Docker::Container} container the mapper / reducer's container
-  def destroy(container)
-    container.delete
-    @container_id = nil
-  end
 end

data/lib/contained_mr/runner_logic.rb

@@ -0,0 +1,38 @@
+# Logic shared by {ContainedMr::Runner} and {ContainedMr::Mock::Runner}.
+module ContainedMr::RunnerLogic
+  # @return {Time} the time when the mapper or reducer starts running
+  attr_reader :started_at
+  # @return {Time} the time when the mapper or reducer stops running or is killed
+  attr_reader :ended_at
+  # @return {Number} the time
+  attr_reader :status_code
+  # @return {Boolean} true if the mapper or reducer was terminated due to
+  #   running for too long
+  attr_reader :timed_out
+
+  # @return {String} the data written by the mapper or reducer to stdout
+  attr_reader :stdout
+  # @return {String} the data written by the mapper or reducer to stderr
+  attr_reader :stderr
+  # @return {String} the contents of the file
+  attr_reader :output
+
+  # @return {String} the unique ID of the Docker container used to run the
+  #   mapper / reducer; this is nil
+  attr_reader :container_id
+
+  # @return {Number} the container's running time, in seconds
+  def ran_for
+    started_at && ended_at && (ended_at - started_at)
+  end
+
+  # The information written to the mapper status files given to the reducer.
+  #
+  # This is saved in files named 1.json, 2.json, ... provided to the reducer.
+  #
+  # @return {Hash<Symbol, Object>} JSON-compatible representation of the
+  #   runner's information
+  def json_file
+    { ran_for: ran_for, exit_code: status_code, timed_out: timed_out }
+  end
+end

data/lib/contained_mr/template.rb

@@ -7,20 +7,15 @@ require 'zip'
 
 # A template is used to spawn multiple Map-Reduce jobs.
 class ContainedMr::Template
-  attr_reader :name_prefix, :item_count, :image_id
+  include ContainedMr::TemplateLogic
 
-  # Sets up the template and builds its Docker base image.
-  #
-  # @param {String} name_prefix prepended to Docker objects, for identification
-  #   purposes
-  # @param {String} id the job's unique identifier
-  # @param {String} zip_io IO implementation that produces the template .zip
+  # @see {ContainedMr.new_template}
   def initialize(name_prefix, id, zip_io)
     @name_prefix = name_prefix
     @id = id
     @image_id = nil
-    @definition = nil
     @item_count = nil
+    @_definition = nil
 
     tar_buffer = StringIO.new
     process_zip zip_io, tar_buffer
@@ -31,6 +26,8 @@ class ContainedMr::Template
   # Tears down the template's state.
   #
   # This removes the template's base Docker image.
+  #
+  # @return {ContainedMr::Template} self
   def destroy!
     unless @image_id.nil?
       # HACK(pwnall): Trick docker-api into issuing a DELETE request by tag.
@@ -38,63 +35,17 @@ class ContainedMr::Template
       image.remove
       @image_id = nil
     end
+    self
   end
 
-  # Computes the Dockerfile used to build a job's mapper image.
-  #
-  # @return {String} the Dockerfile
-  def mapper_dockerfile
-    job_dockerfile @definition['mapper'] || {}, 'input'
-  end
-
-  # Computes the Dockerfile used to build a job's reducer image.
-  #
-  # @return {String} the Dockerfile
-  def reducer_dockerfile
-    job_dockerfile @definition['reducer'] || {}, '.'
-  end
-
-  # @return {String} tag applied to the template's base Docker image
-  def image_tag
-    "#{@name_prefix}/base.#{@id}"
-  end
-
-  # Computes the environment variables to be set in a mapper container.
+  # The class instantiated by {ContainedMr::TemplateLogic#new_job}.
   #
-  # @param {Number} i the mapper number
-  # @return {Array<String>} environment variables to be set in the mapper
-  def mapper_env(i)
-    [ "ITEM=#{i}", "ITEMS=#{@item_count.to_s}" ]
+  # @return {Class} by default {ContainedMr::Job}; tests might want to stub
+  #   this method and have it return {ContainedMr::Mock::Job}
+  def job_class
+    ContainedMr::Job
   end
 
-  # Computes the environment variables to be set in the reducer container.
-  #
-  # @return {Array<String>} environment variables to be set in the mapper
-  def reducer_env
-    [ "ITEMS=#{@item_count.to_s}" ]
-  end
-
-  # @return {String} the map output's path in the mapper Docker container
-  def mapper_output_path
-    (@definition['mapper'] || {})['output'] || '/output'
-  end
-
-  # @return {String} the reducer output's path in the reducer Docker container
-  def reducer_output_path
-    (@definition['reducer'] || {})['output'] || '/output'
-  end
-
-  # @private common code from mapper_dockerfile and reducer_dockerfile
-  def job_dockerfile(job_definition, input_source)
-    <<DOCKER_END
-FROM #{@image_id}
-COPY #{input_source} #{job_definition['input'] || '/input'}
-WORKDIR #{job_definition['chdir'] || '/'}
-ENTRYPOINT #{JSON.dump(job_definition['cmd'] || ['/bin/sh'])}
-DOCKER_END
-  end
-  private :job_dockerfile
-
   # Reads the template .zip and parses the definition.
   #
   # @param {IO} zip_io IO implementation that produces the .zip file
@@ -120,17 +71,7 @@ DOCKER_END
       end
     end
   end
-
-  # Reads the template's definition, using data at the given path.
-  #
-  # @param {IO} yaml_io IO implementation that produces the .yaml file
-  #   containing the definition
-  def read_definition(yaml_io)
-    @definition = YAML.load yaml_io.read
-
-    @item_count = @definition['items'] || 1
-  end
-  private :read_definition
+  private :process_zip
 
   # Builds the template's Docker image, using data at the given path.
   #

data/lib/contained_mr/template_logic.rb

@@ -0,0 +1,91 @@
+# Logic shared by {ContainedMr::Template} and {ContainedMr::Mock::Template}.
+module ContainedMr::TemplateLogic
+  # @return {String} prepended to Docker objects, for identification purposes
+  attr_reader :name_prefix
+
+  # @return {String} the template's unique identifier
+  attr_reader :id
+
+  # @return {Number} the number of mapper jobs specified by this template
+  attr_reader :item_count
+
+  # @return {String} image_id the unique ID of the Docker image used as a base
+  #   for images built by jobs derived from this template
+  attr_reader :image_id
+
+  # Creates a job using this template.
+  #
+  # @param {String} id the job's unique ID
+  # @param {Hash<String, Object>} json_options job options, extracted from JSON
+  # @return {ContainedMr::Job} a newly created job that uses this template
+  def new_job(id, json_options)
+    job_class.new self, id, json_options
+  end
+
+  # Computes the Dockerfile used to build a job's mapper image.
+  #
+  # @return {String} the Dockerfile
+  def mapper_dockerfile
+    job_dockerfile @_definition['mapper'] || {}, 'input'
+  end
+
+  # Computes the Dockerfile used to build a job's reducer image.
+  #
+  # @return {String} the Dockerfile
+  def reducer_dockerfile
+    job_dockerfile @_definition['reducer'] || {}, '.'
+  end
+
+  # @return {String} tag applied to the template's base Docker image
+  def image_tag
+    "#{@name_prefix}/base.#{@id}"
+  end
+
+  # Computes the environment variables to be set in a mapper container.
+  #
+  # @param {Number} i the mapper number
+  # @return {Array<String>} environment variables to be set in the mapper
+  def mapper_env(i)
+    [ "ITEM=#{i}", "ITEMS=#{@item_count.to_s}" ]
+  end
+
+  # Computes the environment variables to be set in the reducer container.
+  #
+  # @return {Array<String>} environment variables to be set in the mapper
+  def reducer_env
+    [ "ITEMS=#{@item_count.to_s}" ]
+  end
+
+  # @return {String} the map output's path in the mapper Docker container
+  def mapper_output_path
+    (@_definition['mapper'] || {})['output'] || '/output'
+  end
+
+  # @return {String} the reducer output's path in the reducer Docker container
+  def reducer_output_path
+    (@_definition['reducer'] || {})['output'] || '/output'
+  end
+
+  # @private common code from mapper_dockerfile and reducer_dockerfile
+  def job_dockerfile(job_definition, input_source)
+    <<DOCKER_END
+FROM #{@image_id}
+COPY #{input_source} #{job_definition['input'] || '/input'}
+WORKDIR #{job_definition['chdir'] || '/'}
+ENTRYPOINT #{JSON.dump(job_definition['cmd'] || ['/bin/sh'])}
+DOCKER_END
+  end
+  private :job_dockerfile
+
+  # Reads the template's definition, using data at the given path.
+  #
+  # @param {IO} yaml_io IO implementation that produces the .yaml file
+  #   containing the definition
+  def read_definition(yaml_io)
+    @_definition = YAML.load yaml_io.read
+    @_definition.freeze
+
+    @item_count = @_definition['items'] || 1
+  end
+  private :read_definition
+end

data/lib/contained_mr.rb

@@ -1,8 +1,15 @@
-# Namespace.
-module ContainedMr
-end
+require_relative 'contained_mr/namespace.rb'
+
+require_relative 'contained_mr/job_logic.rb'
+require_relative 'contained_mr/runner_logic.rb'
+require_relative 'contained_mr/template_logic.rb'
 
 require_relative 'contained_mr/cleaner.rb'
 require_relative 'contained_mr/job.rb'
 require_relative 'contained_mr/runner.rb'
 require_relative 'contained_mr/template.rb'
+
+require_relative 'contained_mr/mock.rb'
+require_relative 'contained_mr/mock/job.rb'
+require_relative 'contained_mr/mock/runner.rb'
+require_relative 'contained_mr/mock/template.rb'

data/test/concerns/job_state_cases.rb

@@ -0,0 +1,75 @@
+module JobStateCases
+  def test_build_mapper_image_twice
+    @job.build_mapper_image File.read('testdata/input.hello')
+    begin
+      @job.build_mapper_image File.read('testdata/input.hello')
+      flunk 'No exception thrown'
+    rescue RuntimeError => e
+      assert_instance_of RuntimeError, e
+      assert_equal 'Mapper image already exists', e.message
+    end
+  end
+
+  def test_run_mapper_without_image
+    begin
+      @job.run_mapper 1
+      flunk 'No exception thrown'
+    rescue RuntimeError => e
+      assert_instance_of RuntimeError, e
+      assert_equal 'Mapper image does not exist', e.message
+    end
+  end
+
+  def test_run_invalid_mapper
+    begin
+      @job.run_mapper 4
+      flunk 'No exception thrown'
+    rescue ArgumentError => e
+      assert_instance_of ArgumentError, e
+      assert_equal 'Invalid mapper number 4', e.message
+    end
+  end
+
+  def test_invalid_mapper_runner
+    begin
+      @job.mapper_runner 4
+      flunk 'No exception thrown'
+    rescue ArgumentError => e
+      assert_instance_of ArgumentError, e
+      assert_equal 'Invalid mapper number 4', e.message
+    end
+  end
+
+  def test_build_reducer_image_without_mapper_results
+    begin
+      @job.build_reducer_image
+      flunk 'No exception thrown'
+    rescue RuntimeError => e
+      assert_instance_of RuntimeError, e
+      assert_equal 'Not all mappers ran', e.message
+    end
+  end
+
+  def test_build_reducer_image_twice
+    @job.build_mapper_image File.read('testdata/input.hello')
+    1.upto(3) { |i| @job.run_mapper i }
+    @job.build_reducer_image
+    begin
+      @job.build_reducer_image
+      flunk 'No exception thrown'
+    rescue RuntimeError => e
+      assert_instance_of RuntimeError, e
+      assert_equal 'Reducer image already exists', e.message
+    end
+  end
+
+  def test_run_reducer_without_image
+    begin
+      @job.run_reducer
+      flunk 'No exception thrown'
+    rescue RuntimeError => e
+      assert_instance_of RuntimeError, e
+      assert_equal 'Reducer image does not exist', e.message
+    end
+  end
+end

data/test/helper.rb

@@ -24,6 +24,7 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'minitest/autorun'
+require 'mocha/mini_test'
 
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))

data/test/test_cleaner.rb

@@ -2,10 +2,10 @@ require 'helper'
 
 class TestCleaner < MiniTest::Test
   def setup
-    @template = ContainedMr::Template.new 'contained_mrtests', 'hello',
+    @template = ContainedMr.new_template 'contained_mrtests', 'hello',
         StringIO.new(File.binread('testdata/hello.zip'))
-    @job = ContainedMr::Job.new @template, 'testjob',
-                                JSON.load(File.read('testdata/job.hello'))
+    @job = @template.new_job 'testjob',
+        JSON.load(File.read('testdata/job.hello'))
     @job.build_mapper_image File.read('testdata/input.hello')
 
     @cleaner = ContainedMr::Cleaner.new 'contained_mrtests'
@@ -22,10 +22,10 @@ class TestCleaner < MiniTest::Test
   end
 
   def test_destroy_all_with_duplicates
-    template2 = ContainedMr::Template.new 'contained_mrtests', 'hello2',
+    template2 = ContainedMr.new_template 'contained_mrtests', 'hello2',
         StringIO.new(File.binread('testdata/hello.zip'))
-    job2 = ContainedMr::Job.new template2, 'testjob2',
-                                JSON.load(File.read('testdata/job.hello'))
+    job2 = template2.new_job 'testjob2',
+        JSON.load(File.read('testdata/job.hello'))
     job2.build_mapper_image File.read('testdata/input.hello')
     @cleaner.destroy_all!
     assert_raises Docker::Error::NotFoundError do

data/test/test_contained_mr.rb

@@ -0,0 +1,15 @@
+require 'helper'
+
+class TestContainedMr < MiniTest::Test
+  def test_template_class
+    assert_equal ContainedMr::Template, ContainedMr.template_class
+  end
+
+  def test_new_template
+    ContainedMr.stubs(:template_class).returns ContainedMr::Mock::Template
+
+    template = ContainedMr.new_template 'contained_mrtests', 'hello',
+        StringIO.new(File.binread('testdata/hello.zip'))
+    assert_instance_of ContainedMr::Mock::Template, template
+  end
+end